| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 200, |
| "global_step": 3884, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.0695502400398254, |
| "epoch": 0.0025749967812540232, |
| "grad_norm": 58.40303421020508, |
| "learning_rate": 3.846153846153846e-08, |
| "logits/chosen": 4.762095425386862, |
| "logits/rejected": 5.165317430713165, |
| "logps/chosen": -326.04670944213865, |
| "logps/rejected": -287.5275459289551, |
| "loss": 0.7019648075103759, |
| "mean_token_accuracy": 0.7099196255207062, |
| "num_tokens": 138906.0, |
| "rewards/accuracies": 0.44375, |
| "rewards/chosen": -0.014671237254515291, |
| "rewards/margins": -0.013322968035936356, |
| "rewards/rejected": -0.0013482690206728875, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.051201492547989, |
| "epoch": 0.0051499935625080465, |
| "grad_norm": 60.10802459716797, |
| "learning_rate": 8.119658119658119e-08, |
| "logits/chosen": 5.049522462404801, |
| "logits/rejected": 5.092235104276472, |
| "logps/chosen": -330.7694374084473, |
| "logps/rejected": -300.17331314086914, |
| "loss": 0.689347505569458, |
| "mean_token_accuracy": 0.7070325821638107, |
| "num_tokens": 280553.0, |
| "rewards/accuracies": 0.575, |
| "rewards/chosen": 0.006806808104738593, |
| "rewards/margins": 0.011950487177819014, |
| "rewards/rejected": -0.005143680749461055, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.111130267381668, |
| "epoch": 0.00772499034376207, |
| "grad_norm": 64.04685974121094, |
| "learning_rate": 1.2393162393162394e-07, |
| "logits/chosen": 5.153677836715249, |
| "logits/rejected": 5.129135751846538, |
| "logps/chosen": -318.63440856933596, |
| "logps/rejected": -291.53640899658205, |
| "loss": 0.702676248550415, |
| "mean_token_accuracy": 0.6977999448776245, |
| "num_tokens": 402696.0, |
| "rewards/accuracies": 0.51875, |
| "rewards/chosen": 0.002496657450683415, |
| "rewards/margins": -0.014800068037584425, |
| "rewards/rejected": 0.017296723928302525, |
| "step": 30 |
| }, |
| { |
| "entropy": 1.0413395404815673, |
| "epoch": 0.010299987125016093, |
| "grad_norm": 61.53692626953125, |
| "learning_rate": 1.6666666666666665e-07, |
| "logits/chosen": 4.89737151113117, |
| "logits/rejected": 4.989192161185598, |
| "logps/chosen": -317.6526679992676, |
| "logps/rejected": -299.5892158508301, |
| "loss": 0.6864168167114257, |
| "mean_token_accuracy": 0.7229504019021988, |
| "num_tokens": 535659.0, |
| "rewards/accuracies": 0.54375, |
| "rewards/chosen": 0.005982771259732544, |
| "rewards/margins": 0.01693534429650754, |
| "rewards/rejected": -0.010952573386020958, |
| "step": 40 |
| }, |
| { |
| "entropy": 1.0844607055187225, |
| "epoch": 0.012874983906270118, |
| "grad_norm": 61.18706512451172, |
| "learning_rate": 2.0940170940170939e-07, |
| "logits/chosen": 5.037751540173592, |
| "logits/rejected": 5.022813626757577, |
| "logps/chosen": -335.12218589782714, |
| "logps/rejected": -308.31434173583983, |
| "loss": 0.6873773574829102, |
| "mean_token_accuracy": 0.7006662994623184, |
| "num_tokens": 667089.0, |
| "rewards/accuracies": 0.51875, |
| "rewards/chosen": 0.01746939681470394, |
| "rewards/margins": 0.016403451142832638, |
| "rewards/rejected": 0.0010659459047019481, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.0144690483808518, |
| "epoch": 0.01544998068752414, |
| "grad_norm": 75.81221771240234, |
| "learning_rate": 2.521367521367521e-07, |
| "logits/chosen": 4.896282725285022, |
| "logits/rejected": 4.959213862117638, |
| "logps/chosen": -303.1107292175293, |
| "logps/rejected": -293.32022323608396, |
| "loss": 0.6931710720062256, |
| "mean_token_accuracy": 0.7159741580486297, |
| "num_tokens": 810423.0, |
| "rewards/accuracies": 0.4875, |
| "rewards/chosen": 0.004395447322167456, |
| "rewards/margins": 0.0054800987243652345, |
| "rewards/rejected": -0.0010846516583114863, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.9541261970996857, |
| "epoch": 0.018024977468778163, |
| "grad_norm": 73.47660827636719, |
| "learning_rate": 2.948717948717949e-07, |
| "logits/chosen": 4.878544510221824, |
| "logits/rejected": 5.001038510432101, |
| "logps/chosen": -328.04087524414064, |
| "logps/rejected": -302.33535919189455, |
| "loss": 0.6965915203094483, |
| "mean_token_accuracy": 0.7324783325195312, |
| "num_tokens": 954004.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.003411970962770283, |
| "rewards/margins": -0.00023952401243150235, |
| "rewards/rejected": 0.0036514950916171074, |
| "step": 70 |
| }, |
| { |
| "entropy": 1.0023272186517715, |
| "epoch": 0.020599974250032186, |
| "grad_norm": 84.27365112304688, |
| "learning_rate": 3.376068376068376e-07, |
| "logits/chosen": 4.891619277720404, |
| "logits/rejected": 4.941218434899946, |
| "logps/chosen": -382.654524230957, |
| "logps/rejected": -341.0330238342285, |
| "loss": 0.6920312881469727, |
| "mean_token_accuracy": 0.7118763566017151, |
| "num_tokens": 1108886.0, |
| "rewards/accuracies": 0.55625, |
| "rewards/chosen": 0.02751660463400185, |
| "rewards/margins": 0.008006168599240481, |
| "rewards/rejected": 0.01951043652370572, |
| "step": 80 |
| }, |
| { |
| "entropy": 1.0618479609489442, |
| "epoch": 0.023174971031286212, |
| "grad_norm": 70.65860748291016, |
| "learning_rate": 3.803418803418803e-07, |
| "logits/chosen": 4.614978686245202, |
| "logits/rejected": 4.850742337522037, |
| "logps/chosen": -320.1764625549316, |
| "logps/rejected": -306.739794921875, |
| "loss": 0.6981391906738281, |
| "mean_token_accuracy": 0.7010324835777283, |
| "num_tokens": 1240538.0, |
| "rewards/accuracies": 0.48125, |
| "rewards/chosen": 0.02626001196913421, |
| "rewards/margins": -0.004135513259097934, |
| "rewards/rejected": 0.03039552546106279, |
| "step": 90 |
| }, |
| { |
| "entropy": 1.0384016394615174, |
| "epoch": 0.025749967812540235, |
| "grad_norm": 59.127098083496094, |
| "learning_rate": 4.2307692307692304e-07, |
| "logits/chosen": 4.8460533212294745, |
| "logits/rejected": 4.824443955007636, |
| "logps/chosen": -348.85643463134767, |
| "logps/rejected": -312.45501098632815, |
| "loss": 0.6569533348083496, |
| "mean_token_accuracy": 0.7068787574768066, |
| "num_tokens": 1382625.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.08904398875311018, |
| "rewards/margins": 0.08204479101113975, |
| "rewards/rejected": 0.0069991969503462315, |
| "step": 100 |
| }, |
| { |
| "entropy": 1.0480757176876068, |
| "epoch": 0.028324964593794258, |
| "grad_norm": 64.17548370361328, |
| "learning_rate": 4.658119658119658e-07, |
| "logits/chosen": 4.7895138574416185, |
| "logits/rejected": 4.869067111731783, |
| "logps/chosen": -348.2337989807129, |
| "logps/rejected": -308.14727478027345, |
| "loss": 0.6725531578063965, |
| "mean_token_accuracy": 0.7132146149873734, |
| "num_tokens": 1514173.0, |
| "rewards/accuracies": 0.6, |
| "rewards/chosen": 0.12068244460970164, |
| "rewards/margins": 0.05134738911874592, |
| "rewards/rejected": 0.06933505749329924, |
| "step": 110 |
| }, |
| { |
| "entropy": 1.0579831272363662, |
| "epoch": 0.03089996137504828, |
| "grad_norm": 74.17851257324219, |
| "learning_rate": 4.999996522409758e-07, |
| "logits/chosen": 4.79368538551119, |
| "logits/rejected": 4.86757195450835, |
| "logps/chosen": -369.29692764282225, |
| "logps/rejected": -331.0557464599609, |
| "loss": 0.6710665702819825, |
| "mean_token_accuracy": 0.7087177097797394, |
| "num_tokens": 1660343.0, |
| "rewards/accuracies": 0.6375, |
| "rewards/chosen": 0.1534318385645747, |
| "rewards/margins": 0.06275590499863029, |
| "rewards/rejected": 0.09067593847867102, |
| "step": 120 |
| }, |
| { |
| "entropy": 1.0356961339712143, |
| "epoch": 0.033474958156302304, |
| "grad_norm": 59.33082580566406, |
| "learning_rate": 4.999874807767132e-07, |
| "logits/chosen": 4.807214629060647, |
| "logits/rejected": 4.723712813209219, |
| "logps/chosen": -326.1460441589355, |
| "logps/rejected": -299.46656799316406, |
| "loss": 0.6591967582702637, |
| "mean_token_accuracy": 0.7120212823152542, |
| "num_tokens": 1802352.0, |
| "rewards/accuracies": 0.6, |
| "rewards/chosen": 0.16570830587297677, |
| "rewards/margins": 0.08986456668935716, |
| "rewards/rejected": 0.07584373848512768, |
| "step": 130 |
| }, |
| { |
| "entropy": 1.0639059126377106, |
| "epoch": 0.036049954937556326, |
| "grad_norm": 46.880950927734375, |
| "learning_rate": 4.999579223287168e-07, |
| "logits/chosen": 4.783742518251371, |
| "logits/rejected": 4.773145675630588, |
| "logps/chosen": -344.53967132568357, |
| "logps/rejected": -279.5810417175293, |
| "loss": 0.6366758346557617, |
| "mean_token_accuracy": 0.7052115440368653, |
| "num_tokens": 1935694.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.20478311218321324, |
| "rewards/margins": 0.13913744706660508, |
| "rewards/rejected": 0.06564566306769848, |
| "step": 140 |
| }, |
| { |
| "entropy": 1.019757255911827, |
| "epoch": 0.03862495171881035, |
| "grad_norm": 61.628849029541016, |
| "learning_rate": 4.999109789528186e-07, |
| "logits/chosen": 4.742164159736975, |
| "logits/rejected": 4.946638459164037, |
| "logps/chosen": -329.5531692504883, |
| "logps/rejected": -289.46395416259764, |
| "loss": 0.6572101593017579, |
| "mean_token_accuracy": 0.7132875680923462, |
| "num_tokens": 2071315.0, |
| "rewards/accuracies": 0.61875, |
| "rewards/chosen": 0.21405786070972682, |
| "rewards/margins": 0.10857852287590504, |
| "rewards/rejected": 0.1054793362505734, |
| "step": 150 |
| }, |
| { |
| "entropy": 1.0863682091236115, |
| "epoch": 0.04119994850006437, |
| "grad_norm": 52.7701301574707, |
| "learning_rate": 4.998466539139966e-07, |
| "logits/chosen": 4.762784820500906, |
| "logits/rejected": 4.796325654968387, |
| "logps/chosen": -273.65675354003906, |
| "logps/rejected": -278.2464324951172, |
| "loss": 0.6301429271697998, |
| "mean_token_accuracy": 0.7196185618638993, |
| "num_tokens": 2205535.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.22666821889579297, |
| "rewards/margins": 0.16818621791899205, |
| "rewards/rejected": 0.058482004422694446, |
| "step": 160 |
| }, |
| { |
| "entropy": 1.028375071287155, |
| "epoch": 0.0437749452813184, |
| "grad_norm": 56.66053009033203, |
| "learning_rate": 4.997649516861489e-07, |
| "logits/chosen": 4.541193592999439, |
| "logits/rejected": 4.769671591019123, |
| "logps/chosen": -325.7829078674316, |
| "logps/rejected": -333.5553314208984, |
| "loss": 0.6510717391967773, |
| "mean_token_accuracy": 0.7231594532728195, |
| "num_tokens": 2345876.0, |
| "rewards/accuracies": 0.63125, |
| "rewards/chosen": 0.23217825312167406, |
| "rewards/margins": 0.13302348610013723, |
| "rewards/rejected": 0.09915476976893842, |
| "step": 170 |
| }, |
| { |
| "entropy": 1.0339969098567963, |
| "epoch": 0.046349942062572425, |
| "grad_norm": 55.01676559448242, |
| "learning_rate": 4.996658779517809e-07, |
| "logits/chosen": 4.670579991176632, |
| "logits/rejected": 4.806632291205961, |
| "logps/chosen": -300.85853881835936, |
| "logps/rejected": -313.85018997192384, |
| "loss": 0.6131317138671875, |
| "mean_token_accuracy": 0.7170714914798737, |
| "num_tokens": 2486861.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.2972709834575653, |
| "rewards/margins": 0.24978987369686365, |
| "rewards/rejected": 0.04748110845685005, |
| "step": 180 |
| }, |
| { |
| "entropy": 1.0799639374017715, |
| "epoch": 0.04892493884382645, |
| "grad_norm": 47.156986236572266, |
| "learning_rate": 4.995494396016116e-07, |
| "logits/chosen": 4.792294079993604, |
| "logits/rejected": 4.859750700385524, |
| "logps/chosen": -313.53778610229494, |
| "logps/rejected": -324.3677551269531, |
| "loss": 0.6257365703582763, |
| "mean_token_accuracy": 0.7067189812660217, |
| "num_tokens": 2614373.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.29387753307819364, |
| "rewards/margins": 0.20768443727865815, |
| "rewards/rejected": 0.08619309198111295, |
| "step": 190 |
| }, |
| { |
| "entropy": 1.0718048125505448, |
| "epoch": 0.05149993562508047, |
| "grad_norm": 59.598426818847656, |
| "learning_rate": 4.994156447340932e-07, |
| "logits/chosen": 4.765590034251408, |
| "logits/rejected": 4.673178326492318, |
| "logps/chosen": -296.8489875793457, |
| "logps/rejected": -295.27930374145507, |
| "loss": 0.6396026611328125, |
| "mean_token_accuracy": 0.7119928777217865, |
| "num_tokens": 2740292.0, |
| "rewards/accuracies": 0.64375, |
| "rewards/chosen": 0.24728852249681949, |
| "rewards/margins": 0.18662491645663976, |
| "rewards/rejected": 0.06066360287368298, |
| "step": 200 |
| }, |
| { |
| "entropy": 1.123349404335022, |
| "epoch": 0.05407493240633449, |
| "grad_norm": 77.97437286376953, |
| "learning_rate": 4.992645026548485e-07, |
| "logits/chosen": 4.384905356176568, |
| "logits/rejected": 4.5785579199739725, |
| "logps/chosen": -337.41533813476565, |
| "logps/rejected": -335.0264404296875, |
| "loss": 0.5705361843109131, |
| "mean_token_accuracy": 0.6963643610477448, |
| "num_tokens": 2867334.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.471590805798769, |
| "rewards/margins": 0.3648594722151756, |
| "rewards/rejected": 0.10673133693635464, |
| "step": 210 |
| }, |
| { |
| "entropy": 1.1385729312896729, |
| "epoch": 0.056649929187588516, |
| "grad_norm": 51.946537017822266, |
| "learning_rate": 4.990960238760234e-07, |
| "logits/chosen": 4.721716537520671, |
| "logits/rejected": 4.601918369467295, |
| "logps/chosen": -351.25513687133787, |
| "logps/rejected": -302.89658813476564, |
| "loss": 0.5859468460083008, |
| "mean_token_accuracy": 0.6919317305088043, |
| "num_tokens": 2993069.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.39948597028851507, |
| "rewards/margins": 0.37867568135261537, |
| "rewards/rejected": 0.020810306444764136, |
| "step": 220 |
| }, |
| { |
| "entropy": 1.0420648545026778, |
| "epoch": 0.05922492596884254, |
| "grad_norm": 63.54645538330078, |
| "learning_rate": 4.989102201155557e-07, |
| "logits/chosen": 4.890828809020718, |
| "logits/rejected": 4.79452456459243, |
| "logps/chosen": -343.53161773681643, |
| "logps/rejected": -325.0349151611328, |
| "loss": 0.6073165893554687, |
| "mean_token_accuracy": 0.7045987546443939, |
| "num_tokens": 3141079.0, |
| "rewards/accuracies": 0.68125, |
| "rewards/chosen": 0.4077755454927683, |
| "rewards/margins": 0.32392252162098883, |
| "rewards/rejected": 0.08385302387177944, |
| "step": 230 |
| }, |
| { |
| "entropy": 1.0679937094449996, |
| "epoch": 0.06179992275009656, |
| "grad_norm": 52.09357833862305, |
| "learning_rate": 4.987071042963607e-07, |
| "logits/chosen": 4.418793507089909, |
| "logits/rejected": 4.522803089162048, |
| "logps/chosen": -355.0448799133301, |
| "logps/rejected": -339.4919242858887, |
| "loss": 0.5773736953735351, |
| "mean_token_accuracy": 0.704371201992035, |
| "num_tokens": 3286962.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.545511468499899, |
| "rewards/margins": 0.40970403887331486, |
| "rewards/rejected": 0.13580743744969367, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.9754935801029205, |
| "epoch": 0.06437491953135059, |
| "grad_norm": 55.18085861206055, |
| "learning_rate": 4.984866905454313e-07, |
| "logits/chosen": 4.734972345059665, |
| "logits/rejected": 4.7931226182846345, |
| "logps/chosen": -290.1737594604492, |
| "logps/rejected": -316.17198638916017, |
| "loss": 0.6571999073028565, |
| "mean_token_accuracy": 0.728389224410057, |
| "num_tokens": 3426548.0, |
| "rewards/accuracies": 0.58125, |
| "rewards/chosen": 0.4490304827690125, |
| "rewards/margins": 0.2581039540469646, |
| "rewards/rejected": 0.19092654064297676, |
| "step": 250 |
| }, |
| { |
| "entropy": 1.0574446082115174, |
| "epoch": 0.06694991631260461, |
| "grad_norm": 45.543148040771484, |
| "learning_rate": 4.982489941928565e-07, |
| "logits/chosen": 4.519881946840187, |
| "logits/rejected": 4.961670236968278, |
| "logps/chosen": -318.65669174194335, |
| "logps/rejected": -312.55185317993164, |
| "loss": 0.5793586730957031, |
| "mean_token_accuracy": 0.7170014798641204, |
| "num_tokens": 3571612.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.45276781246066095, |
| "rewards/margins": 0.4238900698721409, |
| "rewards/rejected": 0.028877736534923315, |
| "step": 260 |
| }, |
| { |
| "entropy": 1.0360048443078995, |
| "epoch": 0.06952491309385864, |
| "grad_norm": 71.59954071044922, |
| "learning_rate": 4.979940317707547e-07, |
| "logits/chosen": 4.615665879567825, |
| "logits/rejected": 4.5951095572387946, |
| "logps/chosen": -347.95579528808594, |
| "logps/rejected": -298.5147361755371, |
| "loss": 0.5869906425476075, |
| "mean_token_accuracy": 0.695509958267212, |
| "num_tokens": 3707243.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.44863925948739053, |
| "rewards/margins": 0.3824565585702658, |
| "rewards/rejected": 0.06618269477039576, |
| "step": 270 |
| }, |
| { |
| "entropy": 1.0836758971214295, |
| "epoch": 0.07209990987511265, |
| "grad_norm": 53.48826599121094, |
| "learning_rate": 4.977218210121237e-07, |
| "logits/chosen": 4.381913341564861, |
| "logits/rejected": 4.4011498928625565, |
| "logps/chosen": -347.5009132385254, |
| "logps/rejected": -301.47351303100584, |
| "loss": 0.5558026790618896, |
| "mean_token_accuracy": 0.6883328318595886, |
| "num_tokens": 3836639.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.5033460140228272, |
| "rewards/margins": 0.45913518890738486, |
| "rewards/rejected": 0.044210815243422984, |
| "step": 280 |
| }, |
| { |
| "entropy": 1.019636595249176, |
| "epoch": 0.07467490665636668, |
| "grad_norm": 58.595001220703125, |
| "learning_rate": 4.974323808496079e-07, |
| "logits/chosen": 4.617471123727146, |
| "logits/rejected": 4.759611293000644, |
| "logps/chosen": -324.77361526489256, |
| "logps/rejected": -318.6325256347656, |
| "loss": 0.5850511074066163, |
| "mean_token_accuracy": 0.7116295546293259, |
| "num_tokens": 3967890.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.377638790756464, |
| "rewards/margins": 0.4224536880850792, |
| "rewards/rejected": -0.044814901798963545, |
| "step": 290 |
| }, |
| { |
| "entropy": 1.0210690379142762, |
| "epoch": 0.0772499034376207, |
| "grad_norm": 51.6544189453125, |
| "learning_rate": 4.971257314141808e-07, |
| "logits/chosen": 4.724067558444576, |
| "logits/rejected": 4.807996827326434, |
| "logps/chosen": -347.1739959716797, |
| "logps/rejected": -319.8342315673828, |
| "loss": 0.605327033996582, |
| "mean_token_accuracy": 0.7064412742853164, |
| "num_tokens": 4109051.0, |
| "rewards/accuracies": 0.68125, |
| "rewards/chosen": 0.37119412459433077, |
| "rewards/margins": 0.3749303735792637, |
| "rewards/rejected": -0.003736240416765213, |
| "step": 300 |
| }, |
| { |
| "entropy": 1.0039023548364638, |
| "epoch": 0.07982490021887473, |
| "grad_norm": 101.59368896484375, |
| "learning_rate": 4.968018940337457e-07, |
| "logits/chosen": 4.615459057356064, |
| "logits/rejected": 4.763312215236125, |
| "logps/chosen": -324.7437110900879, |
| "logps/rejected": -302.2521469116211, |
| "loss": 0.6154751777648926, |
| "mean_token_accuracy": 0.7176793605089188, |
| "num_tokens": 4247174.0, |
| "rewards/accuracies": 0.65, |
| "rewards/chosen": 0.4162539541721344, |
| "rewards/margins": 0.39256238043308256, |
| "rewards/rejected": 0.02369157038629055, |
| "step": 310 |
| }, |
| { |
| "entropy": 1.0267376393079757, |
| "epoch": 0.08239989700012874, |
| "grad_norm": 62.431922912597656, |
| "learning_rate": 4.964608912316513e-07, |
| "logits/chosen": 4.860332914675701, |
| "logits/rejected": 4.91242366277054, |
| "logps/chosen": -331.2693698883057, |
| "logps/rejected": -319.29067611694336, |
| "loss": 0.6155694961547852, |
| "mean_token_accuracy": 0.7094589233398437, |
| "num_tokens": 4384787.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.5132191546261311, |
| "rewards/margins": 0.36463063918054106, |
| "rewards/rejected": 0.14858851991593838, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.9868817120790482, |
| "epoch": 0.08497489378138277, |
| "grad_norm": 61.427738189697266, |
| "learning_rate": 4.961027467251261e-07, |
| "logits/chosen": 4.638749258286775, |
| "logits/rejected": 4.5989357091926495, |
| "logps/chosen": -318.4152290344238, |
| "logps/rejected": -305.52441024780273, |
| "loss": 0.604922866821289, |
| "mean_token_accuracy": 0.7185680598020554, |
| "num_tokens": 4523458.0, |
| "rewards/accuracies": 0.68125, |
| "rewards/chosen": 0.5514455199241638, |
| "rewards/margins": 0.40419873856008054, |
| "rewards/rejected": 0.14724678825587034, |
| "step": 330 |
| }, |
| { |
| "entropy": 1.0020043164491654, |
| "epoch": 0.0875498905626368, |
| "grad_norm": 52.15379333496094, |
| "learning_rate": 4.957274854236283e-07, |
| "logits/chosen": 4.362704721137801, |
| "logits/rejected": 4.4858774130028864, |
| "logps/chosen": -298.45718536376955, |
| "logps/rejected": -275.92749252319334, |
| "loss": 0.6286430358886719, |
| "mean_token_accuracy": 0.7173390120267868, |
| "num_tokens": 4655707.0, |
| "rewards/accuracies": 0.6125, |
| "rewards/chosen": 0.5156877748668194, |
| "rewards/margins": 0.3501556381583214, |
| "rewards/rejected": 0.16553213112056256, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.9908010929822921, |
| "epoch": 0.09012488734389082, |
| "grad_norm": 49.259765625, |
| "learning_rate": 4.953351334271134e-07, |
| "logits/chosen": 4.406444560691517, |
| "logits/rejected": 4.548902690320075, |
| "logps/chosen": -296.87431106567385, |
| "logps/rejected": -289.5457908630371, |
| "loss": 0.5424035072326661, |
| "mean_token_accuracy": 0.7123640716075897, |
| "num_tokens": 4790294.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.5531937591731548, |
| "rewards/margins": 0.5116788432002067, |
| "rewards/rejected": 0.04151492342352867, |
| "step": 350 |
| }, |
| { |
| "entropy": 1.0101291090250015, |
| "epoch": 0.09269988412514485, |
| "grad_norm": 59.696929931640625, |
| "learning_rate": 4.949257180242192e-07, |
| "logits/chosen": 4.385342873631919, |
| "logits/rejected": 4.272401614914889, |
| "logps/chosen": -329.9344657897949, |
| "logps/rejected": -303.8744956970215, |
| "loss": 0.5772665023803711, |
| "mean_token_accuracy": 0.7187704712152481, |
| "num_tokens": 4928018.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.44092832542955873, |
| "rewards/margins": 0.43197181150317193, |
| "rewards/rejected": 0.008956506475806236, |
| "step": 360 |
| }, |
| { |
| "entropy": 1.016788786649704, |
| "epoch": 0.09527488090639886, |
| "grad_norm": 50.9133415222168, |
| "learning_rate": 4.944992676903674e-07, |
| "logits/chosen": 4.506991812732053, |
| "logits/rejected": 4.551043911449192, |
| "logps/chosen": -341.2626007080078, |
| "logps/rejected": -321.827197265625, |
| "loss": 0.5512761116027832, |
| "mean_token_accuracy": 0.713101840019226, |
| "num_tokens": 5081614.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.4262397438287735, |
| "rewards/margins": 0.5591157540678978, |
| "rewards/rejected": -0.13287600912153721, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.985381743311882, |
| "epoch": 0.0978498776876529, |
| "grad_norm": 101.32551574707031, |
| "learning_rate": 4.940558120857834e-07, |
| "logits/chosen": 4.556351895973922, |
| "logits/rejected": 4.600648669219739, |
| "logps/chosen": -338.7858177185059, |
| "logps/rejected": -305.161669921875, |
| "loss": 0.5784684181213379, |
| "mean_token_accuracy": 0.7156250983476639, |
| "num_tokens": 5222452.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.43173090666532515, |
| "rewards/margins": 0.4866475585848093, |
| "rewards/rejected": -0.05491665340960026, |
| "step": 380 |
| }, |
| { |
| "entropy": 1.0119995504617691, |
| "epoch": 0.10042487446890691, |
| "grad_norm": 66.73582458496094, |
| "learning_rate": 4.935953820534332e-07, |
| "logits/chosen": 4.700755701463852, |
| "logits/rejected": 4.563956382940839, |
| "logps/chosen": -350.07495574951173, |
| "logps/rejected": -295.9588035583496, |
| "loss": 0.5613166809082031, |
| "mean_token_accuracy": 0.7058805853128434, |
| "num_tokens": 5361783.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.432503791898489, |
| "rewards/margins": 0.5063534714281559, |
| "rewards/rejected": -0.07384968474507332, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.9758326828479766, |
| "epoch": 0.10299987125016094, |
| "grad_norm": 44.98636245727539, |
| "learning_rate": 4.931180096168784e-07, |
| "logits/chosen": 4.683912871421698, |
| "logits/rejected": 4.797076860659791, |
| "logps/chosen": -325.4021278381348, |
| "logps/rejected": -319.83669128417966, |
| "loss": 0.5478048324584961, |
| "mean_token_accuracy": 0.7172370463609695, |
| "num_tokens": 5498760.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.3713498543947935, |
| "rewards/margins": 0.5634564943611622, |
| "rewards/rejected": -0.1921066589653492, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.9767284691333771, |
| "epoch": 0.10557486803141496, |
| "grad_norm": 47.32701110839844, |
| "learning_rate": 4.926237279780488e-07, |
| "logits/chosen": 4.4485483092319855, |
| "logits/rejected": 4.88916482277722, |
| "logps/chosen": -312.9255798339844, |
| "logps/rejected": -301.4211082458496, |
| "loss": 0.5510597229003906, |
| "mean_token_accuracy": 0.7192813426256179, |
| "num_tokens": 5638362.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.4443170979619026, |
| "rewards/margins": 0.5508027624338865, |
| "rewards/rejected": -0.1064856756478548, |
| "step": 410 |
| }, |
| { |
| "entropy": 1.038025939464569, |
| "epoch": 0.10814986481266899, |
| "grad_norm": 59.02484893798828, |
| "learning_rate": 4.921125715149331e-07, |
| "logits/chosen": 4.492031042311866, |
| "logits/rejected": 4.7620465682604465, |
| "logps/chosen": -304.71643142700196, |
| "logps/rejected": -302.98192901611327, |
| "loss": 0.5734364509582519, |
| "mean_token_accuracy": 0.7094101399183274, |
| "num_tokens": 5774772.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.575700556486845, |
| "rewards/margins": 0.6073308132588864, |
| "rewards/rejected": -0.031630271673202516, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.9706959962844849, |
| "epoch": 0.110724861593923, |
| "grad_norm": 50.152259826660156, |
| "learning_rate": 4.915845757791882e-07, |
| "logits/chosen": 4.301352728074184, |
| "logits/rejected": 4.374018651463462, |
| "logps/chosen": -309.12598037719727, |
| "logps/rejected": -329.5872093200684, |
| "loss": 0.5827741622924805, |
| "mean_token_accuracy": 0.7238694995641708, |
| "num_tokens": 5915818.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.3628706641495228, |
| "rewards/margins": 0.526591594517231, |
| "rewards/rejected": -0.16372093437239527, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.9883161246776581, |
| "epoch": 0.11329985837517703, |
| "grad_norm": 63.66497802734375, |
| "learning_rate": 4.910397774936662e-07, |
| "logits/chosen": 4.324055190943431, |
| "logits/rejected": 4.455768599008789, |
| "logps/chosen": -339.16539764404297, |
| "logps/rejected": -317.9355972290039, |
| "loss": 0.60983304977417, |
| "mean_token_accuracy": 0.7213237226009369, |
| "num_tokens": 6057267.0, |
| "rewards/accuracies": 0.63125, |
| "rewards/chosen": 0.3570538476109505, |
| "rewards/margins": 0.4920197740197182, |
| "rewards/rejected": -0.1349659183062613, |
| "step": 440 |
| }, |
| { |
| "entropy": 1.0575228780508041, |
| "epoch": 0.11587485515643106, |
| "grad_norm": 77.4422378540039, |
| "learning_rate": 4.904782145498601e-07, |
| "logits/chosen": 4.483779070580654, |
| "logits/rejected": 4.545815018757674, |
| "logps/chosen": -311.60619049072267, |
| "logps/rejected": -317.6271324157715, |
| "loss": 0.5732481479644775, |
| "mean_token_accuracy": 0.7105006694793701, |
| "num_tokens": 6185910.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.271255973726511, |
| "rewards/margins": 0.5383441850543023, |
| "rewards/rejected": -0.2670882109552622, |
| "step": 450 |
| }, |
| { |
| "entropy": 1.0017959356307984, |
| "epoch": 0.11844985193768508, |
| "grad_norm": 52.255638122558594, |
| "learning_rate": 4.89899926005269e-07, |
| "logits/chosen": 4.416739378206386, |
| "logits/rejected": 4.47387991202081, |
| "logps/chosen": -306.1744514465332, |
| "logps/rejected": -295.7477088928223, |
| "loss": 0.5897578239440918, |
| "mean_token_accuracy": 0.7148655951023102, |
| "num_tokens": 6315045.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.1683360619470477, |
| "rewards/margins": 0.4851381786167622, |
| "rewards/rejected": -0.31680211387574675, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.9547203719615937, |
| "epoch": 0.12102484871893911, |
| "grad_norm": 65.27824401855469, |
| "learning_rate": 4.893049520806809e-07, |
| "logits/chosen": 4.526039482298581, |
| "logits/rejected": 4.724331283409848, |
| "logps/chosen": -318.0591064453125, |
| "logps/rejected": -308.0281555175781, |
| "loss": 0.5524198532104492, |
| "mean_token_accuracy": 0.7198063462972641, |
| "num_tokens": 6448792.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.2623703435063362, |
| "rewards/margins": 0.555498230457306, |
| "rewards/rejected": -0.29312787503004073, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.9980389684438705, |
| "epoch": 0.12359984550019312, |
| "grad_norm": 41.32709884643555, |
| "learning_rate": 4.886933341573762e-07, |
| "logits/chosen": 4.443947690470212, |
| "logits/rejected": 4.704294055958519, |
| "logps/chosen": -322.1322135925293, |
| "logps/rejected": -324.39286651611326, |
| "loss": 0.5217737197875977, |
| "mean_token_accuracy": 0.7148303747177124, |
| "num_tokens": 6582968.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.1742788452655077, |
| "rewards/margins": 0.75794066041708, |
| "rewards/rejected": -0.5836618229746818, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.9905156761407852, |
| "epoch": 0.12617484228144715, |
| "grad_norm": 63.09601974487305, |
| "learning_rate": 4.880651147742484e-07, |
| "logits/chosen": 4.403596984868758, |
| "logits/rejected": 4.697439525919373, |
| "logps/chosen": -305.22482986450194, |
| "logps/rejected": -354.5737548828125, |
| "loss": 0.5686468124389649, |
| "mean_token_accuracy": 0.7282920807600022, |
| "num_tokens": 6712612.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.2129853243008256, |
| "rewards/margins": 0.6234859123826026, |
| "rewards/rejected": -0.4105006046593189, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.9388987690210342, |
| "epoch": 0.12874983906270118, |
| "grad_norm": 42.92222595214844, |
| "learning_rate": 4.874203376248463e-07, |
| "logits/chosen": 4.488846695070381, |
| "logits/rejected": 4.778856944495108, |
| "logps/chosen": -290.40800247192385, |
| "logps/rejected": -312.62304458618166, |
| "loss": 0.518910264968872, |
| "mean_token_accuracy": 0.726474717259407, |
| "num_tokens": 6847702.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.246230723336339, |
| "rewards/margins": 0.7769725546240807, |
| "rewards/rejected": -0.5307418283075094, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.9601681470870972, |
| "epoch": 0.13132483584395518, |
| "grad_norm": 39.82841110229492, |
| "learning_rate": 4.86759047554335e-07, |
| "logits/chosen": 4.2579053387255765, |
| "logits/rejected": 4.4132857090626825, |
| "logps/chosen": -317.2354568481445, |
| "logps/rejected": -294.6973342895508, |
| "loss": 0.5153061866760253, |
| "mean_token_accuracy": 0.7181984424591065, |
| "num_tokens": 6984334.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.34739205725491046, |
| "rewards/margins": 0.7444691531360149, |
| "rewards/rejected": -0.39707708433270456, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.9526659429073334, |
| "epoch": 0.13389983262520921, |
| "grad_norm": 39.31724548339844, |
| "learning_rate": 4.860812905563763e-07, |
| "logits/chosen": 4.304689935360902, |
| "logits/rejected": 4.547928502321976, |
| "logps/chosen": -318.8589202880859, |
| "logps/rejected": -332.00280685424804, |
| "loss": 0.47237606048583985, |
| "mean_token_accuracy": 0.727465134859085, |
| "num_tokens": 7138124.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.18707619085907937, |
| "rewards/margins": 0.8195438928902149, |
| "rewards/rejected": -0.6324677051045, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.9723318427801132, |
| "epoch": 0.13647482940646324, |
| "grad_norm": 56.447967529296875, |
| "learning_rate": 4.853871137699307e-07, |
| "logits/chosen": 4.589381324158332, |
| "logits/rejected": 4.651859008699785, |
| "logps/chosen": -347.7586387634277, |
| "logps/rejected": -314.0169731140137, |
| "loss": 0.5345488548278808, |
| "mean_token_accuracy": 0.7160911679267883, |
| "num_tokens": 7271610.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.03413632959127426, |
| "rewards/margins": 0.7016147773712873, |
| "rewards/rejected": -0.6674784570932388, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.9283442407846451, |
| "epoch": 0.13904982618771727, |
| "grad_norm": 81.77369689941406, |
| "learning_rate": 4.846765654759777e-07, |
| "logits/chosen": 4.534017620164606, |
| "logits/rejected": 4.336357329893327, |
| "logps/chosen": -333.11379013061526, |
| "logps/rejected": -350.09332885742185, |
| "loss": 0.6328543663024903, |
| "mean_token_accuracy": 0.7222778290510178, |
| "num_tokens": 7418090.0, |
| "rewards/accuracies": 0.6375, |
| "rewards/chosen": -0.07262884676456452, |
| "rewards/margins": 0.5407668419182301, |
| "rewards/rejected": -0.6133956845849753, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.9569640517234802, |
| "epoch": 0.14162482296897128, |
| "grad_norm": 60.66325759887695, |
| "learning_rate": 4.839496950941585e-07, |
| "logits/chosen": 4.465449477689448, |
| "logits/rejected": 4.338006664381983, |
| "logps/chosen": -324.85693740844727, |
| "logps/rejected": -300.08666076660154, |
| "loss": 0.5704984664916992, |
| "mean_token_accuracy": 0.7159777522087097, |
| "num_tokens": 7551340.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.20383757278323172, |
| "rewards/margins": 0.6437537096440792, |
| "rewards/rejected": -0.43991615511476995, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.9533803671598434, |
| "epoch": 0.1441998197502253, |
| "grad_norm": 73.0498046875, |
| "learning_rate": 4.832065531793389e-07, |
| "logits/chosen": 4.39996322075107, |
| "logits/rejected": 4.554421709155159, |
| "logps/chosen": -300.69481964111327, |
| "logps/rejected": -292.7109878540039, |
| "loss": 0.5687302589416504, |
| "mean_token_accuracy": 0.7131290882825851, |
| "num_tokens": 7683673.0, |
| "rewards/accuracies": 0.68125, |
| "rewards/chosen": 0.1011431023478508, |
| "rewards/margins": 0.6272410407662392, |
| "rewards/rejected": -0.5260979343205691, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.9646992802619934, |
| "epoch": 0.14677481653147934, |
| "grad_norm": 54.05545425415039, |
| "learning_rate": 4.824471914180928e-07, |
| "logits/chosen": 4.462153498340667, |
| "logits/rejected": 4.351227799524455, |
| "logps/chosen": -334.3186973571777, |
| "logps/rejected": -304.96626739501954, |
| "loss": 0.6647938251495361, |
| "mean_token_accuracy": 0.7061104267835617, |
| "num_tokens": 7812237.0, |
| "rewards/accuracies": 0.68125, |
| "rewards/chosen": -0.020775986835360527, |
| "rewards/margins": 0.530225483328104, |
| "rewards/rejected": -0.5510014675557613, |
| "step": 570 |
| }, |
| { |
| "entropy": 1.0479345887899398, |
| "epoch": 0.14934981331273336, |
| "grad_norm": 37.81373977661133, |
| "learning_rate": 4.816716626251072e-07, |
| "logits/chosen": 4.184722720620551, |
| "logits/rejected": 4.135806228787347, |
| "logps/chosen": -335.03366928100587, |
| "logps/rejected": -359.79450912475585, |
| "loss": 0.572507095336914, |
| "mean_token_accuracy": 0.6949407935142518, |
| "num_tokens": 7949638.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.14069565683603286, |
| "rewards/margins": 0.6246777504682541, |
| "rewards/rejected": -0.4839820809662342, |
| "step": 580 |
| }, |
| { |
| "entropy": 1.011368888616562, |
| "epoch": 0.1519248100939874, |
| "grad_norm": 44.93890380859375, |
| "learning_rate": 4.808800207395095e-07, |
| "logits/chosen": 4.431858594614932, |
| "logits/rejected": 4.494303066742516, |
| "logps/chosen": -359.7134948730469, |
| "logps/rejected": -331.4195068359375, |
| "loss": 0.5715437412261963, |
| "mean_token_accuracy": 0.7035282164812088, |
| "num_tokens": 8086979.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.1286509022116661, |
| "rewards/margins": 0.6396637663245202, |
| "rewards/rejected": -0.5110128573141992, |
| "step": 590 |
| }, |
| { |
| "entropy": 1.0390411615371704, |
| "epoch": 0.1544998068752414, |
| "grad_norm": 41.1849250793457, |
| "learning_rate": 4.80072320821115e-07, |
| "logits/chosen": 4.509100731375415, |
| "logits/rejected": 4.498800666009457, |
| "logps/chosen": -343.8060928344727, |
| "logps/rejected": -301.508959197998, |
| "loss": 0.5278671741485595, |
| "mean_token_accuracy": 0.6904387384653091, |
| "num_tokens": 8215969.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.2609533634036779, |
| "rewards/margins": 0.7143490366637707, |
| "rewards/rejected": -0.4533956632018089, |
| "step": 600 |
| }, |
| { |
| "entropy": 1.0100758999586106, |
| "epoch": 0.15707480365649543, |
| "grad_norm": 45.980743408203125, |
| "learning_rate": 4.792486190465983e-07, |
| "logits/chosen": 4.436240458904658, |
| "logits/rejected": 4.483234245575103, |
| "logps/chosen": -351.39085540771487, |
| "logps/rejected": -322.53857345581054, |
| "loss": 0.5409737586975097, |
| "mean_token_accuracy": 0.7018147617578506, |
| "num_tokens": 8353478.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.1759142629802227, |
| "rewards/margins": 0.6746158372610808, |
| "rewards/rejected": -0.49870156645774844, |
| "step": 610 |
| }, |
| { |
| "entropy": 1.0743782013654708, |
| "epoch": 0.15964980043774946, |
| "grad_norm": 73.07006072998047, |
| "learning_rate": 4.784089727055858e-07, |
| "logits/chosen": 4.279494154259277, |
| "logits/rejected": 4.438133950466845, |
| "logps/chosen": -329.93651657104493, |
| "logps/rejected": -338.3123947143555, |
| "loss": 0.5927572250366211, |
| "mean_token_accuracy": 0.6971615046262741, |
| "num_tokens": 8494676.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.1258799023926258, |
| "rewards/margins": 0.5927445106208324, |
| "rewards/rejected": -0.466864612698555, |
| "step": 620 |
| }, |
| { |
| "entropy": 1.0129943102598191, |
| "epoch": 0.16222479721900349, |
| "grad_norm": 57.14754867553711, |
| "learning_rate": 4.77553440196671e-07, |
| "logits/chosen": 4.545317591931587, |
| "logits/rejected": 4.609596492216733, |
| "logps/chosen": -314.0488555908203, |
| "logps/rejected": -280.8207015991211, |
| "loss": 0.5136913299560547, |
| "mean_token_accuracy": 0.7004855394363403, |
| "num_tokens": 8624139.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.26730198562145235, |
| "rewards/margins": 0.799684862792492, |
| "rewards/rejected": -0.532382888533175, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.9850052386522293, |
| "epoch": 0.1647997940002575, |
| "grad_norm": 58.24784469604492, |
| "learning_rate": 4.766820810233528e-07, |
| "logits/chosen": 4.495559711713429, |
| "logits/rejected": 4.5218358684585676, |
| "logps/chosen": -324.5868515014648, |
| "logps/rejected": -319.23336639404295, |
| "loss": 0.5680853843688964, |
| "mean_token_accuracy": 0.7023002177476882, |
| "num_tokens": 8768480.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.1788150530308485, |
| "rewards/margins": 0.5665261905640364, |
| "rewards/rejected": -0.3877111442387104, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.8995870023965835, |
| "epoch": 0.16737479078151152, |
| "grad_norm": 56.94486999511719, |
| "learning_rate": 4.757949557898972e-07, |
| "logits/chosen": 4.750891546224381, |
| "logits/rejected": 4.5728916398525445, |
| "logps/chosen": -327.8912872314453, |
| "logps/rejected": -321.7278022766113, |
| "loss": 0.57937593460083, |
| "mean_token_accuracy": 0.7331031769514084, |
| "num_tokens": 8916927.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.29649644866585734, |
| "rewards/margins": 0.7233392115682363, |
| "rewards/rejected": -0.42684278152883054, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.963553175330162, |
| "epoch": 0.16994978756276555, |
| "grad_norm": 67.72953796386719, |
| "learning_rate": 4.7489212619712206e-07, |
| "logits/chosen": 4.477993553359102, |
| "logits/rejected": 4.419784017374876, |
| "logps/chosen": -332.0310920715332, |
| "logps/rejected": -299.7566711425781, |
| "loss": 0.5437621116638184, |
| "mean_token_accuracy": 0.7131728023290634, |
| "num_tokens": 9059558.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.2984531581401825, |
| "rewards/margins": 0.7288076810538768, |
| "rewards/rejected": -0.43035454135388135, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.9489945232868194, |
| "epoch": 0.17252478434401958, |
| "grad_norm": 41.158843994140625, |
| "learning_rate": 4.739736550381056e-07, |
| "logits/chosen": 4.682270884264778, |
| "logits/rejected": 4.493197512040618, |
| "logps/chosen": -304.6476936340332, |
| "logps/rejected": -325.6847457885742, |
| "loss": 0.534097957611084, |
| "mean_token_accuracy": 0.7255629748106003, |
| "num_tokens": 9197633.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.28304303996264935, |
| "rewards/margins": 0.7079235155135393, |
| "rewards/rejected": -0.4248804687522352, |
| "step": 670 |
| }, |
| { |
| "entropy": 1.0493477791547776, |
| "epoch": 0.1750997811252736, |
| "grad_norm": 43.2928352355957, |
| "learning_rate": 4.730396061938192e-07, |
| "logits/chosen": 4.510271629390098, |
| "logits/rejected": 4.432055280745351, |
| "logps/chosen": -365.35415878295896, |
| "logps/rejected": -337.8881271362305, |
| "loss": 0.5792159080505371, |
| "mean_token_accuracy": 0.7000991433858872, |
| "num_tokens": 9333452.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.14067796915769576, |
| "rewards/margins": 0.613553061336279, |
| "rewards/rejected": -0.4728750824928284, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.9644862800836563, |
| "epoch": 0.1776747779065276, |
| "grad_norm": 69.288818359375, |
| "learning_rate": 4.720900446286844e-07, |
| "logits/chosen": 4.534953701150827, |
| "logits/rejected": 4.728891639666656, |
| "logps/chosen": -331.7658752441406, |
| "logps/rejected": -320.43864288330076, |
| "loss": 0.5897587776184082, |
| "mean_token_accuracy": 0.7107261806726456, |
| "num_tokens": 9470396.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.10997514789924026, |
| "rewards/margins": 0.6469641901552677, |
| "rewards/rejected": -0.5369890414178371, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.9663564383983612, |
| "epoch": 0.18024977468778164, |
| "grad_norm": 56.91593933105469, |
| "learning_rate": 4.711250363860543e-07, |
| "logits/chosen": 4.553612024481287, |
| "logits/rejected": 4.682406648335661, |
| "logps/chosen": -354.33360290527344, |
| "logps/rejected": -330.5455451965332, |
| "loss": 0.5636067390441895, |
| "mean_token_accuracy": 0.7119427442550659, |
| "num_tokens": 9616899.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.12148317620158196, |
| "rewards/margins": 0.6505684278905391, |
| "rewards/rejected": -0.5290852546691894, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.9706416428089142, |
| "epoch": 0.18282477146903567, |
| "grad_norm": 84.57388305664062, |
| "learning_rate": 4.7014464858362034e-07, |
| "logits/chosen": 4.64987863105768, |
| "logits/rejected": 4.537342444882523, |
| "logps/chosen": -335.3706756591797, |
| "logps/rejected": -332.0774948120117, |
| "loss": 0.5970035552978515, |
| "mean_token_accuracy": 0.7182563871145249, |
| "num_tokens": 9761467.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.09402619935572147, |
| "rewards/margins": 0.6252138413488865, |
| "rewards/rejected": -0.5311876498162746, |
| "step": 710 |
| }, |
| { |
| "entropy": 1.034638649225235, |
| "epoch": 0.1853997682502897, |
| "grad_norm": 54.29759979248047, |
| "learning_rate": 4.6914894940874437e-07, |
| "logits/chosen": 4.266203497733936, |
| "logits/rejected": 4.523968467287711, |
| "logps/chosen": -325.725659942627, |
| "logps/rejected": -323.9387481689453, |
| "loss": 0.5491708278656006, |
| "mean_token_accuracy": 0.7071366012096405, |
| "num_tokens": 9903698.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.22243431201204658, |
| "rewards/margins": 0.649902455508709, |
| "rewards/rejected": -0.42746814265847205, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.9635148346424103, |
| "epoch": 0.1879747650315437, |
| "grad_norm": 65.74314880371094, |
| "learning_rate": 4.6813800811371574e-07, |
| "logits/chosen": 4.393643202445704, |
| "logits/rejected": 4.69651795868508, |
| "logps/chosen": -316.67277908325195, |
| "logps/rejected": -307.452326965332, |
| "loss": 0.549955940246582, |
| "mean_token_accuracy": 0.7105065941810608, |
| "num_tokens": 10030969.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.1749096255749464, |
| "rewards/margins": 0.706392652913928, |
| "rewards/rejected": -0.5314830277115107, |
| "step": 730 |
| }, |
| { |
| "entropy": 1.0183222383260726, |
| "epoch": 0.19054976181279773, |
| "grad_norm": 86.1967544555664, |
| "learning_rate": 4.671118950109348e-07, |
| "logits/chosen": 4.368303356172162, |
| "logits/rejected": 4.454061870111907, |
| "logps/chosen": -336.9675567626953, |
| "logps/rejected": -344.4869667053223, |
| "loss": 0.5904175758361816, |
| "mean_token_accuracy": 0.7114386945962906, |
| "num_tokens": 10174551.0, |
| "rewards/accuracies": 0.6625, |
| "rewards/chosen": 0.1169948361814022, |
| "rewards/margins": 0.6056915517896414, |
| "rewards/rejected": -0.48869672007858755, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.9779737055301666, |
| "epoch": 0.19312475859405176, |
| "grad_norm": 81.3215103149414, |
| "learning_rate": 4.6607068146802284e-07, |
| "logits/chosen": 4.34994022328082, |
| "logits/rejected": 4.692101749793964, |
| "logps/chosen": -331.00280303955077, |
| "logps/rejected": -304.03379211425784, |
| "loss": 0.5423304557800293, |
| "mean_token_accuracy": 0.7130115896463394, |
| "num_tokens": 10309026.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.28315746448934076, |
| "rewards/margins": 0.6935677349567413, |
| "rewards/rejected": -0.41041027642786504, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.9288118928670883, |
| "epoch": 0.1956997553753058, |
| "grad_norm": 64.25709533691406, |
| "learning_rate": 4.65014439902858e-07, |
| "logits/chosen": 4.646212096793471, |
| "logits/rejected": 4.713876319349232, |
| "logps/chosen": -328.4228065490723, |
| "logps/rejected": -325.85131683349607, |
| "loss": 0.597814416885376, |
| "mean_token_accuracy": 0.715106874704361, |
| "num_tokens": 10454313.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.2064698487520218, |
| "rewards/margins": 0.5923826836049557, |
| "rewards/rejected": -0.38591282814741135, |
| "step": 760 |
| }, |
| { |
| "entropy": 1.0085820376873016, |
| "epoch": 0.1982747521565598, |
| "grad_norm": 60.60913848876953, |
| "learning_rate": 4.639432437785387e-07, |
| "logits/chosen": 4.624968581831299, |
| "logits/rejected": 4.833529951127492, |
| "logps/chosen": -336.28843994140624, |
| "logps/rejected": -309.39982376098635, |
| "loss": 0.5225073814392089, |
| "mean_token_accuracy": 0.7082129001617432, |
| "num_tokens": 10590291.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.3072497993707657, |
| "rewards/margins": 0.7669197037816048, |
| "rewards/rejected": -0.45966988801956177, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.9794066548347473, |
| "epoch": 0.20084974893781382, |
| "grad_norm": 58.9998664855957, |
| "learning_rate": 4.6285716759827424e-07, |
| "logits/chosen": 4.6144834186546575, |
| "logits/rejected": 4.793606974197976, |
| "logps/chosen": -327.2888931274414, |
| "logps/rejected": -325.06821594238284, |
| "loss": 0.5766840934753418, |
| "mean_token_accuracy": 0.7179515421390533, |
| "num_tokens": 10740192.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.14814186543226243, |
| "rewards/margins": 0.6016824893653393, |
| "rewards/rejected": -0.4535406146198511, |
| "step": 780 |
| }, |
| { |
| "entropy": 1.0347339719533921, |
| "epoch": 0.20342474571906785, |
| "grad_norm": 51.062068939208984, |
| "learning_rate": 4.6175628690020285e-07, |
| "logits/chosen": 4.346229749477498, |
| "logits/rejected": 4.816758360147, |
| "logps/chosen": -305.7140487670898, |
| "logps/rejected": -290.80119094848635, |
| "loss": 0.531673812866211, |
| "mean_token_accuracy": 0.7126160234212875, |
| "num_tokens": 10871228.0, |
| "rewards/accuracies": 0.8, |
| "rewards/chosen": 0.2662514135241508, |
| "rewards/margins": 0.765745447576046, |
| "rewards/rejected": -0.49949402883648875, |
| "step": 790 |
| }, |
| { |
| "entropy": 1.0405800729990005, |
| "epoch": 0.20599974250032188, |
| "grad_norm": 43.19007110595703, |
| "learning_rate": 4.60640678252138e-07, |
| "logits/chosen": 4.161079633620957, |
| "logits/rejected": 4.3872889124713, |
| "logps/chosen": -328.34368896484375, |
| "logps/rejected": -308.5476463317871, |
| "loss": 0.5606516838073731, |
| "mean_token_accuracy": 0.6936956197023392, |
| "num_tokens": 10997163.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.2265689242631197, |
| "rewards/margins": 0.6632086362689733, |
| "rewards/rejected": -0.4366397067904472, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.9820335626602172, |
| "epoch": 0.2085747392815759, |
| "grad_norm": 46.55604553222656, |
| "learning_rate": 4.5951041924624286e-07, |
| "logits/chosen": 4.517027936970296, |
| "logits/rejected": 4.5808484038104424, |
| "logps/chosen": -318.0545349121094, |
| "logps/rejected": -309.64340591430664, |
| "loss": 0.5541488647460937, |
| "mean_token_accuracy": 0.7265620589256286, |
| "num_tokens": 11129908.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.3501715624704957, |
| "rewards/margins": 0.7329499304294587, |
| "rewards/rejected": -0.3827783614397049, |
| "step": 810 |
| }, |
| { |
| "entropy": 1.0183537423610687, |
| "epoch": 0.2111497360628299, |
| "grad_norm": 43.764923095703125, |
| "learning_rate": 4.5836558849363384e-07, |
| "logits/chosen": 4.702848554606683, |
| "logits/rejected": 4.6702062948790415, |
| "logps/chosen": -318.51319122314453, |
| "logps/rejected": -289.55165786743163, |
| "loss": 0.4766195297241211, |
| "mean_token_accuracy": 0.6965174585580826, |
| "num_tokens": 11260989.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.40055517572909594, |
| "rewards/margins": 0.8507210377603769, |
| "rewards/rejected": -0.45016585290431976, |
| "step": 820 |
| }, |
| { |
| "entropy": 1.0276739478111268, |
| "epoch": 0.21372473284408394, |
| "grad_norm": 93.66756439208984, |
| "learning_rate": 4.5720626561891306e-07, |
| "logits/chosen": 4.369238347744605, |
| "logits/rejected": 4.546188272271925, |
| "logps/chosen": -362.52023162841795, |
| "logps/rejected": -339.30326080322266, |
| "loss": 0.5464986324310303, |
| "mean_token_accuracy": 0.7055434852838516, |
| "num_tokens": 11400548.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.20844534412026405, |
| "rewards/margins": 0.6820525597780943, |
| "rewards/rejected": -0.47360720857977867, |
| "step": 830 |
| }, |
| { |
| "entropy": 1.0048970103263855, |
| "epoch": 0.21629972962533797, |
| "grad_norm": 39.34489822387695, |
| "learning_rate": 4.5603253125463013e-07, |
| "logits/chosen": 4.482431808569223, |
| "logits/rejected": 4.461442789980365, |
| "logps/chosen": -344.8500915527344, |
| "logps/rejected": -313.5320655822754, |
| "loss": 0.6112454414367676, |
| "mean_token_accuracy": 0.7048583775758743, |
| "num_tokens": 11532663.0, |
| "rewards/accuracies": 0.65, |
| "rewards/chosen": 0.17530339695513247, |
| "rewards/margins": 0.6294477324932813, |
| "rewards/rejected": -0.45414433553814887, |
| "step": 840 |
| }, |
| { |
| "entropy": 1.0157939851284028, |
| "epoch": 0.218874726406592, |
| "grad_norm": 50.64561080932617, |
| "learning_rate": 4.548444670356743e-07, |
| "logits/chosen": 4.558716367494338, |
| "logits/rejected": 4.622169300415546, |
| "logps/chosen": -295.56880950927734, |
| "logps/rejected": -281.37263641357424, |
| "loss": 0.5642531871795654, |
| "mean_token_accuracy": 0.6964393675327301, |
| "num_tokens": 11662432.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.2496818548068404, |
| "rewards/margins": 0.6377269882708788, |
| "rewards/rejected": -0.3880451485514641, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.9604170709848404, |
| "epoch": 0.221449723187846, |
| "grad_norm": 49.767303466796875, |
| "learning_rate": 4.536421555935965e-07, |
| "logits/chosen": 4.543303368009663, |
| "logits/rejected": 4.610104424823825, |
| "logps/chosen": -352.4889404296875, |
| "logps/rejected": -309.16469802856443, |
| "loss": 0.5757847785949707, |
| "mean_token_accuracy": 0.7144225656986236, |
| "num_tokens": 11799597.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.25048075765371325, |
| "rewards/margins": 0.6514817461371422, |
| "rewards/rejected": -0.40100100487470625, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.9212517082691193, |
| "epoch": 0.22402471996910003, |
| "grad_norm": 44.281654357910156, |
| "learning_rate": 4.524256805508623e-07, |
| "logits/chosen": 4.4288841618762085, |
| "logits/rejected": 4.522083677187061, |
| "logps/chosen": -284.46302261352537, |
| "logps/rejected": -299.4017608642578, |
| "loss": 0.5767347335815429, |
| "mean_token_accuracy": 0.7335207104682923, |
| "num_tokens": 11932852.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.31010360531508924, |
| "rewards/margins": 0.6455920591950417, |
| "rewards/rejected": -0.33548846244812014, |
| "step": 870 |
| }, |
| { |
| "entropy": 1.0051367849111557, |
| "epoch": 0.22659971675035406, |
| "grad_norm": 59.0626335144043, |
| "learning_rate": 4.5119512651503554e-07, |
| "logits/chosen": 4.433084857610975, |
| "logits/rejected": 4.87764424226992, |
| "logps/chosen": -325.5847366333008, |
| "logps/rejected": -313.316064453125, |
| "loss": 0.532440185546875, |
| "mean_token_accuracy": 0.7039438933134079, |
| "num_tokens": 12057229.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.3609015665948391, |
| "rewards/margins": 0.9245390221476555, |
| "rewards/rejected": -0.5636374469846487, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.9578097075223923, |
| "epoch": 0.2291747135316081, |
| "grad_norm": 82.41941833496094, |
| "learning_rate": 4.499505790728943e-07, |
| "logits/chosen": 4.321722093432549, |
| "logits/rejected": 4.647098786096395, |
| "logps/chosen": -321.9654907226562, |
| "logps/rejected": -307.7423271179199, |
| "loss": 0.537187147140503, |
| "mean_token_accuracy": 0.7209613204002381, |
| "num_tokens": 12205196.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.33095078393816946, |
| "rewards/margins": 0.769245932251215, |
| "rewards/rejected": -0.4382951531559229, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.9880760461091995, |
| "epoch": 0.23174971031286212, |
| "grad_norm": 44.68686294555664, |
| "learning_rate": 4.486921247844777e-07, |
| "logits/chosen": 4.494354658873168, |
| "logits/rejected": 4.507425478709289, |
| "logps/chosen": -305.7351020812988, |
| "logps/rejected": -276.03930740356446, |
| "loss": 0.5129434585571289, |
| "mean_token_accuracy": 0.7095896989107132, |
| "num_tokens": 12334089.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.36211476698517797, |
| "rewards/margins": 0.8058889979496598, |
| "rewards/rejected": -0.4437742181122303, |
| "step": 900 |
| }, |
| { |
| "entropy": 1.0145466029644012, |
| "epoch": 0.23432470709411612, |
| "grad_norm": 84.48532104492188, |
| "learning_rate": 4.474198511770659e-07, |
| "logits/chosen": 4.206393560107395, |
| "logits/rejected": 4.203391496213106, |
| "logps/chosen": -380.53197174072267, |
| "logps/rejected": -352.91199493408203, |
| "loss": 0.6008936882019043, |
| "mean_token_accuracy": 0.709510189294815, |
| "num_tokens": 12477504.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.3297712683677673, |
| "rewards/margins": 0.7578461959958076, |
| "rewards/rejected": -0.42807492688298227, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.9396564811468124, |
| "epoch": 0.23689970387537015, |
| "grad_norm": 59.96947479248047, |
| "learning_rate": 4.4613384673909206e-07, |
| "logits/chosen": 4.477632786838245, |
| "logits/rejected": 4.655172462661108, |
| "logps/chosen": -306.4482566833496, |
| "logps/rejected": -309.2312377929687, |
| "loss": 0.5252578735351563, |
| "mean_token_accuracy": 0.7247987031936646, |
| "num_tokens": 12620019.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.4170497950166464, |
| "rewards/margins": 0.7426737532019615, |
| "rewards/rejected": -0.32562397196888926, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.9839500367641449, |
| "epoch": 0.23947470065662418, |
| "grad_norm": 51.99100875854492, |
| "learning_rate": 4.448342009139883e-07, |
| "logits/chosen": 4.521783636596377, |
| "logits/rejected": 4.74909994208237, |
| "logps/chosen": -304.9866119384766, |
| "logps/rejected": -305.9053276062012, |
| "loss": 0.6351909637451172, |
| "mean_token_accuracy": 0.7072886407375336, |
| "num_tokens": 12746450.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.3074321709573269, |
| "rewards/margins": 0.5390587091445923, |
| "rewards/rejected": -0.23162655010819436, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.9272339820861817, |
| "epoch": 0.24204969743787821, |
| "grad_norm": 50.78702926635742, |
| "learning_rate": 4.435210040939644e-07, |
| "logits/chosen": 4.49347232077586, |
| "logits/rejected": 4.544626873447705, |
| "logps/chosen": -298.0924583435059, |
| "logps/rejected": -311.46923828125, |
| "loss": 0.5839786529541016, |
| "mean_token_accuracy": 0.724303224682808, |
| "num_tokens": 12897894.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.3771603710949421, |
| "rewards/margins": 0.6256467439234257, |
| "rewards/rejected": -0.2484863717108965, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.9667321592569351, |
| "epoch": 0.24462469421913222, |
| "grad_norm": 67.10762786865234, |
| "learning_rate": 4.421943476137212e-07, |
| "logits/chosen": 4.021056633771452, |
| "logits/rejected": 4.102243549720436, |
| "logps/chosen": -342.9797004699707, |
| "logps/rejected": -341.395401763916, |
| "loss": 0.5504098892211914, |
| "mean_token_accuracy": 0.7200477600097657, |
| "num_tokens": 13040090.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.2565610365010798, |
| "rewards/margins": 0.7125073965638876, |
| "rewards/rejected": -0.45594635233283043, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.9595152884721756, |
| "epoch": 0.24719969100038625, |
| "grad_norm": 47.71066665649414, |
| "learning_rate": 4.408543237440978e-07, |
| "logits/chosen": 4.39034454098633, |
| "logits/rejected": 4.761796312543795, |
| "logps/chosen": -335.67313919067385, |
| "logps/rejected": -313.7569305419922, |
| "loss": 0.5047685623168945, |
| "mean_token_accuracy": 0.7175188481807708, |
| "num_tokens": 13177252.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.28581122644245627, |
| "rewards/margins": 0.8943164490163327, |
| "rewards/rejected": -0.6085052292793989, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.9980293989181519, |
| "epoch": 0.24977468778164028, |
| "grad_norm": 52.98997497558594, |
| "learning_rate": 4.395010256856544e-07, |
| "logits/chosen": 4.402309676885595, |
| "logits/rejected": 4.488470372113004, |
| "logps/chosen": -338.8411064147949, |
| "logps/rejected": -308.0730743408203, |
| "loss": 0.5538735389709473, |
| "mean_token_accuracy": 0.711775341629982, |
| "num_tokens": 13308594.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.38073269501328466, |
| "rewards/margins": 0.6910081386566163, |
| "rewards/rejected": -0.3102754525840282, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.9796093523502349, |
| "epoch": 0.2523496845628943, |
| "grad_norm": 72.55328369140625, |
| "learning_rate": 4.381345475621897e-07, |
| "logits/chosen": 4.3565436245435745, |
| "logits/rejected": 4.169848814716963, |
| "logps/chosen": -346.8167358398438, |
| "logps/rejected": -293.40828170776365, |
| "loss": 0.5575485706329346, |
| "mean_token_accuracy": 0.702203905582428, |
| "num_tokens": 13441680.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.48410786855965854, |
| "rewards/margins": 0.6693111173808575, |
| "rewards/rejected": -0.18520323652774096, |
| "step": 980 |
| }, |
| { |
| "entropy": 1.045410931110382, |
| "epoch": 0.2549246813441483, |
| "grad_norm": 38.534996032714844, |
| "learning_rate": 4.367549844141946e-07, |
| "logits/chosen": 4.20656935170718, |
| "logits/rejected": 4.301880442441847, |
| "logps/chosen": -330.08270797729494, |
| "logps/rejected": -319.90564880371096, |
| "loss": 0.5081873416900635, |
| "mean_token_accuracy": 0.7055037915706635, |
| "num_tokens": 13570019.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.5979794513434171, |
| "rewards/margins": 0.8163566261529922, |
| "rewards/rejected": -0.21837717592716216, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.9823104649782181, |
| "epoch": 0.25749967812540236, |
| "grad_norm": 50.6489372253418, |
| "learning_rate": 4.353624321922421e-07, |
| "logits/chosen": 4.286461799453784, |
| "logits/rejected": 4.583997969007628, |
| "logps/chosen": -316.2755645751953, |
| "logps/rejected": -288.8273071289062, |
| "loss": 0.5642371654510498, |
| "mean_token_accuracy": 0.7020954936742783, |
| "num_tokens": 13703625.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.4080685555934906, |
| "rewards/margins": 0.6109554670751095, |
| "rewards/rejected": -0.20288692265748978, |
| "step": 1000 |
| }, |
| { |
| "entropy": 1.0343029111623765, |
| "epoch": 0.26007467490665637, |
| "grad_norm": 66.1665267944336, |
| "learning_rate": 4.3395698775031373e-07, |
| "logits/chosen": 4.268548969788038, |
| "logits/rejected": 4.119006348300696, |
| "logps/chosen": -357.6877983093262, |
| "logps/rejected": -311.49485168457034, |
| "loss": 0.5452314376831054, |
| "mean_token_accuracy": 0.7012816995382309, |
| "num_tokens": 13846374.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.5230955958366394, |
| "rewards/margins": 0.7875805646181107, |
| "rewards/rejected": -0.26448496505618097, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.9775324791669846, |
| "epoch": 0.26264967168791037, |
| "grad_norm": 72.12793731689453, |
| "learning_rate": 4.3253874883906305e-07, |
| "logits/chosen": 4.074687471749744, |
| "logits/rejected": 4.233662853315577, |
| "logps/chosen": -340.43592529296876, |
| "logps/rejected": -325.31726837158203, |
| "loss": 0.5151017189025879, |
| "mean_token_accuracy": 0.7066153079271317, |
| "num_tokens": 13989379.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.5307824924588204, |
| "rewards/margins": 0.7811033889651299, |
| "rewards/rejected": -0.25032090246677396, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.9814414829015732, |
| "epoch": 0.2652246684691644, |
| "grad_norm": 64.95217895507812, |
| "learning_rate": 4.311078140990173e-07, |
| "logits/chosen": 4.479037367671388, |
| "logits/rejected": 4.75286217597192, |
| "logps/chosen": -325.9545074462891, |
| "logps/rejected": -284.51468658447266, |
| "loss": 0.5569169998168946, |
| "mean_token_accuracy": 0.7042251348495483, |
| "num_tokens": 14123739.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.32408043220639227, |
| "rewards/margins": 0.7771117240190506, |
| "rewards/rejected": -0.45303131118416784, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.9413322359323502, |
| "epoch": 0.26779966525041843, |
| "grad_norm": 80.55500030517578, |
| "learning_rate": 4.2966428305371636e-07, |
| "logits/chosen": 4.377699481875437, |
| "logits/rejected": 4.389535161226265, |
| "logps/chosen": -313.2835182189941, |
| "logps/rejected": -316.5220802307129, |
| "loss": 0.5732664108276367, |
| "mean_token_accuracy": 0.7221273809671402, |
| "num_tokens": 14265000.0, |
| "rewards/accuracies": 0.68125, |
| "rewards/chosen": 0.4250858768820763, |
| "rewards/margins": 0.7080753080546855, |
| "rewards/rejected": -0.2829894308000803, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.9776986628770828, |
| "epoch": 0.2703746620316725, |
| "grad_norm": 51.449913024902344, |
| "learning_rate": 4.2820825610279133e-07, |
| "logits/chosen": 4.661412588919254, |
| "logits/rejected": 4.593736025078175, |
| "logps/chosen": -339.15430603027346, |
| "logps/rejected": -318.2574661254883, |
| "loss": 0.4670609474182129, |
| "mean_token_accuracy": 0.7175067514181137, |
| "num_tokens": 14400976.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.38831937648355963, |
| "rewards/margins": 0.949066436290741, |
| "rewards/rejected": -0.5607470721006393, |
| "step": 1050 |
| }, |
| { |
| "entropy": 1.020277938246727, |
| "epoch": 0.2729496588129265, |
| "grad_norm": 55.74995040893555, |
| "learning_rate": 4.2673983451498095e-07, |
| "logits/chosen": 4.402389798338661, |
| "logits/rejected": 4.487614058865336, |
| "logps/chosen": -313.8472129821777, |
| "logps/rejected": -328.69329528808595, |
| "loss": 0.614504051208496, |
| "mean_token_accuracy": 0.7057614833116531, |
| "num_tokens": 14543371.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.3766850482672453, |
| "rewards/margins": 0.6233583599328995, |
| "rewards/rejected": -0.24667333364486693, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.9874202489852906, |
| "epoch": 0.2755246555941805, |
| "grad_norm": 59.98944091796875, |
| "learning_rate": 4.252591204210887e-07, |
| "logits/chosen": 4.263326733993418, |
| "logits/rejected": 4.384204437259707, |
| "logps/chosen": -322.31773834228517, |
| "logps/rejected": -316.5559844970703, |
| "loss": 0.5732818126678467, |
| "mean_token_accuracy": 0.7176870882511139, |
| "num_tokens": 14693676.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.4322884988039732, |
| "rewards/margins": 0.7766366615891457, |
| "rewards/rejected": -0.34434815123677254, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.9693405300378799, |
| "epoch": 0.27809965237543455, |
| "grad_norm": 45.86904525756836, |
| "learning_rate": 4.2376621680687917e-07, |
| "logits/chosen": 4.387672304444868, |
| "logits/rejected": 4.787616007597828, |
| "logps/chosen": -320.0915107727051, |
| "logps/rejected": -304.12927017211916, |
| "loss": 0.4748681545257568, |
| "mean_token_accuracy": 0.7157050430774688, |
| "num_tokens": 14828096.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.45522386208176613, |
| "rewards/margins": 0.992213748395443, |
| "rewards/rejected": -0.5369898892939091, |
| "step": 1080 |
| }, |
| { |
| "entropy": 1.0544966250658034, |
| "epoch": 0.28067464915668855, |
| "grad_norm": 59.417510986328125, |
| "learning_rate": 4.222612275059156e-07, |
| "logits/chosen": 4.638836982554174, |
| "logits/rejected": 4.424454369076098, |
| "logps/chosen": -385.91309356689453, |
| "logps/rejected": -338.5951675415039, |
| "loss": 0.5437905311584472, |
| "mean_token_accuracy": 0.6905349731445313, |
| "num_tokens": 14963351.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.5071282530203461, |
| "rewards/margins": 0.8644575633108615, |
| "rewards/rejected": -0.3573293127119541, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.9879767268896102, |
| "epoch": 0.28324964593794255, |
| "grad_norm": 63.74018859863281, |
| "learning_rate": 4.2074425719233766e-07, |
| "logits/chosen": 4.488359396642159, |
| "logits/rejected": 4.643684549172062, |
| "logps/chosen": -331.8841720581055, |
| "logps/rejected": -321.42944717407227, |
| "loss": 0.5911607265472412, |
| "mean_token_accuracy": 0.7219341546297073, |
| "num_tokens": 15108671.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.40470806770026685, |
| "rewards/margins": 0.7509341590106487, |
| "rewards/rejected": -0.3462261028587818, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.9788440942764283, |
| "epoch": 0.2858246427191966, |
| "grad_norm": 76.77590942382812, |
| "learning_rate": 4.192154113735814e-07, |
| "logits/chosen": 4.421786018683489, |
| "logits/rejected": 4.490996666769295, |
| "logps/chosen": -314.8667678833008, |
| "logps/rejected": -286.60582275390624, |
| "loss": 0.577817440032959, |
| "mean_token_accuracy": 0.7135196805000306, |
| "num_tokens": 15247587.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.42707300968468187, |
| "rewards/margins": 0.6557801038026809, |
| "rewards/rejected": -0.22870709635317327, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.9727748841047287, |
| "epoch": 0.2883996395004506, |
| "grad_norm": 57.815582275390625, |
| "learning_rate": 4.1767479638304136e-07, |
| "logits/chosen": 4.1840160801063835, |
| "logits/rejected": 4.439244682258356, |
| "logps/chosen": -317.1370674133301, |
| "logps/rejected": -316.2620231628418, |
| "loss": 0.46355476379394533, |
| "mean_token_accuracy": 0.7234881222248077, |
| "num_tokens": 15386538.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.4425433840602636, |
| "rewards/margins": 1.1140578791499138, |
| "rewards/rejected": -0.6715144798159599, |
| "step": 1120 |
| }, |
| { |
| "entropy": 1.0229500859975815, |
| "epoch": 0.29097463628170467, |
| "grad_norm": 70.49669647216797, |
| "learning_rate": 4.161225193726744e-07, |
| "logits/chosen": 4.247028300686612, |
| "logits/rejected": 4.342862855944697, |
| "logps/chosen": -329.0539245605469, |
| "logps/rejected": -294.3794647216797, |
| "loss": 0.7115816593170166, |
| "mean_token_accuracy": 0.7092172116041183, |
| "num_tokens": 15526554.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.16585742607712745, |
| "rewards/margins": 0.5084913685917855, |
| "rewards/rejected": -0.34263395629823207, |
| "step": 1130 |
| }, |
| { |
| "entropy": 1.0134399235248566, |
| "epoch": 0.29354963306295867, |
| "grad_norm": 49.2717399597168, |
| "learning_rate": 4.1455868830554753e-07, |
| "logits/chosen": 4.687399029849365, |
| "logits/rejected": 4.804292485515946, |
| "logps/chosen": -313.1935607910156, |
| "logps/rejected": -308.4917694091797, |
| "loss": 0.47522659301757814, |
| "mean_token_accuracy": 0.7117983579635621, |
| "num_tokens": 15663517.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.3684566400945187, |
| "rewards/margins": 0.9885708034038544, |
| "rewards/rejected": -0.6201141782104969, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.9662601947784424, |
| "epoch": 0.2961246298442127, |
| "grad_norm": 46.927555084228516, |
| "learning_rate": 4.1298341194832874e-07, |
| "logits/chosen": 4.2654172623623, |
| "logits/rejected": 4.462777736977772, |
| "logps/chosen": -332.715348815918, |
| "logps/rejected": -314.6715911865234, |
| "loss": 0.5290184020996094, |
| "mean_token_accuracy": 0.7249146103858948, |
| "num_tokens": 15804723.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.17502789571881294, |
| "rewards/margins": 0.9185936354100704, |
| "rewards/rejected": -0.7435657449066639, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.9269982099533081, |
| "epoch": 0.29869962662546673, |
| "grad_norm": 43.15007400512695, |
| "learning_rate": 4.1139679986372217e-07, |
| "logits/chosen": 4.3429068940834075, |
| "logits/rejected": 4.419197189323593, |
| "logps/chosen": -319.6227722167969, |
| "logps/rejected": -288.4176681518555, |
| "loss": 0.5703027248382568, |
| "mean_token_accuracy": 0.7230746954679489, |
| "num_tokens": 15940851.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.15694138407707214, |
| "rewards/margins": 0.802145817130804, |
| "rewards/rejected": -0.6452044431120157, |
| "step": 1160 |
| }, |
| { |
| "entropy": 1.0573670476675034, |
| "epoch": 0.30127462340672073, |
| "grad_norm": 59.737918853759766, |
| "learning_rate": 4.0979896240284794e-07, |
| "logits/chosen": 4.556767603242213, |
| "logits/rejected": 4.557870560061238, |
| "logps/chosen": -357.4036674499512, |
| "logps/rejected": -333.00369415283205, |
| "loss": 0.6517429351806641, |
| "mean_token_accuracy": 0.6915355741977691, |
| "num_tokens": 16077951.0, |
| "rewards/accuracies": 0.6625, |
| "rewards/chosen": 0.0758826458826661, |
| "rewards/margins": 0.5559393465518951, |
| "rewards/rejected": -0.4800567090511322, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.9671867489814758, |
| "epoch": 0.3038496201879748, |
| "grad_norm": 63.33560562133789, |
| "learning_rate": 4.0819001069756693e-07, |
| "logits/chosen": 4.426397280001497, |
| "logits/rejected": 4.558353541171919, |
| "logps/chosen": -340.3173118591309, |
| "logps/rejected": -305.58366165161135, |
| "loss": 0.6510348796844483, |
| "mean_token_accuracy": 0.7134972125291824, |
| "num_tokens": 16219068.0, |
| "rewards/accuracies": 0.6625, |
| "rewards/chosen": 0.09389384500682355, |
| "rewards/margins": 0.5385100539773703, |
| "rewards/rejected": -0.44461619537323716, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.994009718298912, |
| "epoch": 0.3064246169692288, |
| "grad_norm": 39.513572692871094, |
| "learning_rate": 4.0657005665275145e-07, |
| "logits/chosen": 4.424516966732384, |
| "logits/rejected": 4.583500376884508, |
| "logps/chosen": -367.8083465576172, |
| "logps/rejected": -335.01172103881834, |
| "loss": 0.5242624759674073, |
| "mean_token_accuracy": 0.7107499301433563, |
| "num_tokens": 16359511.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.2147902548313141, |
| "rewards/margins": 0.7728373639285564, |
| "rewards/rejected": -0.558047103881836, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.9978521913290024, |
| "epoch": 0.3089996137504828, |
| "grad_norm": 71.0284423828125, |
| "learning_rate": 4.0493921293850226e-07, |
| "logits/chosen": 4.563816555341373, |
| "logits/rejected": 4.472360028324781, |
| "logps/chosen": -350.8389205932617, |
| "logps/rejected": -326.0243545532227, |
| "loss": 0.5175440311431885, |
| "mean_token_accuracy": 0.7131787866353989, |
| "num_tokens": 16499453.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.046732995100319384, |
| "rewards/margins": 0.7765126742422581, |
| "rewards/rejected": -0.729779689013958, |
| "step": 1200 |
| }, |
| { |
| "entropy": 1.031485903263092, |
| "epoch": 0.31157461053173685, |
| "grad_norm": 46.33605194091797, |
| "learning_rate": 4.032975929823119e-07, |
| "logits/chosen": 4.554355201437686, |
| "logits/rejected": 4.689299279318398, |
| "logps/chosen": -358.5169845581055, |
| "logps/rejected": -337.0670585632324, |
| "loss": 0.5396876335144043, |
| "mean_token_accuracy": 0.6951931953430176, |
| "num_tokens": 16638240.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.0658013928681612, |
| "rewards/margins": 0.8561005435883999, |
| "rewards/rejected": -0.790299154818058, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.9911634683609009, |
| "epoch": 0.31414960731299085, |
| "grad_norm": 41.01152420043945, |
| "learning_rate": 4.01645310961176e-07, |
| "logits/chosen": 4.613249416296803, |
| "logits/rejected": 4.713178879447741, |
| "logps/chosen": -323.03479080200196, |
| "logps/rejected": -302.49776458740234, |
| "loss": 0.44008312225341795, |
| "mean_token_accuracy": 0.7046775192022323, |
| "num_tokens": 16762530.0, |
| "rewards/accuracies": 0.81875, |
| "rewards/chosen": 0.050583545863628385, |
| "rewards/margins": 1.049423012137413, |
| "rewards/rejected": -0.9988394796848297, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.9754254132509231, |
| "epoch": 0.31672460409424485, |
| "grad_norm": 66.57528686523438, |
| "learning_rate": 3.999824817936518e-07, |
| "logits/chosen": 4.369396370883204, |
| "logits/rejected": 4.565405825816332, |
| "logps/chosen": -309.5333885192871, |
| "logps/rejected": -310.53377532958984, |
| "loss": 0.5147846221923829, |
| "mean_token_accuracy": 0.7178112894296647, |
| "num_tokens": 16888973.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.13259877637028694, |
| "rewards/margins": 0.9616010505706072, |
| "rewards/rejected": -0.8290023021399975, |
| "step": 1230 |
| }, |
| { |
| "entropy": 1.001852485537529, |
| "epoch": 0.3192996008754989, |
| "grad_norm": 93.46293640136719, |
| "learning_rate": 3.9830922113186546e-07, |
| "logits/chosen": 4.4342765187607505, |
| "logits/rejected": 4.428724925246377, |
| "logps/chosen": -339.5704040527344, |
| "logps/rejected": -337.2366798400879, |
| "loss": 0.5879923820495605, |
| "mean_token_accuracy": 0.7226002395153046, |
| "num_tokens": 17035377.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": -0.16235284619033336, |
| "rewards/margins": 0.7105340275913476, |
| "rewards/rejected": -0.8728868663311005, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.9102651327848434, |
| "epoch": 0.3218745976567529, |
| "grad_norm": 64.64633178710938, |
| "learning_rate": 3.9662564535346855e-07, |
| "logits/chosen": 4.385838511702223, |
| "logits/rejected": 4.631670609424354, |
| "logps/chosen": -317.2974662780762, |
| "logps/rejected": -309.2920379638672, |
| "loss": 0.5887946128845215, |
| "mean_token_accuracy": 0.7323829025030136, |
| "num_tokens": 17171999.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.03849662467837334, |
| "rewards/margins": 0.7032071411609649, |
| "rewards/rejected": -0.7417037427425385, |
| "step": 1250 |
| }, |
| { |
| "entropy": 1.0048375755548478, |
| "epoch": 0.32444959443800697, |
| "grad_norm": 69.2200698852539, |
| "learning_rate": 3.9493187155354346e-07, |
| "logits/chosen": 4.491866410164571, |
| "logits/rejected": 4.637912356655077, |
| "logps/chosen": -348.9070739746094, |
| "logps/rejected": -355.15400009155275, |
| "loss": 0.5262919902801514, |
| "mean_token_accuracy": 0.7100594013929367, |
| "num_tokens": 17308547.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.006792543455958366, |
| "rewards/margins": 0.9484427526593209, |
| "rewards/rejected": -0.9416502088308334, |
| "step": 1260 |
| }, |
| { |
| "entropy": 1.0846450448036193, |
| "epoch": 0.327024591219261, |
| "grad_norm": 75.90949249267578, |
| "learning_rate": 3.932280175364595e-07, |
| "logits/chosen": 4.498207897925901, |
| "logits/rejected": 4.635352861664876, |
| "logps/chosen": -319.4342674255371, |
| "logps/rejected": -311.4297561645508, |
| "loss": 0.4927949905395508, |
| "mean_token_accuracy": 0.6965655654668808, |
| "num_tokens": 17433786.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.13532297760248185, |
| "rewards/margins": 0.943923632055521, |
| "rewards/rejected": -0.808600626140833, |
| "step": 1270 |
| }, |
| { |
| "entropy": 1.0548436164855957, |
| "epoch": 0.329599588000515, |
| "grad_norm": 58.16775131225586, |
| "learning_rate": 3.915142018076794e-07, |
| "logits/chosen": 4.290384003766279, |
| "logits/rejected": 4.634074884239211, |
| "logps/chosen": -356.6481246948242, |
| "logps/rejected": -339.0784294128418, |
| "loss": 0.5433180809020997, |
| "mean_token_accuracy": 0.7074305713176727, |
| "num_tokens": 17564705.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.11096932906657457, |
| "rewards/margins": 0.8167812839150429, |
| "rewards/rejected": -0.7058119729161263, |
| "step": 1280 |
| }, |
| { |
| "entropy": 1.0007994264364242, |
| "epoch": 0.33217458478176903, |
| "grad_norm": 44.396663665771484, |
| "learning_rate": 3.897905435655171e-07, |
| "logits/chosen": 4.342449735180563, |
| "logits/rejected": 4.586052330703876, |
| "logps/chosen": -333.563484954834, |
| "logps/rejected": -305.6269012451172, |
| "loss": 0.4774781703948975, |
| "mean_token_accuracy": 0.7060365378856659, |
| "num_tokens": 17698024.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.344276400655508, |
| "rewards/margins": 1.051966665685177, |
| "rewards/rejected": -0.7076902478933335, |
| "step": 1290 |
| }, |
| { |
| "entropy": 1.0160721749067307, |
| "epoch": 0.33474958156302304, |
| "grad_norm": 61.9525260925293, |
| "learning_rate": 3.8805716269284714e-07, |
| "logits/chosen": 4.30281845903861, |
| "logits/rejected": 4.424997012653698, |
| "logps/chosen": -355.3688453674316, |
| "logps/rejected": -352.252995300293, |
| "loss": 0.5308104991912842, |
| "mean_token_accuracy": 0.7141296565532684, |
| "num_tokens": 17836151.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.3676468536257744, |
| "rewards/margins": 0.8782583214342594, |
| "rewards/rejected": -0.5106114767491817, |
| "step": 1300 |
| }, |
| { |
| "entropy": 1.0081484764814377, |
| "epoch": 0.3373245783442771, |
| "grad_norm": 68.5790023803711, |
| "learning_rate": 3.863141797487669e-07, |
| "logits/chosen": 4.18447886895724, |
| "logits/rejected": 4.236553585044361, |
| "logps/chosen": -295.8922348022461, |
| "logps/rejected": -309.9488128662109, |
| "loss": 0.5453038215637207, |
| "mean_token_accuracy": 0.7150882959365845, |
| "num_tokens": 17973049.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.25624501071870326, |
| "rewards/margins": 0.9517513796687126, |
| "rewards/rejected": -0.6955063514411449, |
| "step": 1310 |
| }, |
| { |
| "entropy": 1.0145260840654373, |
| "epoch": 0.3398995751255311, |
| "grad_norm": 73.98221588134766, |
| "learning_rate": 3.8456171596021145e-07, |
| "logits/chosen": 4.320287637804293, |
| "logits/rejected": 4.558357023509025, |
| "logps/chosen": -334.7983528137207, |
| "logps/rejected": -328.91476211547854, |
| "loss": 0.5630992889404297, |
| "mean_token_accuracy": 0.7118307948112488, |
| "num_tokens": 18120880.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.2879774123430252, |
| "rewards/margins": 0.9250785499811173, |
| "rewards/rejected": -0.6371011391282082, |
| "step": 1320 |
| }, |
| { |
| "entropy": 1.0538078635931014, |
| "epoch": 0.3424745719067851, |
| "grad_norm": 54.6861686706543, |
| "learning_rate": 3.827998932135222e-07, |
| "logits/chosen": 4.52472381130375, |
| "logits/rejected": 4.5028530702015725, |
| "logps/chosen": -345.592847442627, |
| "logps/rejected": -305.8528594970703, |
| "loss": 0.475890064239502, |
| "mean_token_accuracy": 0.7067580848932267, |
| "num_tokens": 18253011.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.43196534886956217, |
| "rewards/margins": 0.9813137754797936, |
| "rewards/rejected": -0.5493484251201153, |
| "step": 1330 |
| }, |
| { |
| "entropy": 1.0047302454710008, |
| "epoch": 0.34504956868803915, |
| "grad_norm": 69.02996826171875, |
| "learning_rate": 3.810288340459689e-07, |
| "logits/chosen": 4.361984383670553, |
| "logits/rejected": 4.361619629463221, |
| "logps/chosen": -350.0064292907715, |
| "logps/rejected": -344.76106185913085, |
| "loss": 0.6270887374877929, |
| "mean_token_accuracy": 0.7146157413721085, |
| "num_tokens": 18397200.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0734201766550541, |
| "rewards/margins": 0.6079637352377176, |
| "rewards/rejected": -0.5345435634255409, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.9514402151107788, |
| "epoch": 0.34762456546929316, |
| "grad_norm": 80.87300872802734, |
| "learning_rate": 3.792486616372279e-07, |
| "logits/chosen": 4.443909030636971, |
| "logits/rejected": 4.460360202042288, |
| "logps/chosen": -329.0527824401855, |
| "logps/rejected": -343.86602249145506, |
| "loss": 0.5502840995788574, |
| "mean_token_accuracy": 0.7286958664655685, |
| "num_tokens": 18539055.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.1653471700847149, |
| "rewards/margins": 0.7919495921581984, |
| "rewards/rejected": -0.6266024131327868, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.9700424164533615, |
| "epoch": 0.3501995622505472, |
| "grad_norm": 57.26204299926758, |
| "learning_rate": 3.7745949980081425e-07, |
| "logits/chosen": 4.3594056296038275, |
| "logits/rejected": 4.593635610352538, |
| "logps/chosen": -282.6548042297363, |
| "logps/rejected": -285.6140678405762, |
| "loss": 0.5153086662292481, |
| "mean_token_accuracy": 0.7336988478899003, |
| "num_tokens": 18669012.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.40514476522803305, |
| "rewards/margins": 0.9818641506135464, |
| "rewards/rejected": -0.5767194028943777, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.9874075561761856, |
| "epoch": 0.3527745590318012, |
| "grad_norm": 65.18054962158203, |
| "learning_rate": 3.7566147297547047e-07, |
| "logits/chosen": 4.258672967953727, |
| "logits/rejected": 4.426676722125068, |
| "logps/chosen": -297.9508171081543, |
| "logps/rejected": -308.9977172851562, |
| "loss": 0.6036244392395019, |
| "mean_token_accuracy": 0.7138050466775894, |
| "num_tokens": 18801198.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.3781992755830288, |
| "rewards/margins": 0.6525024998933077, |
| "rewards/rejected": -0.27430321276187897, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.9998229205608368, |
| "epoch": 0.3553495558130552, |
| "grad_norm": 49.18499755859375, |
| "learning_rate": 3.738547062165116e-07, |
| "logits/chosen": 4.28802518209317, |
| "logits/rejected": 4.515345606367839, |
| "logps/chosen": -344.1229049682617, |
| "logps/rejected": -343.2588516235352, |
| "loss": 0.605926513671875, |
| "mean_token_accuracy": 0.7159162849187851, |
| "num_tokens": 18949925.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.18270959258079528, |
| "rewards/margins": 0.7173453290015459, |
| "rewards/rejected": -0.5346357382833957, |
| "step": 1380 |
| }, |
| { |
| "entropy": 1.0593062907457351, |
| "epoch": 0.3579245525943093, |
| "grad_norm": 69.1773910522461, |
| "learning_rate": 3.7203932518712734e-07, |
| "logits/chosen": 4.3985165466469995, |
| "logits/rejected": 4.648432641296062, |
| "logps/chosen": -331.2413200378418, |
| "logps/rejected": -317.4773864746094, |
| "loss": 0.6042040824890137, |
| "mean_token_accuracy": 0.7046501845121383, |
| "num_tokens": 19071642.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.26102426275610924, |
| "rewards/margins": 0.6165750399231911, |
| "rewards/rejected": -0.35555077604949475, |
| "step": 1390 |
| }, |
| { |
| "entropy": 1.0222451746463777, |
| "epoch": 0.3604995493755633, |
| "grad_norm": 61.763160705566406, |
| "learning_rate": 3.702154561496423e-07, |
| "logits/chosen": 4.1127401210487795, |
| "logits/rejected": 4.284693891063448, |
| "logps/chosen": -318.31056213378906, |
| "logps/rejected": -295.82591705322267, |
| "loss": 0.5423140048980712, |
| "mean_token_accuracy": 0.7130143046379089, |
| "num_tokens": 19213285.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.2595692839473486, |
| "rewards/margins": 0.8101298615336419, |
| "rewards/rejected": -0.5505605675280094, |
| "step": 1400 |
| }, |
| { |
| "entropy": 1.0160330265760422, |
| "epoch": 0.3630745461568173, |
| "grad_norm": 77.30133819580078, |
| "learning_rate": 3.683832259567341e-07, |
| "logits/chosen": 4.499462280513063, |
| "logits/rejected": 4.526303443830601, |
| "logps/chosen": -337.373348236084, |
| "logps/rejected": -337.92723617553713, |
| "loss": 0.5844311237335205, |
| "mean_token_accuracy": 0.7160362660884857, |
| "num_tokens": 19348248.0, |
| "rewards/accuracies": 0.66875, |
| "rewards/chosen": 0.39422787353396416, |
| "rewards/margins": 0.6997637458145618, |
| "rewards/rejected": -0.3055358748883009, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.9416633933782578, |
| "epoch": 0.36564954293807134, |
| "grad_norm": 43.26420974731445, |
| "learning_rate": 3.6654276204261043e-07, |
| "logits/chosen": 4.5233009591843665, |
| "logits/rejected": 4.598622006642471, |
| "logps/chosen": -323.8805061340332, |
| "logps/rejected": -315.74465255737306, |
| "loss": 0.5712813854217529, |
| "mean_token_accuracy": 0.7368648052215576, |
| "num_tokens": 19493233.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.342887281998992, |
| "rewards/margins": 0.748133927769959, |
| "rewards/rejected": -0.4052466508001089, |
| "step": 1420 |
| }, |
| { |
| "entropy": 1.0600703418254853, |
| "epoch": 0.36822453971932534, |
| "grad_norm": 60.04607009887695, |
| "learning_rate": 3.64694192414146e-07, |
| "logits/chosen": 4.482818825251574, |
| "logits/rejected": 4.374184376101063, |
| "logps/chosen": -346.8372741699219, |
| "logps/rejected": -318.7594619750977, |
| "loss": 0.5119295597076416, |
| "mean_token_accuracy": 0.7020805537700653, |
| "num_tokens": 19627479.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.5068976059556007, |
| "rewards/margins": 0.9839373663067817, |
| "rewards/rejected": -0.4770397573709488, |
| "step": 1430 |
| }, |
| { |
| "entropy": 1.0695737957954408, |
| "epoch": 0.3707995365005794, |
| "grad_norm": 66.99427795410156, |
| "learning_rate": 3.6283764564197947e-07, |
| "logits/chosen": 4.178093219183054, |
| "logits/rejected": 4.626961773469877, |
| "logps/chosen": -291.58864135742186, |
| "logps/rejected": -294.04699249267577, |
| "loss": 0.496781587600708, |
| "mean_token_accuracy": 0.7073402464389801, |
| "num_tokens": 19752558.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.40894790813326837, |
| "rewards/margins": 0.8948969356715679, |
| "rewards/rejected": -0.485949020460248, |
| "step": 1440 |
| }, |
| { |
| "entropy": 1.0322244733572006, |
| "epoch": 0.3733745332818334, |
| "grad_norm": 43.88212585449219, |
| "learning_rate": 3.6097325085157126e-07, |
| "logits/chosen": 4.574426650801122, |
| "logits/rejected": 4.561950068538189, |
| "logps/chosen": -354.1114860534668, |
| "logps/rejected": -308.34269485473635, |
| "loss": 0.5143645286560059, |
| "mean_token_accuracy": 0.7042175978422165, |
| "num_tokens": 19892893.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.3972885079681873, |
| "rewards/margins": 0.8085891950875521, |
| "rewards/rejected": -0.4113006766885519, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.9956100940704345, |
| "epoch": 0.3759495300630874, |
| "grad_norm": 31.5433406829834, |
| "learning_rate": 3.5910113771422234e-07, |
| "logits/chosen": 4.565448950462079, |
| "logits/rejected": 4.586300662800534, |
| "logps/chosen": -320.2218307495117, |
| "logps/rejected": -307.1070892333984, |
| "loss": 0.5700956821441651, |
| "mean_token_accuracy": 0.7226622521877288, |
| "num_tokens": 20032058.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.3020356763154268, |
| "rewards/margins": 0.6896195448935032, |
| "rewards/rejected": -0.38758386448025706, |
| "step": 1460 |
| }, |
| { |
| "entropy": 1.0120248764753341, |
| "epoch": 0.37852452684434146, |
| "grad_norm": 59.42012405395508, |
| "learning_rate": 3.572214364380559e-07, |
| "logits/chosen": 4.384914371666054, |
| "logits/rejected": 4.491667970276556, |
| "logps/chosen": -344.7418045043945, |
| "logps/rejected": -315.87948684692384, |
| "loss": 0.47869324684143066, |
| "mean_token_accuracy": 0.7147729307413101, |
| "num_tokens": 20168549.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.47410065568983556, |
| "rewards/margins": 1.035487013310194, |
| "rewards/rejected": -0.5613863591104746, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.9572999686002731, |
| "epoch": 0.38109952362559546, |
| "grad_norm": 34.17436218261719, |
| "learning_rate": 3.5533427775896073e-07, |
| "logits/chosen": 4.303627878264292, |
| "logits/rejected": 4.463746868966824, |
| "logps/chosen": -319.0563430786133, |
| "logps/rejected": -348.01474838256837, |
| "loss": 0.5226101875305176, |
| "mean_token_accuracy": 0.7151152938604355, |
| "num_tokens": 20315971.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.36733451671898365, |
| "rewards/margins": 0.8375594437122345, |
| "rewards/rejected": -0.4702249463647604, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.9907828450202942, |
| "epoch": 0.3836745204068495, |
| "grad_norm": 35.07666015625, |
| "learning_rate": 3.534397929314988e-07, |
| "logits/chosen": 4.499505547386105, |
| "logits/rejected": 4.559010892291426, |
| "logps/chosen": -343.87607803344724, |
| "logps/rejected": -307.31992568969724, |
| "loss": 0.46512527465820314, |
| "mean_token_accuracy": 0.7122376918792724, |
| "num_tokens": 20448199.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.41770453490316867, |
| "rewards/margins": 1.0182910978794097, |
| "rewards/rejected": -0.6005865626037121, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.9885411828756332, |
| "epoch": 0.3862495171881035, |
| "grad_norm": 33.83668518066406, |
| "learning_rate": 3.51538113719776e-07, |
| "logits/chosen": 4.585875986271088, |
| "logits/rejected": 4.692986495571829, |
| "logps/chosen": -342.3488739013672, |
| "logps/rejected": -335.82422943115233, |
| "loss": 0.5086318969726562, |
| "mean_token_accuracy": 0.7093215554952621, |
| "num_tokens": 20592523.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.42292424775660037, |
| "rewards/margins": 0.9259717024862766, |
| "rewards/rejected": -0.5030474781990051, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.9731161743402481, |
| "epoch": 0.3888245139693575, |
| "grad_norm": 163.65415954589844, |
| "learning_rate": 3.496293723882776e-07, |
| "logits/chosen": 4.448863749394176, |
| "logits/rejected": 4.3950092747068945, |
| "logps/chosen": -330.829727935791, |
| "logps/rejected": -316.89535598754884, |
| "loss": 0.5070469379425049, |
| "mean_token_accuracy": 0.7174982994794845, |
| "num_tokens": 20735839.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.4557771056890488, |
| "rewards/margins": 0.8968748055398464, |
| "rewards/rejected": -0.44109772704541683, |
| "step": 1510 |
| }, |
| { |
| "entropy": 1.0547097712755202, |
| "epoch": 0.3913995107506116, |
| "grad_norm": 50.195186614990234, |
| "learning_rate": 3.4771370169266973e-07, |
| "logits/chosen": 4.553992445601459, |
| "logits/rejected": 4.251664231203607, |
| "logps/chosen": -335.4343605041504, |
| "logps/rejected": -325.768017578125, |
| "loss": 0.5149908542633057, |
| "mean_token_accuracy": 0.6950643092393876, |
| "num_tokens": 20876847.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.42625158168375493, |
| "rewards/margins": 0.9463345900177955, |
| "rewards/rejected": -0.5200830094516278, |
| "step": 1520 |
| }, |
| { |
| "entropy": 1.037656196951866, |
| "epoch": 0.3939745075318656, |
| "grad_norm": 59.47888946533203, |
| "learning_rate": 3.457912348705653e-07, |
| "logits/chosen": 4.438172884191337, |
| "logits/rejected": 4.536922108194634, |
| "logps/chosen": -335.8483200073242, |
| "logps/rejected": -330.42218856811525, |
| "loss": 0.522545337677002, |
| "mean_token_accuracy": 0.7065093517303467, |
| "num_tokens": 21006575.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.2367303378880024, |
| "rewards/margins": 0.9508240945637226, |
| "rewards/rejected": -0.7140937626361847, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.9364173501729965, |
| "epoch": 0.3965495043131196, |
| "grad_norm": 82.34601593017578, |
| "learning_rate": 3.438621056322576e-07, |
| "logits/chosen": 4.373839777033968, |
| "logits/rejected": 4.7794054819041225, |
| "logps/chosen": -298.2203536987305, |
| "logps/rejected": -336.3901840209961, |
| "loss": 0.5983014106750488, |
| "mean_token_accuracy": 0.7462542623281478, |
| "num_tokens": 21167374.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.2251519788056612, |
| "rewards/margins": 0.813858188688755, |
| "rewards/rejected": -0.5887062273919582, |
| "step": 1540 |
| }, |
| { |
| "entropy": 1.0086449295282365, |
| "epoch": 0.39912450109437364, |
| "grad_norm": 60.29326629638672, |
| "learning_rate": 3.4192644815142035e-07, |
| "logits/chosen": 4.212622763483619, |
| "logits/rejected": 4.329810739793858, |
| "logps/chosen": -321.5377799987793, |
| "logps/rejected": -311.98607025146487, |
| "loss": 0.46921277046203613, |
| "mean_token_accuracy": 0.7168540626764297, |
| "num_tokens": 21299826.0, |
| "rewards/accuracies": 0.8, |
| "rewards/chosen": 0.3227839909493923, |
| "rewards/margins": 0.9929535686969757, |
| "rewards/rejected": -0.6701695755124092, |
| "step": 1550 |
| }, |
| { |
| "entropy": 1.0051219373941422, |
| "epoch": 0.40169949787562764, |
| "grad_norm": 64.10111999511719, |
| "learning_rate": 3.3998439705577587e-07, |
| "logits/chosen": 4.336165566431968, |
| "logits/rejected": 4.434400236337686, |
| "logps/chosen": -340.68482971191406, |
| "logps/rejected": -334.61559906005857, |
| "loss": 0.5105271339416504, |
| "mean_token_accuracy": 0.7102333426475524, |
| "num_tokens": 21437856.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.20026715993881225, |
| "rewards/margins": 0.8899520173668861, |
| "rewards/rejected": -0.689684884250164, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.9653350681066513, |
| "epoch": 0.4042744946568817, |
| "grad_norm": 57.03731918334961, |
| "learning_rate": 3.3803608741773137e-07, |
| "logits/chosen": 4.112749837286741, |
| "logits/rejected": 4.341371862717738, |
| "logps/chosen": -324.7610061645508, |
| "logps/rejected": -320.7888603210449, |
| "loss": 0.5658888339996337, |
| "mean_token_accuracy": 0.7201475411653518, |
| "num_tokens": 21578723.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.16358607187867164, |
| "rewards/margins": 0.8124723732471466, |
| "rewards/rejected": -0.6488862888887524, |
| "step": 1570 |
| }, |
| { |
| "entropy": 1.0187890470027923, |
| "epoch": 0.4068494914381357, |
| "grad_norm": 48.15790939331055, |
| "learning_rate": 3.3608165474498463e-07, |
| "logits/chosen": 4.511571607388313, |
| "logits/rejected": 4.608273734629913, |
| "logps/chosen": -290.37970581054685, |
| "logps/rejected": -274.29333267211916, |
| "loss": 0.5271549224853516, |
| "mean_token_accuracy": 0.7093697398900985, |
| "num_tokens": 21698766.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.45765426084399224, |
| "rewards/margins": 0.9066750418394804, |
| "rewards/rejected": -0.44902077317237854, |
| "step": 1580 |
| }, |
| { |
| "entropy": 1.0284575790166854, |
| "epoch": 0.4094244882193897, |
| "grad_norm": 45.4733772277832, |
| "learning_rate": 3.341212349710991e-07, |
| "logits/chosen": 4.1811476539506955, |
| "logits/rejected": 4.1478347758355625, |
| "logps/chosen": -348.23877105712893, |
| "logps/rejected": -321.8086814880371, |
| "loss": 0.5493577480316162, |
| "mean_token_accuracy": 0.7045906782150269, |
| "num_tokens": 21826816.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.2270603898912668, |
| "rewards/margins": 0.7640220403671265, |
| "rewards/rejected": -0.5369616780430079, |
| "step": 1590 |
| }, |
| { |
| "entropy": 1.0212238907814026, |
| "epoch": 0.41199948500064376, |
| "grad_norm": 89.39241790771484, |
| "learning_rate": 3.3215496444604964e-07, |
| "logits/chosen": 4.532756426511286, |
| "logits/rejected": 4.629493190244512, |
| "logps/chosen": -339.993953704834, |
| "logps/rejected": -311.24218521118166, |
| "loss": 0.5322290420532226, |
| "mean_token_accuracy": 0.7075557291507721, |
| "num_tokens": 21968861.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.2599800042808056, |
| "rewards/margins": 0.9229485109448433, |
| "rewards/rejected": -0.6629685159772635, |
| "step": 1600 |
| }, |
| { |
| "entropy": 1.0632631063461304, |
| "epoch": 0.41457448178189776, |
| "grad_norm": 41.36448287963867, |
| "learning_rate": 3.3018297992673924e-07, |
| "logits/chosen": 4.507129678744716, |
| "logits/rejected": 4.7003194517972515, |
| "logps/chosen": -340.9761138916016, |
| "logps/rejected": -311.91967544555666, |
| "loss": 0.4883114814758301, |
| "mean_token_accuracy": 0.6983400255441665, |
| "num_tokens": 22106242.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.480826123803854, |
| "rewards/margins": 1.0077935695648192, |
| "rewards/rejected": -0.526967428997159, |
| "step": 1610 |
| }, |
| { |
| "entropy": 1.0190553188323974, |
| "epoch": 0.4171494785631518, |
| "grad_norm": 93.61823272705078, |
| "learning_rate": 3.282054185674871e-07, |
| "logits/chosen": 4.566390577731012, |
| "logits/rejected": 4.165170132755452, |
| "logps/chosen": -347.6525405883789, |
| "logps/rejected": -335.3510269165039, |
| "loss": 0.5531125545501709, |
| "mean_token_accuracy": 0.6961113244295121, |
| "num_tokens": 22247306.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.30568349808454515, |
| "rewards/margins": 0.7751612067222595, |
| "rewards/rejected": -0.46947770081460477, |
| "step": 1620 |
| }, |
| { |
| "entropy": 1.0008302301168441, |
| "epoch": 0.4197244753444058, |
| "grad_norm": 71.16085815429688, |
| "learning_rate": 3.2622241791048964e-07, |
| "logits/chosen": 4.582875345835534, |
| "logits/rejected": 4.507340951874982, |
| "logps/chosen": -308.279963684082, |
| "logps/rejected": -283.0868232727051, |
| "loss": 0.669698429107666, |
| "mean_token_accuracy": 0.7173137634992599, |
| "num_tokens": 22381348.0, |
| "rewards/accuracies": 0.60625, |
| "rewards/chosen": 0.3391913536936045, |
| "rewards/margins": 0.5125614479184151, |
| "rewards/rejected": -0.17337006516754627, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.9887471556663513, |
| "epoch": 0.4222994721256598, |
| "grad_norm": 51.84771728515625, |
| "learning_rate": 3.242341158762543e-07, |
| "logits/chosen": 4.416430634733047, |
| "logits/rejected": 4.549421839075295, |
| "logps/chosen": -342.06642150878906, |
| "logps/rejected": -331.0380546569824, |
| "loss": 0.4823248863220215, |
| "mean_token_accuracy": 0.7201027363538742, |
| "num_tokens": 22528483.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.3955455271527171, |
| "rewards/margins": 1.0124763235449792, |
| "rewards/rejected": -0.6169307995587587, |
| "step": 1640 |
| }, |
| { |
| "entropy": 1.0046769827604294, |
| "epoch": 0.4248744689069139, |
| "grad_norm": 45.738059997558594, |
| "learning_rate": 3.222406507540066e-07, |
| "logits/chosen": 4.169730618839731, |
| "logits/rejected": 4.504782422516071, |
| "logps/chosen": -318.0526466369629, |
| "logps/rejected": -323.7085357666016, |
| "loss": 0.4680798530578613, |
| "mean_token_accuracy": 0.7223367244005203, |
| "num_tokens": 22667434.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.32473539747297764, |
| "rewards/margins": 1.0296514950692655, |
| "rewards/rejected": -0.7049161233007908, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.9911033987998963, |
| "epoch": 0.4274494656881679, |
| "grad_norm": 62.42966079711914, |
| "learning_rate": 3.2024216119207215e-07, |
| "logits/chosen": 4.388264832498901, |
| "logits/rejected": 4.388930285050414, |
| "logps/chosen": -344.140438079834, |
| "logps/rejected": -330.0677917480469, |
| "loss": 0.5633188724517822, |
| "mean_token_accuracy": 0.7085156321525574, |
| "num_tokens": 22799973.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.24126881808042527, |
| "rewards/margins": 0.7521100163459777, |
| "rewards/rejected": -0.510841203480959, |
| "step": 1660 |
| }, |
| { |
| "entropy": 1.0932520985603333, |
| "epoch": 0.4300244624694219, |
| "grad_norm": 56.770660400390625, |
| "learning_rate": 3.182387861882337e-07, |
| "logits/chosen": 4.264878523978969, |
| "logits/rejected": 4.41654907158112, |
| "logps/chosen": -350.5899627685547, |
| "logps/rejected": -378.4750503540039, |
| "loss": 0.5012292385101318, |
| "mean_token_accuracy": 0.7019307494163514, |
| "num_tokens": 22942405.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.3956910580396652, |
| "rewards/margins": 1.0485083594918252, |
| "rewards/rejected": -0.6528172954916954, |
| "step": 1670 |
| }, |
| { |
| "entropy": 1.0346484899520874, |
| "epoch": 0.43259945925067594, |
| "grad_norm": 60.96067810058594, |
| "learning_rate": 3.162306650800632e-07, |
| "logits/chosen": 4.381951759660245, |
| "logits/rejected": 4.3761065400867185, |
| "logps/chosen": -319.1274444580078, |
| "logps/rejected": -284.38352127075194, |
| "loss": 0.5894469738006591, |
| "mean_token_accuracy": 0.6966201603412628, |
| "num_tokens": 23073290.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.222568079829216, |
| "rewards/margins": 0.7801389444619418, |
| "rewards/rejected": -0.5575708828866481, |
| "step": 1680 |
| }, |
| { |
| "entropy": 1.0130227506160736, |
| "epoch": 0.43517445603192995, |
| "grad_norm": 88.51482391357422, |
| "learning_rate": 3.1421793753523085e-07, |
| "logits/chosen": 4.153179485330449, |
| "logits/rejected": 4.468525957066152, |
| "logps/chosen": -347.164697265625, |
| "logps/rejected": -333.5994499206543, |
| "loss": 0.5185692787170411, |
| "mean_token_accuracy": 0.7093759685754776, |
| "num_tokens": 23208834.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.19649834968149663, |
| "rewards/margins": 0.9458184786140919, |
| "rewards/rejected": -0.7493201218545437, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.9987653195858002, |
| "epoch": 0.437749452813184, |
| "grad_norm": 59.26670837402344, |
| "learning_rate": 3.1220074354179134e-07, |
| "logits/chosen": 4.447430238036, |
| "logits/rejected": 4.345364868742253, |
| "logps/chosen": -335.15634689331057, |
| "logps/rejected": -322.3852783203125, |
| "loss": 0.5455127239227295, |
| "mean_token_accuracy": 0.707678034901619, |
| "num_tokens": 23339859.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.10780363157391548, |
| "rewards/margins": 0.8008625339716673, |
| "rewards/rejected": -0.6930589221417904, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.9553779780864715, |
| "epoch": 0.440324449594438, |
| "grad_norm": 67.1646728515625, |
| "learning_rate": 3.101792233984468e-07, |
| "logits/chosen": 4.317778320205003, |
| "logits/rejected": 4.481137207239907, |
| "logps/chosen": -333.0953964233398, |
| "logps/rejected": -353.4780700683594, |
| "loss": 0.5301013469696045, |
| "mean_token_accuracy": 0.72568961083889, |
| "num_tokens": 23483179.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.2193666160106659, |
| "rewards/margins": 0.8763925023376942, |
| "rewards/rejected": -0.6570258844643831, |
| "step": 1710 |
| }, |
| { |
| "entropy": 1.0156360119581223, |
| "epoch": 0.442899446375692, |
| "grad_norm": 63.09981155395508, |
| "learning_rate": 3.081535177047896e-07, |
| "logits/chosen": 4.205427199102777, |
| "logits/rejected": 4.318676160866055, |
| "logps/chosen": -335.0963165283203, |
| "logps/rejected": -340.5821472167969, |
| "loss": 0.5540195941925049, |
| "mean_token_accuracy": 0.7033586442470551, |
| "num_tokens": 23624445.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.2030243966728449, |
| "rewards/margins": 0.8909270189702511, |
| "rewards/rejected": -0.6879026241600513, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.9994647204875946, |
| "epoch": 0.44547444315694606, |
| "grad_norm": 59.10143280029297, |
| "learning_rate": 3.061237673515228e-07, |
| "logits/chosen": 4.475974708431387, |
| "logits/rejected": 4.4807601896793, |
| "logps/chosen": -341.10973205566404, |
| "logps/rejected": -337.8475936889648, |
| "loss": 0.5034106254577637, |
| "mean_token_accuracy": 0.7194854825735092, |
| "num_tokens": 23772998.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.2501206941902637, |
| "rewards/margins": 1.0072236776351928, |
| "rewards/rejected": -0.7571029774844646, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.9580833286046981, |
| "epoch": 0.44804943993820007, |
| "grad_norm": 65.97066497802734, |
| "learning_rate": 3.040901135106614e-07, |
| "logits/chosen": 4.6918887133458735, |
| "logits/rejected": 4.679432626279566, |
| "logps/chosen": -314.99814758300784, |
| "logps/rejected": -301.6247161865234, |
| "loss": 0.5239108085632325, |
| "mean_token_accuracy": 0.7052569478750229, |
| "num_tokens": 23904394.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.13549816319718958, |
| "rewards/margins": 0.9139077693223954, |
| "rewards/rejected": -0.7784095827490092, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.9939440310001373, |
| "epoch": 0.4506244367194541, |
| "grad_norm": 47.2442741394043, |
| "learning_rate": 3.020526976257133e-07, |
| "logits/chosen": 4.3844166827864575, |
| "logits/rejected": 4.618266336702991, |
| "logps/chosen": -324.671704864502, |
| "logps/rejected": -311.7133255004883, |
| "loss": 0.5350105285644531, |
| "mean_token_accuracy": 0.7167317628860473, |
| "num_tokens": 24029143.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.28501694574952124, |
| "rewards/margins": 0.875099265575409, |
| "rewards/rejected": -0.59008234962821, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.9728594481945038, |
| "epoch": 0.4531994335007081, |
| "grad_norm": 52.896759033203125, |
| "learning_rate": 3.000116614018423e-07, |
| "logits/chosen": 4.347222045065487, |
| "logits/rejected": 4.491772509839542, |
| "logps/chosen": -345.4105422973633, |
| "logps/rejected": -311.8397155761719, |
| "loss": 0.5155512809753418, |
| "mean_token_accuracy": 0.7080474406480789, |
| "num_tokens": 24168843.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.22332061640918255, |
| "rewards/margins": 0.9133861869573593, |
| "rewards/rejected": -0.6900655575096607, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.9716287791728974, |
| "epoch": 0.45577443028196213, |
| "grad_norm": 69.11155700683594, |
| "learning_rate": 2.9796714679601153e-07, |
| "logits/chosen": 4.07666419766162, |
| "logits/rejected": 4.541536803168063, |
| "logps/chosen": -333.53333587646483, |
| "logps/rejected": -364.5257148742676, |
| "loss": 0.5594016551971436, |
| "mean_token_accuracy": 0.7196703106164932, |
| "num_tokens": 24313406.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.09260394554585219, |
| "rewards/margins": 0.7315813470631838, |
| "rewards/rejected": -0.6389774307608604, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.968604001402855, |
| "epoch": 0.4583494270632162, |
| "grad_norm": 75.13587951660156, |
| "learning_rate": 2.9591929600711064e-07, |
| "logits/chosen": 4.4954023446079905, |
| "logits/rejected": 4.318362773308535, |
| "logps/chosen": -354.32204360961913, |
| "logps/rejected": -324.70655517578126, |
| "loss": 0.6192696571350098, |
| "mean_token_accuracy": 0.7113049805164338, |
| "num_tokens": 24458893.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.11326459385454654, |
| "rewards/margins": 0.6298577159643173, |
| "rewards/rejected": -0.5165931291878223, |
| "step": 1780 |
| }, |
| { |
| "entropy": 1.0130292028188705, |
| "epoch": 0.4609244238444702, |
| "grad_norm": 42.6855354309082, |
| "learning_rate": 2.9386825146606556e-07, |
| "logits/chosen": 4.242645254840911, |
| "logits/rejected": 4.208730296274518, |
| "logps/chosen": -302.8190490722656, |
| "logps/rejected": -287.4302803039551, |
| "loss": 0.5245326519012451, |
| "mean_token_accuracy": 0.7104728162288666, |
| "num_tokens": 24592529.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.13887676037847996, |
| "rewards/margins": 0.9169051334261894, |
| "rewards/rejected": -0.7780283778905869, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.939438796043396, |
| "epoch": 0.46349942062572425, |
| "grad_norm": 53.44596481323242, |
| "learning_rate": 2.918141558259323e-07, |
| "logits/chosen": 4.425804985311169, |
| "logits/rejected": 4.579291347565833, |
| "logps/chosen": -314.8403045654297, |
| "logps/rejected": -319.444881439209, |
| "loss": 0.5266315937042236, |
| "mean_token_accuracy": 0.7189203172922134, |
| "num_tokens": 24727873.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.22815715186297894, |
| "rewards/margins": 0.9557700768113137, |
| "rewards/rejected": -0.7276129089295864, |
| "step": 1800 |
| }, |
| { |
| "entropy": 1.017608442902565, |
| "epoch": 0.46607441740697825, |
| "grad_norm": 75.32769775390625, |
| "learning_rate": 2.897571519519749e-07, |
| "logits/chosen": 4.325712846533475, |
| "logits/rejected": 4.327608559472024, |
| "logps/chosen": -321.98156585693357, |
| "logps/rejected": -333.84306411743165, |
| "loss": 0.5792953968048096, |
| "mean_token_accuracy": 0.7041509211063385, |
| "num_tokens": 24867266.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.039030957221984866, |
| "rewards/margins": 0.7474001370370388, |
| "rewards/rejected": -0.7083691634237766, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.8716294199228287, |
| "epoch": 0.46864941418823225, |
| "grad_norm": 69.67833709716797, |
| "learning_rate": 2.8769738291172937e-07, |
| "logits/chosen": 4.729007949358574, |
| "logits/rejected": 4.748424687543743, |
| "logps/chosen": -286.38878173828124, |
| "logps/rejected": -315.22359466552734, |
| "loss": 0.4433790683746338, |
| "mean_token_accuracy": 0.7467392474412918, |
| "num_tokens": 25008627.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.3098715987056494, |
| "rewards/margins": 1.0342625588178636, |
| "rewards/rejected": -0.7243909515440464, |
| "step": 1820 |
| }, |
| { |
| "entropy": 1.0027720093727113, |
| "epoch": 0.4712244109694863, |
| "grad_norm": 58.2140007019043, |
| "learning_rate": 2.856349919650529e-07, |
| "logits/chosen": 4.342801686238408, |
| "logits/rejected": 4.463185245965893, |
| "logps/chosen": -312.2874137878418, |
| "logps/rejected": -302.03475723266604, |
| "loss": 0.49503297805786134, |
| "mean_token_accuracy": 0.7056602656841278, |
| "num_tokens": 25144657.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.2989152632653713, |
| "rewards/margins": 1.038015954941511, |
| "rewards/rejected": -0.739100681990385, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.9499877840280533, |
| "epoch": 0.4737994077507403, |
| "grad_norm": 66.1916275024414, |
| "learning_rate": 2.835701225541601e-07, |
| "logits/chosen": 4.750521566268771, |
| "logits/rejected": 4.538883212279146, |
| "logps/chosen": -328.4700454711914, |
| "logps/rejected": -303.7730224609375, |
| "loss": 0.5446010112762452, |
| "mean_token_accuracy": 0.7132113128900528, |
| "num_tokens": 25277668.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.16393240839242934, |
| "rewards/margins": 0.8867032006382942, |
| "rewards/rejected": -0.7227707915008068, |
| "step": 1840 |
| }, |
| { |
| "entropy": 1.031875056028366, |
| "epoch": 0.4763744045319943, |
| "grad_norm": 66.70272064208984, |
| "learning_rate": 2.815029182936461e-07, |
| "logits/chosen": 4.415904153295584, |
| "logits/rejected": 4.60825803093547, |
| "logps/chosen": -355.8702056884766, |
| "logps/rejected": -331.3533233642578, |
| "loss": 0.4908586025238037, |
| "mean_token_accuracy": 0.6996257096529007, |
| "num_tokens": 25417326.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.27313601449131963, |
| "rewards/margins": 1.024405437707901, |
| "rewards/rejected": -0.751269418001175, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.9652377277612686, |
| "epoch": 0.47894940131324837, |
| "grad_norm": 51.2459716796875, |
| "learning_rate": 2.7943352296049817e-07, |
| "logits/chosen": 4.381936753944478, |
| "logits/rejected": 4.503009937215559, |
| "logps/chosen": -319.6825668334961, |
| "logps/rejected": -317.8706504821777, |
| "loss": 0.49027070999145506, |
| "mean_token_accuracy": 0.7136070847511291, |
| "num_tokens": 25549942.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.12705591693520546, |
| "rewards/margins": 1.0133403688669205, |
| "rewards/rejected": -0.8862844713032245, |
| "step": 1860 |
| }, |
| { |
| "entropy": 0.9866015404462815, |
| "epoch": 0.48152439809450237, |
| "grad_norm": 48.989219665527344, |
| "learning_rate": 2.773620804840959e-07, |
| "logits/chosen": 4.288209644175543, |
| "logits/rejected": 4.304385423177918, |
| "logps/chosen": -311.4419464111328, |
| "logps/rejected": -301.3541030883789, |
| "loss": 0.5566805839538574, |
| "mean_token_accuracy": 0.7075719892978668, |
| "num_tokens": 25680942.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.1245550274848938, |
| "rewards/margins": 0.8268403589725495, |
| "rewards/rejected": -0.7022853277623653, |
| "step": 1870 |
| }, |
| { |
| "entropy": 0.9705629885196686, |
| "epoch": 0.48409939487575643, |
| "grad_norm": 52.36819076538086, |
| "learning_rate": 2.7528873493620057e-07, |
| "logits/chosen": 4.472752466035425, |
| "logits/rejected": 4.677669941701231, |
| "logps/chosen": -368.9326988220215, |
| "logps/rejected": -323.53197250366213, |
| "loss": 0.509603214263916, |
| "mean_token_accuracy": 0.7059235632419586, |
| "num_tokens": 25819603.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.1403234925121069, |
| "rewards/margins": 0.8788333332166076, |
| "rewards/rejected": -0.7385098546743393, |
| "step": 1880 |
| }, |
| { |
| "entropy": 0.995087307691574, |
| "epoch": 0.48667439165701043, |
| "grad_norm": 62.67254638671875, |
| "learning_rate": 2.732136305209347e-07, |
| "logits/chosen": 4.383197580484635, |
| "logits/rejected": 4.69916580317545, |
| "logps/chosen": -340.8980545043945, |
| "logps/rejected": -320.3641952514648, |
| "loss": 0.5173936367034913, |
| "mean_token_accuracy": 0.7051976472139359, |
| "num_tokens": 25951235.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.16847603507339953, |
| "rewards/margins": 1.022395919263363, |
| "rewards/rejected": -0.8539198614656925, |
| "step": 1890 |
| }, |
| { |
| "entropy": 0.9622721552848816, |
| "epoch": 0.48924938843826443, |
| "grad_norm": 45.17765426635742, |
| "learning_rate": 2.711369115647525e-07, |
| "logits/chosen": 4.274382834525982, |
| "logits/rejected": 4.567524067046074, |
| "logps/chosen": -298.41376571655275, |
| "logps/rejected": -279.6089782714844, |
| "loss": 0.5523276329040527, |
| "mean_token_accuracy": 0.7140445560216904, |
| "num_tokens": 26086978.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.1847171064466238, |
| "rewards/margins": 0.7757710345089436, |
| "rewards/rejected": -0.591053931415081, |
| "step": 1900 |
| }, |
| { |
| "entropy": 0.9933630079030991, |
| "epoch": 0.4918243852195185, |
| "grad_norm": 48.34173583984375, |
| "learning_rate": 2.690587225064019e-07, |
| "logits/chosen": 4.248255535456623, |
| "logits/rejected": 4.705522751035231, |
| "logps/chosen": -320.63128356933595, |
| "logps/rejected": -322.36269226074216, |
| "loss": 0.5072047233581543, |
| "mean_token_accuracy": 0.7172810703516006, |
| "num_tokens": 26215024.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.07452852092683315, |
| "rewards/margins": 1.0095476165413857, |
| "rewards/rejected": -0.9350190743803978, |
| "step": 1910 |
| }, |
| { |
| "entropy": 1.03436821103096, |
| "epoch": 0.4943993820007725, |
| "grad_norm": 43.470863342285156, |
| "learning_rate": 2.669792078868785e-07, |
| "logits/chosen": 4.445727344457447, |
| "logits/rejected": 4.469930971599314, |
| "logps/chosen": -319.6712905883789, |
| "logps/rejected": -319.20749282836914, |
| "loss": 0.4812634944915771, |
| "mean_token_accuracy": 0.6964850157499314, |
| "num_tokens": 26353198.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.27224585637450216, |
| "rewards/margins": 1.200338999927044, |
| "rewards/rejected": -0.9280931562185287, |
| "step": 1920 |
| }, |
| { |
| "entropy": 0.9880954831838608, |
| "epoch": 0.49697437878202655, |
| "grad_norm": 60.059425354003906, |
| "learning_rate": 2.6489851233937254e-07, |
| "logits/chosen": 4.360795670967695, |
| "logits/rejected": 4.647031218142274, |
| "logps/chosen": -337.48805923461913, |
| "logps/rejected": -315.1701011657715, |
| "loss": 0.5536554336547852, |
| "mean_token_accuracy": 0.7152973026037216, |
| "num_tokens": 26490614.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.12552479170262815, |
| "rewards/margins": 0.8430479489266872, |
| "rewards/rejected": -0.9685727544128895, |
| "step": 1930 |
| }, |
| { |
| "entropy": 0.930216184258461, |
| "epoch": 0.49954937556328055, |
| "grad_norm": 57.515750885009766, |
| "learning_rate": 2.628167805792094e-07, |
| "logits/chosen": 4.484900366278743, |
| "logits/rejected": 4.580445152126293, |
| "logps/chosen": -337.2224060058594, |
| "logps/rejected": -303.9986000061035, |
| "loss": 0.47016096115112305, |
| "mean_token_accuracy": 0.7162662833929062, |
| "num_tokens": 26633836.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.27869669497013094, |
| "rewards/margins": 1.0826191306114197, |
| "rewards/rejected": -0.803922425955534, |
| "step": 1940 |
| }, |
| { |
| "entropy": 0.9742702841758728, |
| "epoch": 0.5021243723445346, |
| "grad_norm": 39.161773681640625, |
| "learning_rate": 2.607341573937847e-07, |
| "logits/chosen": 4.283901718371682, |
| "logits/rejected": 4.398404167911892, |
| "logps/chosen": -346.7345329284668, |
| "logps/rejected": -307.449609375, |
| "loss": 0.6053523063659668, |
| "mean_token_accuracy": 0.7093511551618576, |
| "num_tokens": 26772785.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.0554534487426281, |
| "rewards/margins": 0.7310394756495953, |
| "rewards/rejected": -0.6755860306322574, |
| "step": 1950 |
| }, |
| { |
| "entropy": 1.0009024798870088, |
| "epoch": 0.5046993691257886, |
| "grad_norm": 65.297119140625, |
| "learning_rate": 2.5865078763249374e-07, |
| "logits/chosen": 4.496857392609219, |
| "logits/rejected": 4.232990657321963, |
| "logps/chosen": -321.77712478637693, |
| "logps/rejected": -332.6124717712402, |
| "loss": 0.5037482261657715, |
| "mean_token_accuracy": 0.7017572969198227, |
| "num_tokens": 26910902.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.061662624776363376, |
| "rewards/margins": 1.0938476547598839, |
| "rewards/rejected": -1.0321850061416626, |
| "step": 1960 |
| }, |
| { |
| "entropy": 0.9611604005098343, |
| "epoch": 0.5072743659070427, |
| "grad_norm": 57.55832290649414, |
| "learning_rate": 2.565668161966575e-07, |
| "logits/chosen": 4.354855323553224, |
| "logits/rejected": 4.303488830095201, |
| "logps/chosen": -384.1266235351562, |
| "logps/rejected": -349.87569961547854, |
| "loss": 0.610211992263794, |
| "mean_token_accuracy": 0.7133847624063492, |
| "num_tokens": 27067130.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.10608787275850773, |
| "rewards/margins": 0.7891268633306027, |
| "rewards/rejected": -0.6830389894545078, |
| "step": 1970 |
| }, |
| { |
| "entropy": 0.9564415842294693, |
| "epoch": 0.5098493626882966, |
| "grad_norm": 45.074100494384766, |
| "learning_rate": 2.5448238802944384e-07, |
| "logits/chosen": 4.411397196718537, |
| "logits/rejected": 4.597745429208741, |
| "logps/chosen": -306.9679183959961, |
| "logps/rejected": -336.1702850341797, |
| "loss": 0.5392940998077392, |
| "mean_token_accuracy": 0.7255575984716416, |
| "num_tokens": 27204645.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.14039542600512506, |
| "rewards/margins": 0.9036742106080056, |
| "rewards/rejected": -0.7632787860929966, |
| "step": 1980 |
| }, |
| { |
| "entropy": 0.9326091140508652, |
| "epoch": 0.5124243594695507, |
| "grad_norm": 46.505699157714844, |
| "learning_rate": 2.523976481057874e-07, |
| "logits/chosen": 4.324125159695785, |
| "logits/rejected": 4.345816921216889, |
| "logps/chosen": -316.32717361450193, |
| "logps/rejected": -306.45115127563474, |
| "loss": 0.5290246486663819, |
| "mean_token_accuracy": 0.708903294801712, |
| "num_tokens": 27345750.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.1300957791507244, |
| "rewards/margins": 0.841148529201746, |
| "rewards/rejected": -0.7110527500510215, |
| "step": 1990 |
| }, |
| { |
| "entropy": 1.0109939306974411, |
| "epoch": 0.5149993562508047, |
| "grad_norm": 44.72649002075195, |
| "learning_rate": 2.5031274142230576e-07, |
| "logits/chosen": 4.510163501684517, |
| "logits/rejected": 4.325580930649053, |
| "logps/chosen": -332.6693832397461, |
| "logps/rejected": -321.6415710449219, |
| "loss": 0.5182650566101075, |
| "mean_token_accuracy": 0.7005341708660126, |
| "num_tokens": 27480837.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.2247800800949335, |
| "rewards/margins": 0.8958122998476028, |
| "rewards/rejected": -0.6710322130471468, |
| "step": 2000 |
| }, |
| { |
| "entropy": 0.9889083057641983, |
| "epoch": 0.5175743530320587, |
| "grad_norm": 58.55170822143555, |
| "learning_rate": 2.4822781298721473e-07, |
| "logits/chosen": 4.579308480886338, |
| "logits/rejected": 4.72775180226153, |
| "logps/chosen": -332.1877388000488, |
| "logps/rejected": -307.91585540771484, |
| "loss": 0.5243938446044922, |
| "mean_token_accuracy": 0.704783484339714, |
| "num_tokens": 27619535.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.07122649792581796, |
| "rewards/margins": 0.8846869133412838, |
| "rewards/rejected": -0.8134604260325432, |
| "step": 2010 |
| }, |
| { |
| "entropy": 0.9544982284307479, |
| "epoch": 0.5201493498133127, |
| "grad_norm": 91.9818344116211, |
| "learning_rate": 2.4614300781024314e-07, |
| "logits/chosen": 4.510144881846659, |
| "logits/rejected": 4.53069605625107, |
| "logps/chosen": -339.508846282959, |
| "logps/rejected": -335.85532531738284, |
| "loss": 0.5688719749450684, |
| "mean_token_accuracy": 0.7240165054798127, |
| "num_tokens": 27762485.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": -0.06838485449552537, |
| "rewards/margins": 1.0121475353837013, |
| "rewards/rejected": -1.0805324025452137, |
| "step": 2020 |
| }, |
| { |
| "entropy": 0.983922865986824, |
| "epoch": 0.5227243465945668, |
| "grad_norm": 57.940250396728516, |
| "learning_rate": 2.440584708925469e-07, |
| "logits/chosen": 4.572384071478625, |
| "logits/rejected": 4.769022806281224, |
| "logps/chosen": -337.8588836669922, |
| "logps/rejected": -308.79794082641604, |
| "loss": 0.47387022972106935, |
| "mean_token_accuracy": 0.7007263749837875, |
| "num_tokens": 27904937.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.19583048969507216, |
| "rewards/margins": 0.9539054423570633, |
| "rewards/rejected": -0.7580749623477459, |
| "step": 2030 |
| }, |
| { |
| "entropy": 0.9752614676952363, |
| "epoch": 0.5252993433758207, |
| "grad_norm": 51.395687103271484, |
| "learning_rate": 2.4197434721662453e-07, |
| "logits/chosen": 4.555002191529634, |
| "logits/rejected": 4.620000321200531, |
| "logps/chosen": -284.87156143188474, |
| "logps/rejected": -286.2675308227539, |
| "loss": 0.50951247215271, |
| "mean_token_accuracy": 0.7125829070806503, |
| "num_tokens": 28030489.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.20675006583333017, |
| "rewards/margins": 0.9260029315948486, |
| "rewards/rejected": -0.719252859801054, |
| "step": 2040 |
| }, |
| { |
| "entropy": 1.0117036342620849, |
| "epoch": 0.5278743401570748, |
| "grad_norm": 60.130470275878906, |
| "learning_rate": 2.3989078173623224e-07, |
| "logits/chosen": 4.333817315595856, |
| "logits/rejected": 4.47111231732413, |
| "logps/chosen": -344.92832641601564, |
| "logps/rejected": -313.57475128173826, |
| "loss": 0.5121257305145264, |
| "mean_token_accuracy": 0.7067907392978668, |
| "num_tokens": 28166707.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.12466732561588287, |
| "rewards/margins": 0.9099546924233437, |
| "rewards/rejected": -0.7852873660624027, |
| "step": 2050 |
| }, |
| { |
| "entropy": 0.982884556055069, |
| "epoch": 0.5304493369383289, |
| "grad_norm": 51.04621124267578, |
| "learning_rate": 2.3780791936630366e-07, |
| "logits/chosen": 4.4138088975037535, |
| "logits/rejected": 4.569439114318762, |
| "logps/chosen": -301.08812103271487, |
| "logps/rejected": -289.3797378540039, |
| "loss": 0.5101655960083008, |
| "mean_token_accuracy": 0.718460476398468, |
| "num_tokens": 28301909.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.07634416520595551, |
| "rewards/margins": 1.081505411863327, |
| "rewards/rejected": -1.0051612228155136, |
| "step": 2060 |
| }, |
| { |
| "entropy": 0.9662475287914276, |
| "epoch": 0.5330243337195828, |
| "grad_norm": 58.388267517089844, |
| "learning_rate": 2.3572590497286988e-07, |
| "logits/chosen": 4.402496806036643, |
| "logits/rejected": 4.572015991746902, |
| "logps/chosen": -306.4839553833008, |
| "logps/rejected": -327.9119476318359, |
| "loss": 0.5104366302490234, |
| "mean_token_accuracy": 0.7234220892190933, |
| "num_tokens": 28441535.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": -0.0008957050740718842, |
| "rewards/margins": 0.96529905423522, |
| "rewards/rejected": -0.9661947466433048, |
| "step": 2070 |
| }, |
| { |
| "entropy": 0.9819912523031235, |
| "epoch": 0.5355993305008369, |
| "grad_norm": 52.43979263305664, |
| "learning_rate": 2.3364488336298363e-07, |
| "logits/chosen": 4.221476865602507, |
| "logits/rejected": 4.190381915320627, |
| "logps/chosen": -368.37193832397463, |
| "logps/rejected": -325.319921875, |
| "loss": 0.5923614025115966, |
| "mean_token_accuracy": 0.6977939248085022, |
| "num_tokens": 28586273.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.05655666179955006, |
| "rewards/margins": 0.7294862203299999, |
| "rewards/rejected": -0.6729295663535595, |
| "step": 2080 |
| }, |
| { |
| "entropy": 0.9639265269041062, |
| "epoch": 0.5381743272820909, |
| "grad_norm": 45.972591400146484, |
| "learning_rate": 2.315649992746488e-07, |
| "logits/chosen": 4.343052913177901, |
| "logits/rejected": 4.35323733514684, |
| "logps/chosen": -316.8616611480713, |
| "logps/rejected": -304.16934967041016, |
| "loss": 0.5641426563262939, |
| "mean_token_accuracy": 0.7136311322450638, |
| "num_tokens": 28724776.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.0941460222005844, |
| "rewards/margins": 0.7165497049689293, |
| "rewards/rejected": -0.6224036753177643, |
| "step": 2090 |
| }, |
| { |
| "entropy": 0.9250091910362244, |
| "epoch": 0.540749324063345, |
| "grad_norm": 84.40534210205078, |
| "learning_rate": 2.294863973667525e-07, |
| "logits/chosen": 4.494238621336267, |
| "logits/rejected": 4.506737881799265, |
| "logps/chosen": -323.89574279785154, |
| "logps/rejected": -318.222599029541, |
| "loss": 0.5222518920898438, |
| "mean_token_accuracy": 0.7188939899206161, |
| "num_tokens": 28870500.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.10355070717632771, |
| "rewards/margins": 0.9211230382323266, |
| "rewards/rejected": -0.8175723269581795, |
| "step": 2100 |
| }, |
| { |
| "entropy": 0.9546181976795196, |
| "epoch": 0.5433243208445989, |
| "grad_norm": 75.78016662597656, |
| "learning_rate": 2.2740922220900462e-07, |
| "logits/chosen": 4.565091308177751, |
| "logits/rejected": 4.517123364012567, |
| "logps/chosen": -345.64021530151365, |
| "logps/rejected": -328.6051383972168, |
| "loss": 0.5602751731872558, |
| "mean_token_accuracy": 0.7249366581439972, |
| "num_tokens": 29007529.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.18785378597676755, |
| "rewards/margins": 0.9705551497638225, |
| "rewards/rejected": -0.782701363414526, |
| "step": 2110 |
| }, |
| { |
| "entropy": 0.9966236472129821, |
| "epoch": 0.545899317625853, |
| "grad_norm": 54.55280303955078, |
| "learning_rate": 2.2533361827188264e-07, |
| "logits/chosen": 4.558325017683928, |
| "logits/rejected": 4.712489848699003, |
| "logps/chosen": -333.2004821777344, |
| "logps/rejected": -316.39307327270507, |
| "loss": 0.49320497512817385, |
| "mean_token_accuracy": 0.7029337018728257, |
| "num_tokens": 29149979.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.05198279172182083, |
| "rewards/margins": 1.0450399309396743, |
| "rewards/rejected": -0.9930571526288986, |
| "step": 2120 |
| }, |
| { |
| "entropy": 0.9686165571212768, |
| "epoch": 0.548474314407107, |
| "grad_norm": 78.59687042236328, |
| "learning_rate": 2.2325972991658306e-07, |
| "logits/chosen": 4.431529517037036, |
| "logits/rejected": 4.4224376064404805, |
| "logps/chosen": -363.5590950012207, |
| "logps/rejected": -340.79425888061525, |
| "loss": 0.4562540054321289, |
| "mean_token_accuracy": 0.7150961607694626, |
| "num_tokens": 29302390.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": -0.03903376907110214, |
| "rewards/margins": 1.1957050636410713, |
| "rewards/rejected": -1.2347388774156571, |
| "step": 2130 |
| }, |
| { |
| "entropy": 0.9467013031244278, |
| "epoch": 0.551049311188361, |
| "grad_norm": 37.64112091064453, |
| "learning_rate": 2.2118770138498167e-07, |
| "logits/chosen": 4.643776640567135, |
| "logits/rejected": 4.611367951965497, |
| "logps/chosen": -309.24220275878906, |
| "logps/rejected": -328.61732330322263, |
| "loss": 0.5229010105133056, |
| "mean_token_accuracy": 0.7172821938991547, |
| "num_tokens": 29436778.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": -0.11375233680009841, |
| "rewards/margins": 1.0564178586006165, |
| "rewards/rejected": -1.1701701939105988, |
| "step": 2140 |
| }, |
| { |
| "entropy": 0.940066397190094, |
| "epoch": 0.553624307969615, |
| "grad_norm": 73.62627410888672, |
| "learning_rate": 2.1911767678960063e-07, |
| "logits/chosen": 4.652108957909488, |
| "logits/rejected": 4.715885326711968, |
| "logps/chosen": -331.7106544494629, |
| "logps/rejected": -311.7936897277832, |
| "loss": 0.48636856079101565, |
| "mean_token_accuracy": 0.7166824817657471, |
| "num_tokens": 29579500.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.12549700997769833, |
| "rewards/margins": 1.1011155344545842, |
| "rewards/rejected": -0.9756185300648212, |
| "step": 2150 |
| }, |
| { |
| "entropy": 0.9694902867078781, |
| "epoch": 0.5561993047508691, |
| "grad_norm": 61.72739028930664, |
| "learning_rate": 2.1704980010358542e-07, |
| "logits/chosen": 4.602444838255726, |
| "logits/rejected": 4.799831230044429, |
| "logps/chosen": -331.89060668945314, |
| "logps/rejected": -331.9415641784668, |
| "loss": 0.5689295768737793, |
| "mean_token_accuracy": 0.7171097755432129, |
| "num_tokens": 29708042.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -0.16421702206134797, |
| "rewards/margins": 0.9721270004287362, |
| "rewards/rejected": -1.1363440088927745, |
| "step": 2160 |
| }, |
| { |
| "entropy": 0.9587494820356369, |
| "epoch": 0.558774301532123, |
| "grad_norm": 72.0654525756836, |
| "learning_rate": 2.1498421515069171e-07, |
| "logits/chosen": 4.567917515442895, |
| "logits/rejected": 4.510748764102793, |
| "logps/chosen": -348.9438766479492, |
| "logps/rejected": -326.5355796813965, |
| "loss": 0.5357170104980469, |
| "mean_token_accuracy": 0.7194243431091308, |
| "num_tokens": 29853333.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.10092468392103911, |
| "rewards/margins": 0.9197949707508087, |
| "rewards/rejected": -0.8188702866435051, |
| "step": 2170 |
| }, |
| { |
| "entropy": 1.0412338376045227, |
| "epoch": 0.5613492983133771, |
| "grad_norm": 74.85493469238281, |
| "learning_rate": 2.1292106559528147e-07, |
| "logits/chosen": 4.408832687099317, |
| "logits/rejected": 4.534272302491532, |
| "logps/chosen": -300.82046737670896, |
| "logps/rejected": -297.51931228637693, |
| "loss": 0.4682475566864014, |
| "mean_token_accuracy": 0.6946048319339753, |
| "num_tokens": 29988640.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.2746724203228951, |
| "rewards/margins": 1.281447449326515, |
| "rewards/rejected": -1.0067750446498394, |
| "step": 2180 |
| }, |
| { |
| "entropy": 0.9617623418569565, |
| "epoch": 0.5639242950946312, |
| "grad_norm": 59.6571159362793, |
| "learning_rate": 2.108604949323316e-07, |
| "logits/chosen": 4.478970340786703, |
| "logits/rejected": 4.599134912622948, |
| "logps/chosen": -311.82995834350584, |
| "logps/rejected": -325.49252166748045, |
| "loss": 0.47739534378051757, |
| "mean_token_accuracy": 0.7158630758523941, |
| "num_tokens": 30124183.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.1551291227340698, |
| "rewards/margins": 1.0965686649084092, |
| "rewards/rejected": -0.9414395779371262, |
| "step": 2190 |
| }, |
| { |
| "entropy": 0.9513105571269989, |
| "epoch": 0.5664992918758851, |
| "grad_norm": 60.03646469116211, |
| "learning_rate": 2.088026464774534e-07, |
| "logits/chosen": 4.214004904189293, |
| "logits/rejected": 4.465290087756984, |
| "logps/chosen": -327.4006637573242, |
| "logps/rejected": -319.3096549987793, |
| "loss": 0.5218130111694336, |
| "mean_token_accuracy": 0.7127509474754333, |
| "num_tokens": 30271400.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.17084717899560928, |
| "rewards/margins": 1.027149811387062, |
| "rewards/rejected": -1.1979969710111618, |
| "step": 2200 |
| }, |
| { |
| "entropy": 0.9537039786577225, |
| "epoch": 0.5690742886571392, |
| "grad_norm": 44.66596603393555, |
| "learning_rate": 2.0674766335692445e-07, |
| "logits/chosen": 4.303240877933076, |
| "logits/rejected": 4.435057485614915, |
| "logps/chosen": -321.6051383972168, |
| "logps/rejected": -317.53667602539065, |
| "loss": 0.5237411499023438, |
| "mean_token_accuracy": 0.7187791794538498, |
| "num_tokens": 30408848.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.05160765647888184, |
| "rewards/margins": 0.9479971334338189, |
| "rewards/rejected": -0.9996047995984554, |
| "step": 2210 |
| }, |
| { |
| "entropy": 0.9782517611980438, |
| "epoch": 0.5716492854383932, |
| "grad_norm": 39.292198181152344, |
| "learning_rate": 2.0469568849773453e-07, |
| "logits/chosen": 4.094074952160854, |
| "logits/rejected": 4.335160989134377, |
| "logps/chosen": -288.1838668823242, |
| "logps/rejected": -290.0323829650879, |
| "loss": 0.4903146743774414, |
| "mean_token_accuracy": 0.7082525342702866, |
| "num_tokens": 30540736.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.17911316305398942, |
| "rewards/margins": 1.014538937807083, |
| "rewards/rejected": -0.835425791144371, |
| "step": 2220 |
| }, |
| { |
| "entropy": 0.9824255883693696, |
| "epoch": 0.5742242822196473, |
| "grad_norm": 39.250972747802734, |
| "learning_rate": 2.026468646176446e-07, |
| "logits/chosen": 4.0822044548447, |
| "logits/rejected": 4.332966515519744, |
| "logps/chosen": -305.91218147277834, |
| "logps/rejected": -340.64088401794436, |
| "loss": 0.45061264038085935, |
| "mean_token_accuracy": 0.7125394314527511, |
| "num_tokens": 30682984.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.1678448911756277, |
| "rewards/margins": 1.1703709587454796, |
| "rewards/rejected": -1.0025260582566262, |
| "step": 2230 |
| }, |
| { |
| "entropy": 0.9766992688179016, |
| "epoch": 0.5767992790009012, |
| "grad_norm": 52.70896530151367, |
| "learning_rate": 2.0060133421526027e-07, |
| "logits/chosen": 4.443482124148224, |
| "logits/rejected": 4.4387098977954285, |
| "logps/chosen": -329.52354125976564, |
| "logps/rejected": -299.1279350280762, |
| "loss": 0.5766659259796143, |
| "mean_token_accuracy": 0.7177125632762908, |
| "num_tokens": 30819840.0, |
| "rewards/accuracies": 0.6625, |
| "rewards/chosen": -0.01829192042350769, |
| "rewards/margins": 0.9055951349437237, |
| "rewards/rejected": -0.9238870710134506, |
| "step": 2240 |
| }, |
| { |
| "entropy": 0.9193742334842682, |
| "epoch": 0.5793742757821553, |
| "grad_norm": 63.07462692260742, |
| "learning_rate": 1.985592395601215e-07, |
| "logits/chosen": 4.402515272289742, |
| "logits/rejected": 4.686042422271998, |
| "logps/chosen": -321.8515800476074, |
| "logps/rejected": -332.01871795654296, |
| "loss": 0.5270612239837646, |
| "mean_token_accuracy": 0.7229076892137527, |
| "num_tokens": 30962735.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.09906055107712745, |
| "rewards/margins": 0.9753070950508118, |
| "rewards/rejected": -0.8762465573847293, |
| "step": 2250 |
| }, |
| { |
| "entropy": 1.0063863813877105, |
| "epoch": 0.5819492725634093, |
| "grad_norm": 55.75368118286133, |
| "learning_rate": 1.9652072268280675e-07, |
| "logits/chosen": 4.4105326671434835, |
| "logits/rejected": 4.474947993035541, |
| "logps/chosen": -361.05382080078124, |
| "logps/rejected": -325.942626953125, |
| "loss": 0.548794174194336, |
| "mean_token_accuracy": 0.69999920129776, |
| "num_tokens": 31104260.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.08069399669766426, |
| "rewards/margins": 0.8124750137329102, |
| "rewards/rejected": -0.8931690063327551, |
| "step": 2260 |
| }, |
| { |
| "entropy": 0.9078447461128235, |
| "epoch": 0.5845242693446633, |
| "grad_norm": 71.37114715576172, |
| "learning_rate": 1.944859253650552e-07, |
| "logits/chosen": 4.3516229551206225, |
| "logits/rejected": 4.715358662549993, |
| "logps/chosen": -310.53897018432616, |
| "logps/rejected": -311.94091567993166, |
| "loss": 0.47094364166259767, |
| "mean_token_accuracy": 0.7269211143255234, |
| "num_tokens": 31250584.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": -0.03624087125062943, |
| "rewards/margins": 1.1930074781179427, |
| "rewards/rejected": -1.2292483642697334, |
| "step": 2270 |
| }, |
| { |
| "entropy": 0.952119767665863, |
| "epoch": 0.5870992661259173, |
| "grad_norm": 65.18846130371094, |
| "learning_rate": 1.9245498912990552e-07, |
| "logits/chosen": 4.3513078256927535, |
| "logits/rejected": 4.516444060921208, |
| "logps/chosen": -320.1900581359863, |
| "logps/rejected": -319.03382568359376, |
| "loss": 0.4853237152099609, |
| "mean_token_accuracy": 0.7136391550302505, |
| "num_tokens": 31391197.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": -0.00556834451854229, |
| "rewards/margins": 1.0650015987455845, |
| "rewards/rejected": -1.0705699451267718, |
| "step": 2280 |
| }, |
| { |
| "entropy": 1.0051916390657425, |
| "epoch": 0.5896742629071714, |
| "grad_norm": 60.125362396240234, |
| "learning_rate": 1.9042805523185224e-07, |
| "logits/chosen": 4.136806267133233, |
| "logits/rejected": 4.373445331875144, |
| "logps/chosen": -347.1316879272461, |
| "logps/rejected": -330.32265548706056, |
| "loss": 0.5287572383880615, |
| "mean_token_accuracy": 0.699402442574501, |
| "num_tokens": 31525878.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.14854914993047713, |
| "rewards/margins": 0.9958145916461945, |
| "rewards/rejected": -0.8472654297947884, |
| "step": 2290 |
| }, |
| { |
| "entropy": 0.9312989771366119, |
| "epoch": 0.5922492596884253, |
| "grad_norm": 47.99911880493164, |
| "learning_rate": 1.8840526464702187e-07, |
| "logits/chosen": 4.427626339654708, |
| "logits/rejected": 4.65078433694862, |
| "logps/chosen": -323.1201881408691, |
| "logps/rejected": -335.28249282836913, |
| "loss": 0.48755817413330077, |
| "mean_token_accuracy": 0.7307901710271836, |
| "num_tokens": 31669884.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.07594985216856003, |
| "rewards/margins": 1.0695114340633154, |
| "rewards/rejected": -0.9935616031289101, |
| "step": 2300 |
| }, |
| { |
| "entropy": 1.0104268401861192, |
| "epoch": 0.5948242564696794, |
| "grad_norm": 56.197410583496094, |
| "learning_rate": 1.8638675806336776e-07, |
| "logits/chosen": 4.1933119218725965, |
| "logits/rejected": 4.310008726056373, |
| "logps/chosen": -382.15055084228516, |
| "logps/rejected": -380.93878707885744, |
| "loss": 0.4801497936248779, |
| "mean_token_accuracy": 0.7114536106586457, |
| "num_tokens": 31812649.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.029319702088832854, |
| "rewards/margins": 1.0714796632528305, |
| "rewards/rejected": -1.042159953713417, |
| "step": 2310 |
| }, |
| { |
| "entropy": 1.0495285987854004, |
| "epoch": 0.5973992532509335, |
| "grad_norm": 55.15418243408203, |
| "learning_rate": 1.8437267587088457e-07, |
| "logits/chosen": 4.433101498582317, |
| "logits/rejected": 4.382495202363422, |
| "logps/chosen": -335.66616287231443, |
| "logps/rejected": -305.47173233032225, |
| "loss": 0.5913334369659424, |
| "mean_token_accuracy": 0.6869961947202683, |
| "num_tokens": 31955299.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": -0.1284914530813694, |
| "rewards/margins": 0.8338379189372063, |
| "rewards/rejected": -0.9623293563723564, |
| "step": 2320 |
| }, |
| { |
| "entropy": 0.9694363415241242, |
| "epoch": 0.5999742500321874, |
| "grad_norm": 74.43938446044922, |
| "learning_rate": 1.823631581518446e-07, |
| "logits/chosen": 4.342257492087476, |
| "logits/rejected": 4.333678647145307, |
| "logps/chosen": -306.73903198242186, |
| "logps/rejected": -328.825910949707, |
| "loss": 0.5228142738342285, |
| "mean_token_accuracy": 0.703788360953331, |
| "num_tokens": 32088877.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.07906525041908026, |
| "rewards/margins": 1.1507192701101303, |
| "rewards/rejected": -1.0716539956629276, |
| "step": 2330 |
| }, |
| { |
| "entropy": 0.9773776948451995, |
| "epoch": 0.6025492468134415, |
| "grad_norm": 62.20151138305664, |
| "learning_rate": 1.8035834467105442e-07, |
| "logits/chosen": 4.194597717165923, |
| "logits/rejected": 4.441323402074882, |
| "logps/chosen": -319.6354751586914, |
| "logps/rejected": -337.572087097168, |
| "loss": 0.5659647941589355, |
| "mean_token_accuracy": 0.7077054262161255, |
| "num_tokens": 32228732.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": -0.014954057708382606, |
| "rewards/margins": 0.9510046035051346, |
| "rewards/rejected": -0.9659586466848851, |
| "step": 2340 |
| }, |
| { |
| "entropy": 1.0027844697237014, |
| "epoch": 0.6051242435946955, |
| "grad_norm": 41.43119430541992, |
| "learning_rate": 1.78358374866134e-07, |
| "logits/chosen": 4.454848762213653, |
| "logits/rejected": 4.433416337835228, |
| "logps/chosen": -327.75243911743166, |
| "logps/rejected": -332.934854888916, |
| "loss": 0.5612287998199463, |
| "mean_token_accuracy": 0.7106603384017944, |
| "num_tokens": 32374141.0, |
| "rewards/accuracies": 0.6625, |
| "rewards/chosen": 0.16538103260099887, |
| "rewards/margins": 0.9340221837162972, |
| "rewards/rejected": -0.768641147762537, |
| "step": 2350 |
| }, |
| { |
| "entropy": 0.9379937499761581, |
| "epoch": 0.6076992403759496, |
| "grad_norm": 94.86339569091797, |
| "learning_rate": 1.763633878378191e-07, |
| "logits/chosen": 4.312113700941093, |
| "logits/rejected": 4.511808621681159, |
| "logps/chosen": -318.45185470581055, |
| "logps/rejected": -320.56103057861327, |
| "loss": 0.5496462821960449, |
| "mean_token_accuracy": 0.7269904494285584, |
| "num_tokens": 32504564.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.05075220391154289, |
| "rewards/margins": 0.9266988523304462, |
| "rewards/rejected": -0.8759466748684644, |
| "step": 2360 |
| }, |
| { |
| "entropy": 0.9562380164861679, |
| "epoch": 0.6102742371572035, |
| "grad_norm": 39.19052505493164, |
| "learning_rate": 1.743735223402861e-07, |
| "logits/chosen": 4.421962083003923, |
| "logits/rejected": 4.475477171427551, |
| "logps/chosen": -309.61843185424806, |
| "logps/rejected": -303.66740493774415, |
| "loss": 0.5537517547607422, |
| "mean_token_accuracy": 0.7129708468914032, |
| "num_tokens": 32647109.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.13659467920660973, |
| "rewards/margins": 0.8689602486789226, |
| "rewards/rejected": -0.7323655590415001, |
| "step": 2370 |
| }, |
| { |
| "entropy": 0.9923003166913986, |
| "epoch": 0.6128492339384576, |
| "grad_norm": 60.206581115722656, |
| "learning_rate": 1.7238891677150157e-07, |
| "logits/chosen": 4.403583009165136, |
| "logits/rejected": 4.752100390667911, |
| "logps/chosen": -316.19840774536135, |
| "logps/rejected": -323.16968765258787, |
| "loss": 0.48160157203674314, |
| "mean_token_accuracy": 0.7170157432556152, |
| "num_tokens": 32772750.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.09984740689396858, |
| "rewards/margins": 1.0032228454947472, |
| "rewards/rejected": -0.9033754229545593, |
| "step": 2380 |
| }, |
| { |
| "entropy": 0.9690987765789032, |
| "epoch": 0.6154242307197116, |
| "grad_norm": 54.384742736816406, |
| "learning_rate": 1.7040970916359693e-07, |
| "logits/chosen": 4.389778037955121, |
| "logits/rejected": 4.460626551878917, |
| "logps/chosen": -298.3137832641602, |
| "logps/rejected": -292.2226303100586, |
| "loss": 0.48703317642211913, |
| "mean_token_accuracy": 0.703352552652359, |
| "num_tokens": 32896953.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.1597156472504139, |
| "rewards/margins": 1.1024471048265696, |
| "rewards/rejected": -0.9427314698696136, |
| "step": 2390 |
| }, |
| { |
| "entropy": 0.9835656076669693, |
| "epoch": 0.6179992275009656, |
| "grad_norm": 85.71637725830078, |
| "learning_rate": 1.6843603717326733e-07, |
| "logits/chosen": 4.281219507832217, |
| "logits/rejected": 4.6274769425241455, |
| "logps/chosen": -329.02880783081054, |
| "logps/rejected": -329.16113433837893, |
| "loss": 0.5334079265594482, |
| "mean_token_accuracy": 0.7152191966772079, |
| "num_tokens": 33024356.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.21461917832493782, |
| "rewards/margins": 1.0890141651034355, |
| "rewards/rejected": -0.8743950046598912, |
| "step": 2400 |
| }, |
| { |
| "entropy": 0.9442358911037445, |
| "epoch": 0.6205742242822196, |
| "grad_norm": 63.576297760009766, |
| "learning_rate": 1.664680380721981e-07, |
| "logits/chosen": 4.415484036032346, |
| "logits/rejected": 4.553295847904644, |
| "logps/chosen": -344.47364807128906, |
| "logps/rejected": -302.069393157959, |
| "loss": 0.5017064094543457, |
| "mean_token_accuracy": 0.7172648161649704, |
| "num_tokens": 33171454.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.01607281230390072, |
| "rewards/margins": 1.0055124536156654, |
| "rewards/rejected": -0.9894396156072617, |
| "step": 2410 |
| }, |
| { |
| "entropy": 1.0014096409082414, |
| "epoch": 0.6231492210634737, |
| "grad_norm": 53.30178451538086, |
| "learning_rate": 1.6450584873751723e-07, |
| "logits/chosen": 4.5171440294255785, |
| "logits/rejected": 4.685752037350361, |
| "logps/chosen": -329.97341079711913, |
| "logps/rejected": -309.59109725952146, |
| "loss": 0.6140494346618652, |
| "mean_token_accuracy": 0.701422393321991, |
| "num_tokens": 33309587.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.17000230513513087, |
| "rewards/margins": 0.6981745153665543, |
| "rewards/rejected": -0.5281722396612167, |
| "step": 2420 |
| }, |
| { |
| "entropy": 0.9768936216831208, |
| "epoch": 0.6257242178447276, |
| "grad_norm": 69.7081069946289, |
| "learning_rate": 1.6254960564227484e-07, |
| "logits/chosen": 4.329045615259164, |
| "logits/rejected": 4.660475225543115, |
| "logps/chosen": -324.4511905670166, |
| "logps/rejected": -323.71324462890624, |
| "loss": 0.49992985725402833, |
| "mean_token_accuracy": 0.7213058680295944, |
| "num_tokens": 33445123.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.13972454741597176, |
| "rewards/margins": 1.0301117934286594, |
| "rewards/rejected": -0.8903872383758425, |
| "step": 2430 |
| }, |
| { |
| "entropy": 1.019405362010002, |
| "epoch": 0.6282992146259817, |
| "grad_norm": 54.75837707519531, |
| "learning_rate": 1.6059944484595207e-07, |
| "logits/chosen": 4.291267243289228, |
| "logits/rejected": 4.509141716767159, |
| "logps/chosen": -328.6258911132812, |
| "logps/rejected": -336.042707824707, |
| "loss": 0.5735377788543701, |
| "mean_token_accuracy": 0.701374489068985, |
| "num_tokens": 33579270.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": -0.011339339613914489, |
| "rewards/margins": 0.7998064674437046, |
| "rewards/rejected": -0.8111458018422126, |
| "step": 2440 |
| }, |
| { |
| "entropy": 0.9283707648515701, |
| "epoch": 0.6308742114072358, |
| "grad_norm": 69.55551147460938, |
| "learning_rate": 1.5865550198499715e-07, |
| "logits/chosen": 4.602162313008973, |
| "logits/rejected": 4.594571587360844, |
| "logps/chosen": -310.7761627197266, |
| "logps/rejected": -310.3740631103516, |
| "loss": 0.5215212821960449, |
| "mean_token_accuracy": 0.7308780759572983, |
| "num_tokens": 33717844.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.05332138538360596, |
| "rewards/margins": 0.9332276426255703, |
| "rewards/rejected": -0.8799062594771385, |
| "step": 2450 |
| }, |
| { |
| "entropy": 0.9819339632987976, |
| "epoch": 0.6334492081884897, |
| "grad_norm": 80.49530029296875, |
| "learning_rate": 1.5671791226339232e-07, |
| "logits/chosen": 4.500601053193192, |
| "logits/rejected": 4.426130239662253, |
| "logps/chosen": -326.73095932006834, |
| "logps/rejected": -338.1872108459473, |
| "loss": 0.5379696846008301, |
| "mean_token_accuracy": 0.7148545265197754, |
| "num_tokens": 33852296.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.029502155259251595, |
| "rewards/margins": 0.88140639513731, |
| "rewards/rejected": -0.9109085343778134, |
| "step": 2460 |
| }, |
| { |
| "entropy": 1.000627225637436, |
| "epoch": 0.6360242049697438, |
| "grad_norm": 51.98258972167969, |
| "learning_rate": 1.547868104432501e-07, |
| "logits/chosen": 4.514064345831087, |
| "logits/rejected": 4.54695116008311, |
| "logps/chosen": -359.0768356323242, |
| "logps/rejected": -355.0493965148926, |
| "loss": 0.5708075523376465, |
| "mean_token_accuracy": 0.7088312357664108, |
| "num_tokens": 33995837.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": -0.022996275499463082, |
| "rewards/margins": 0.8100681878626347, |
| "rewards/rejected": -0.8330644458532334, |
| "step": 2470 |
| }, |
| { |
| "entropy": 0.9869578510522843, |
| "epoch": 0.6385992017509978, |
| "grad_norm": 58.026180267333984, |
| "learning_rate": 1.5286233083543997e-07, |
| "logits/chosen": 4.404094705198541, |
| "logits/rejected": 4.741168195342172, |
| "logps/chosen": -318.848078918457, |
| "logps/rejected": -329.31809997558594, |
| "loss": 0.5148727893829346, |
| "mean_token_accuracy": 0.7177086800336838, |
| "num_tokens": 34129212.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.0676919974386692, |
| "rewards/margins": 1.0266894429922104, |
| "rewards/rejected": -0.9589974403381347, |
| "step": 2480 |
| }, |
| { |
| "entropy": 0.9847066342830658, |
| "epoch": 0.6411741985322519, |
| "grad_norm": 73.6479263305664, |
| "learning_rate": 1.5094460729024726e-07, |
| "logits/chosen": 4.557787003296725, |
| "logits/rejected": 4.52986587523226, |
| "logps/chosen": -316.85884857177734, |
| "logps/rejected": -306.20886306762696, |
| "loss": 0.5156795501708984, |
| "mean_token_accuracy": 0.7074246108531952, |
| "num_tokens": 34264275.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": -0.027447796985507013, |
| "rewards/margins": 1.0159383706748486, |
| "rewards/rejected": -1.0433861635625363, |
| "step": 2490 |
| }, |
| { |
| "entropy": 1.0174211353063582, |
| "epoch": 0.6437491953135058, |
| "grad_norm": 53.14724349975586, |
| "learning_rate": 1.4903377318806371e-07, |
| "logits/chosen": 4.508990891910876, |
| "logits/rejected": 4.518605516985489, |
| "logps/chosen": -333.36309204101565, |
| "logps/rejected": -317.0201698303223, |
| "loss": 0.5784246444702148, |
| "mean_token_accuracy": 0.7028531193733215, |
| "num_tokens": 34402752.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.05209047831594944, |
| "rewards/margins": 0.8696140378713608, |
| "rewards/rejected": -0.9217044733464718, |
| "step": 2500 |
| }, |
| { |
| "entropy": 0.9878538995981216, |
| "epoch": 0.6463241920947599, |
| "grad_norm": 56.932682037353516, |
| "learning_rate": 1.4712996143011037e-07, |
| "logits/chosen": 4.578728757916737, |
| "logits/rejected": 4.4917693545516215, |
| "logps/chosen": -340.391495513916, |
| "logps/rejected": -313.9547912597656, |
| "loss": 0.5982888221740723, |
| "mean_token_accuracy": 0.718751209974289, |
| "num_tokens": 34547977.0, |
| "rewards/accuracies": 0.64375, |
| "rewards/chosen": -0.02102447971701622, |
| "rewards/margins": 0.8910175286233425, |
| "rewards/rejected": -0.9120420113205909, |
| "step": 2510 |
| }, |
| { |
| "entropy": 1.0095184803009034, |
| "epoch": 0.6488991888760139, |
| "grad_norm": 94.94068908691406, |
| "learning_rate": 1.4523330442919411e-07, |
| "logits/chosen": 4.567168908759323, |
| "logits/rejected": 4.278807339492092, |
| "logps/chosen": -350.88896255493165, |
| "logps/rejected": -324.29125823974607, |
| "loss": 0.584876298904419, |
| "mean_token_accuracy": 0.6987872898578644, |
| "num_tokens": 34688434.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.01219734326004982, |
| "rewards/margins": 0.7425737954676151, |
| "rewards/rejected": -0.7303764313459397, |
| "step": 2520 |
| }, |
| { |
| "entropy": 0.9569864004850388, |
| "epoch": 0.6514741856572679, |
| "grad_norm": 52.566650390625, |
| "learning_rate": 1.43343934100499e-07, |
| "logits/chosen": 4.308113043827216, |
| "logits/rejected": 4.494314852834433, |
| "logps/chosen": -341.7305908203125, |
| "logps/rejected": -324.85329284667966, |
| "loss": 0.45043315887451174, |
| "mean_token_accuracy": 0.7243666499853134, |
| "num_tokens": 34838666.0, |
| "rewards/accuracies": 0.80625, |
| "rewards/chosen": 0.11741002500057221, |
| "rewards/margins": 1.2841462165117263, |
| "rewards/rejected": -1.1667361974716186, |
| "step": 2530 |
| }, |
| { |
| "entropy": 1.00680111348629, |
| "epoch": 0.654049182438522, |
| "grad_norm": 50.91255569458008, |
| "learning_rate": 1.4146198185240974e-07, |
| "logits/chosen": 4.45827913044039, |
| "logits/rejected": 4.550586306424201, |
| "logps/chosen": -360.7131713867187, |
| "logps/rejected": -294.3333106994629, |
| "loss": 0.5361333370208741, |
| "mean_token_accuracy": 0.7095499604940414, |
| "num_tokens": 34968051.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.20790192447602748, |
| "rewards/margins": 0.9727953381836414, |
| "rewards/rejected": -0.7648934207856655, |
| "step": 2540 |
| }, |
| { |
| "entropy": 0.9529173254966736, |
| "epoch": 0.656624179219776, |
| "grad_norm": 67.677490234375, |
| "learning_rate": 1.3958757857737386e-07, |
| "logits/chosen": 4.4871638388789705, |
| "logits/rejected": 4.6531658406392005, |
| "logps/chosen": -298.5674705505371, |
| "logps/rejected": -290.7885871887207, |
| "loss": 0.5661542415618896, |
| "mean_token_accuracy": 0.7199174910783768, |
| "num_tokens": 35088183.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": -0.008777842298150063, |
| "rewards/margins": 0.8344557493925094, |
| "rewards/rejected": -0.8432335741817951, |
| "step": 2550 |
| }, |
| { |
| "entropy": 0.9984112858772278, |
| "epoch": 0.65919917600103, |
| "grad_norm": 29.420278549194336, |
| "learning_rate": 1.3772085464279675e-07, |
| "logits/chosen": 4.400277528325536, |
| "logits/rejected": 4.468486933165357, |
| "logps/chosen": -317.15233001708987, |
| "logps/rejected": -320.6111999511719, |
| "loss": 0.5059976577758789, |
| "mean_token_accuracy": 0.7072679579257966, |
| "num_tokens": 35218826.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.10976639427244664, |
| "rewards/margins": 1.1474495999515057, |
| "rewards/rejected": -1.0376832112669945, |
| "step": 2560 |
| }, |
| { |
| "entropy": 0.955791312456131, |
| "epoch": 0.661774172782284, |
| "grad_norm": 107.9035873413086, |
| "learning_rate": 1.3586193988197453e-07, |
| "logits/chosen": 4.595885989595397, |
| "logits/rejected": 4.541103714693393, |
| "logps/chosen": -340.237361907959, |
| "logps/rejected": -327.3079864501953, |
| "loss": 0.582268476486206, |
| "mean_token_accuracy": 0.7249088078737259, |
| "num_tokens": 35358921.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": -0.051393065601587296, |
| "rewards/margins": 0.7945620276033878, |
| "rewards/rejected": -0.8459550872445106, |
| "step": 2570 |
| }, |
| { |
| "entropy": 1.0618447273969651, |
| "epoch": 0.6643491695635381, |
| "grad_norm": 54.83210754394531, |
| "learning_rate": 1.3401096358506476e-07, |
| "logits/chosen": 4.331693795635436, |
| "logits/rejected": 4.644192663326451, |
| "logps/chosen": -360.5722297668457, |
| "logps/rejected": -333.0255714416504, |
| "loss": 0.5570868968963623, |
| "mean_token_accuracy": 0.6983685493469238, |
| "num_tokens": 35502346.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.15737950913608073, |
| "rewards/margins": 0.8818966701626778, |
| "rewards/rejected": -1.0392761876806618, |
| "step": 2580 |
| }, |
| { |
| "entropy": 0.9186001747846604, |
| "epoch": 0.6669241663447921, |
| "grad_norm": 50.079200744628906, |
| "learning_rate": 1.3216805449009332e-07, |
| "logits/chosen": 4.687191874734405, |
| "logits/rejected": 4.603947518986728, |
| "logps/chosen": -303.88953018188477, |
| "logps/rejected": -309.2432388305664, |
| "loss": 0.42603626251220705, |
| "mean_token_accuracy": 0.7322681874036789, |
| "num_tokens": 35632285.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.16260901503264905, |
| "rewards/margins": 1.1081829860806465, |
| "rewards/rejected": -0.945573964342475, |
| "step": 2590 |
| }, |
| { |
| "entropy": 0.9982148051261902, |
| "epoch": 0.6694991631260461, |
| "grad_norm": 40.70330810546875, |
| "learning_rate": 1.3033334077400044e-07, |
| "logits/chosen": 4.402947773957061, |
| "logits/rejected": 4.453650887548149, |
| "logps/chosen": -337.1904579162598, |
| "logps/rejected": -315.55087051391604, |
| "loss": 0.511660099029541, |
| "mean_token_accuracy": 0.6992412239313126, |
| "num_tokens": 35762029.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.027360453456640243, |
| "rewards/margins": 1.019322784245014, |
| "rewards/rejected": -0.9919623393565417, |
| "step": 2600 |
| }, |
| { |
| "entropy": 1.0451726645231247, |
| "epoch": 0.6720741599073001, |
| "grad_norm": 63.92889404296875, |
| "learning_rate": 1.2850695004372686e-07, |
| "logits/chosen": 4.40729617097136, |
| "logits/rejected": 4.577327136628008, |
| "logps/chosen": -321.3320426940918, |
| "logps/rejected": -323.710400390625, |
| "loss": 0.4458984375, |
| "mean_token_accuracy": 0.7113217651844025, |
| "num_tokens": 35895643.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.1277332842350006, |
| "rewards/margins": 1.0387729123234748, |
| "rewards/rejected": -0.9110396340489387, |
| "step": 2610 |
| }, |
| { |
| "entropy": 0.9464722245931625, |
| "epoch": 0.6746491566885542, |
| "grad_norm": 73.1920394897461, |
| "learning_rate": 1.2668900932733695e-07, |
| "logits/chosen": 4.404658291124084, |
| "logits/rejected": 4.62908634142962, |
| "logps/chosen": -319.60567169189454, |
| "logps/rejected": -343.53348007202146, |
| "loss": 0.5372816562652588, |
| "mean_token_accuracy": 0.7348382592201232, |
| "num_tokens": 36045193.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.048183007165789604, |
| "rewards/margins": 0.9457500413060188, |
| "rewards/rejected": -0.8975670389831066, |
| "step": 2620 |
| }, |
| { |
| "entropy": 1.0007650017738343, |
| "epoch": 0.6772241534698081, |
| "grad_norm": 51.42596435546875, |
| "learning_rate": 1.248796450651855e-07, |
| "logits/chosen": 4.343561634989639, |
| "logits/rejected": 4.435974349810258, |
| "logps/chosen": -322.3106376647949, |
| "logps/rejected": -290.7184883117676, |
| "loss": 0.5228497982025146, |
| "mean_token_accuracy": 0.7019443780183792, |
| "num_tokens": 36171248.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.00893600434064865, |
| "rewards/margins": 0.8154723040759564, |
| "rewards/rejected": -0.806536290794611, |
| "step": 2630 |
| }, |
| { |
| "entropy": 0.9472282439470291, |
| "epoch": 0.6797991502510622, |
| "grad_norm": 62.84988021850586, |
| "learning_rate": 1.230789831011223e-07, |
| "logits/chosen": 4.436925138350209, |
| "logits/rejected": 4.608209699765224, |
| "logps/chosen": -323.9646141052246, |
| "logps/rejected": -316.40479888916013, |
| "loss": 0.49807300567626955, |
| "mean_token_accuracy": 0.7247744888067246, |
| "num_tokens": 36303477.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.08416349403560161, |
| "rewards/margins": 1.0327687010169029, |
| "rewards/rejected": -0.9486052393913269, |
| "step": 2640 |
| }, |
| { |
| "entropy": 0.9816150844097138, |
| "epoch": 0.6823741470323162, |
| "grad_norm": 64.08195495605469, |
| "learning_rate": 1.2128714867373977e-07, |
| "logits/chosen": 4.419836130212663, |
| "logits/rejected": 4.61189906946818, |
| "logps/chosen": -337.7493865966797, |
| "logps/rejected": -317.8284484863281, |
| "loss": 0.4829238414764404, |
| "mean_token_accuracy": 0.7087979733943939, |
| "num_tokens": 36446619.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.10568625628948211, |
| "rewards/margins": 1.0961877018213273, |
| "rewards/rejected": -0.9905014172196388, |
| "step": 2650 |
| }, |
| { |
| "entropy": 1.009509739279747, |
| "epoch": 0.6849491438135702, |
| "grad_norm": 50.11562728881836, |
| "learning_rate": 1.1950426640766314e-07, |
| "logits/chosen": 4.408743922486776, |
| "logits/rejected": 4.44753181965117, |
| "logps/chosen": -339.89669647216795, |
| "logps/rejected": -310.1831184387207, |
| "loss": 0.4841722011566162, |
| "mean_token_accuracy": 0.7028984099626541, |
| "num_tokens": 36578459.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.11314288526773453, |
| "rewards/margins": 1.1558299280703068, |
| "rewards/rejected": -1.042687040567398, |
| "step": 2660 |
| }, |
| { |
| "entropy": 0.9401783347129822, |
| "epoch": 0.6875241405948243, |
| "grad_norm": 35.34513854980469, |
| "learning_rate": 1.177304603048819e-07, |
| "logits/chosen": 4.57718549025906, |
| "logits/rejected": 4.888864889147892, |
| "logps/chosen": -303.99970321655275, |
| "logps/rejected": -301.6977195739746, |
| "loss": 0.5017678260803222, |
| "mean_token_accuracy": 0.7172846794128418, |
| "num_tokens": 36719334.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": -0.029189274087548255, |
| "rewards/margins": 0.9483950551599264, |
| "rewards/rejected": -0.977584321796894, |
| "step": 2670 |
| }, |
| { |
| "entropy": 1.0551141589879989, |
| "epoch": 0.6900991373760783, |
| "grad_norm": 61.1798095703125, |
| "learning_rate": 1.1596585373612525e-07, |
| "logits/chosen": 4.541497685054158, |
| "logits/rejected": 4.65840884700258, |
| "logps/chosen": -334.9914184570313, |
| "logps/rejected": -346.8951156616211, |
| "loss": 0.4910571098327637, |
| "mean_token_accuracy": 0.692557942867279, |
| "num_tokens": 36849707.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.06038952544331551, |
| "rewards/margins": 1.144432232528925, |
| "rewards/rejected": -1.0840427078306676, |
| "step": 2680 |
| }, |
| { |
| "entropy": 0.9442234963178635, |
| "epoch": 0.6926741341573323, |
| "grad_norm": 47.575958251953125, |
| "learning_rate": 1.1421056943228233e-07, |
| "logits/chosen": 4.404051901571395, |
| "logits/rejected": 4.524723351667393, |
| "logps/chosen": -332.95044174194334, |
| "logps/rejected": -323.7625015258789, |
| "loss": 0.5233668327331543, |
| "mean_token_accuracy": 0.7174067795276642, |
| "num_tokens": 36992329.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.06839351318776607, |
| "rewards/margins": 1.0658936589956283, |
| "rewards/rejected": -0.9975001350045204, |
| "step": 2690 |
| }, |
| { |
| "entropy": 0.9637559443712235, |
| "epoch": 0.6952491309385863, |
| "grad_norm": 55.56652069091797, |
| "learning_rate": 1.1246472947586494e-07, |
| "logits/chosen": 4.281454236228097, |
| "logits/rejected": 4.303310600480479, |
| "logps/chosen": -345.25632553100587, |
| "logps/rejected": -318.4501441955566, |
| "loss": 0.5608476638793946, |
| "mean_token_accuracy": 0.7196205258369446, |
| "num_tokens": 37136230.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": -0.12094516679644585, |
| "rewards/margins": 1.0600071348249913, |
| "rewards/rejected": -1.1809523344039916, |
| "step": 2700 |
| }, |
| { |
| "entropy": 1.0444719851017, |
| "epoch": 0.6978241277198404, |
| "grad_norm": 48.556495666503906, |
| "learning_rate": 1.1072845529251742e-07, |
| "logits/chosen": 4.281288189609699, |
| "logits/rejected": 4.4158216345852, |
| "logps/chosen": -349.4377815246582, |
| "logps/rejected": -323.73373184204104, |
| "loss": 0.48833293914794923, |
| "mean_token_accuracy": 0.6984021216630936, |
| "num_tokens": 37267297.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.1243469189852476, |
| "rewards/margins": 1.0905327267944813, |
| "rewards/rejected": -0.966185811161995, |
| "step": 2710 |
| }, |
| { |
| "entropy": 1.0229749262332917, |
| "epoch": 0.7003991245010944, |
| "grad_norm": 75.61956787109375, |
| "learning_rate": 1.0900186764257127e-07, |
| "logits/chosen": 4.459821607232414, |
| "logits/rejected": 4.460600032873114, |
| "logps/chosen": -347.7802589416504, |
| "logps/rejected": -340.0708801269531, |
| "loss": 0.5790718078613282, |
| "mean_token_accuracy": 0.7006754904985428, |
| "num_tokens": 37406239.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.026463273912668228, |
| "rewards/margins": 0.9200003072619438, |
| "rewards/rejected": -0.9464636027812958, |
| "step": 2720 |
| }, |
| { |
| "entropy": 0.9510831981897354, |
| "epoch": 0.7029741212823484, |
| "grad_norm": 63.93190002441406, |
| "learning_rate": 1.0728508661264521e-07, |
| "logits/chosen": 4.4445433403575905, |
| "logits/rejected": 4.43910397449009, |
| "logps/chosen": -338.7940216064453, |
| "logps/rejected": -314.04123840332034, |
| "loss": 0.5228251457214356, |
| "mean_token_accuracy": 0.717621025443077, |
| "num_tokens": 37531340.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.23341020233929158, |
| "rewards/margins": 0.9731593623757362, |
| "rewards/rejected": -0.739749164134264, |
| "step": 2730 |
| }, |
| { |
| "entropy": 0.9506629645824433, |
| "epoch": 0.7055491180636024, |
| "grad_norm": 55.87628936767578, |
| "learning_rate": 1.0557823160729431e-07, |
| "logits/chosen": 4.33434736424992, |
| "logits/rejected": 4.626212192499031, |
| "logps/chosen": -331.46177368164064, |
| "logps/rejected": -308.30202560424806, |
| "loss": 0.45470657348632815, |
| "mean_token_accuracy": 0.716735503077507, |
| "num_tokens": 37678597.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.21226003915071487, |
| "rewards/margins": 1.2418378800153733, |
| "rewards/rejected": -1.02957783639431, |
| "step": 2740 |
| }, |
| { |
| "entropy": 1.0071748048067093, |
| "epoch": 0.7081241148448565, |
| "grad_norm": 48.8303337097168, |
| "learning_rate": 1.0388142134070424e-07, |
| "logits/chosen": 4.286711720057296, |
| "logits/rejected": 4.364682946553529, |
| "logps/chosen": -330.69200325012207, |
| "logps/rejected": -327.5856483459473, |
| "loss": 0.5594644546508789, |
| "mean_token_accuracy": 0.7034598410129547, |
| "num_tokens": 37812907.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.13423703461885453, |
| "rewards/margins": 0.7663955822587013, |
| "rewards/rejected": -0.6321585506200791, |
| "step": 2750 |
| }, |
| { |
| "entropy": 0.9429391235113144, |
| "epoch": 0.7106991116261104, |
| "grad_norm": 48.388492584228516, |
| "learning_rate": 1.0219477382843464e-07, |
| "logits/chosen": 4.194248919370122, |
| "logits/rejected": 4.5460081216541575, |
| "logps/chosen": -294.94847259521487, |
| "logps/rejected": -297.35206146240233, |
| "loss": 0.5030807018280029, |
| "mean_token_accuracy": 0.7219616591930389, |
| "num_tokens": 37939278.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.15515281893312932, |
| "rewards/margins": 0.9358277872204781, |
| "rewards/rejected": -0.7806749522686005, |
| "step": 2760 |
| }, |
| { |
| "entropy": 0.8929346352815628, |
| "epoch": 0.7132741084073645, |
| "grad_norm": 63.80548858642578, |
| "learning_rate": 1.0051840637921174e-07, |
| "logits/chosen": 4.409810811158247, |
| "logits/rejected": 4.542762150569493, |
| "logps/chosen": -291.65123825073243, |
| "logps/rejected": -307.8862419128418, |
| "loss": 0.5681679725646973, |
| "mean_token_accuracy": 0.7367508620023727, |
| "num_tokens": 38070789.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.05143854729831219, |
| "rewards/margins": 0.8403750531375408, |
| "rewards/rejected": -0.7889365002512931, |
| "step": 2770 |
| }, |
| { |
| "entropy": 0.9484794080257416, |
| "epoch": 0.7158491051886186, |
| "grad_norm": 36.92988204956055, |
| "learning_rate": 9.885243558676822e-08, |
| "logits/chosen": 4.419774984410074, |
| "logits/rejected": 4.647377253694863, |
| "logps/chosen": -306.4446388244629, |
| "logps/rejected": -284.5430198669434, |
| "loss": 0.5263370513916016, |
| "mean_token_accuracy": 0.7101660877466202, |
| "num_tokens": 38210686.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.20834870748221873, |
| "rewards/margins": 0.9040569610893726, |
| "rewards/rejected": -0.6957082599401474, |
| "step": 2780 |
| }, |
| { |
| "entropy": 0.9955869674682617, |
| "epoch": 0.7184241019698725, |
| "grad_norm": 50.16758346557617, |
| "learning_rate": 9.719697732173481e-08, |
| "logits/chosen": 4.425591581073951, |
| "logits/rejected": 4.527296195863723, |
| "logps/chosen": -319.05909194946287, |
| "logps/rejected": -336.0451545715332, |
| "loss": 0.5405101776123047, |
| "mean_token_accuracy": 0.7104849278926849, |
| "num_tokens": 38353204.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.17830976098775864, |
| "rewards/margins": 0.8981378510594368, |
| "rewards/rejected": -0.7198280796408654, |
| "step": 2790 |
| }, |
| { |
| "entropy": 1.0241438031196595, |
| "epoch": 0.7209990987511266, |
| "grad_norm": 47.5545654296875, |
| "learning_rate": 9.555214672358128e-08, |
| "logits/chosen": 4.299143100957586, |
| "logits/rejected": 4.705484058148562, |
| "logps/chosen": -307.576830291748, |
| "logps/rejected": -335.4364730834961, |
| "loss": 0.4629045963287354, |
| "mean_token_accuracy": 0.7091218382120132, |
| "num_tokens": 38484000.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.32216081991791723, |
| "rewards/margins": 1.1764334067702293, |
| "rewards/rejected": -0.8542725712060928, |
| "step": 2800 |
| }, |
| { |
| "entropy": 1.0045750170946122, |
| "epoch": 0.7235740955323806, |
| "grad_norm": 50.36190414428711, |
| "learning_rate": 9.391805819260753e-08, |
| "logits/chosen": 4.461994885639138, |
| "logits/rejected": 4.51211255949246, |
| "logps/chosen": -332.54803771972655, |
| "logps/rejected": -342.58244705200195, |
| "loss": 0.5819994449615479, |
| "mean_token_accuracy": 0.7181883543729782, |
| "num_tokens": 38628258.0, |
| "rewards/accuracies": 0.7125, |
| "rewards/chosen": 0.03722733799368143, |
| "rewards/margins": 0.823909766972065, |
| "rewards/rejected": -0.7866824433207512, |
| "step": 2810 |
| }, |
| { |
| "entropy": 0.9943287044763565, |
| "epoch": 0.7261490923136346, |
| "grad_norm": 50.188690185546875, |
| "learning_rate": 9.229482538198804e-08, |
| "logits/chosen": 4.086052531524615, |
| "logits/rejected": 4.108102212518812, |
| "logps/chosen": -336.1692123413086, |
| "logps/rejected": -332.55058059692385, |
| "loss": 0.5688528537750244, |
| "mean_token_accuracy": 0.7213399589061738, |
| "num_tokens": 38770992.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.0018285416066646577, |
| "rewards/margins": 0.9121986500918865, |
| "rewards/rejected": -0.9103701081126928, |
| "step": 2820 |
| }, |
| { |
| "entropy": 0.9544013321399689, |
| "epoch": 0.7287240890948886, |
| "grad_norm": 53.01747512817383, |
| "learning_rate": 9.068256118986631e-08, |
| "logits/chosen": 4.1799197337194895, |
| "logits/rejected": 4.455103215239529, |
| "logps/chosen": -317.2519187927246, |
| "logps/rejected": -294.7378273010254, |
| "loss": 0.49945316314697263, |
| "mean_token_accuracy": 0.7185042858123779, |
| "num_tokens": 38906315.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.2399644384160638, |
| "rewards/margins": 1.0651536084711553, |
| "rewards/rejected": -0.8251891598105431, |
| "step": 2830 |
| }, |
| { |
| "entropy": 0.9642573595046997, |
| "epoch": 0.7312990858761427, |
| "grad_norm": 92.45647430419922, |
| "learning_rate": 8.908137775150257e-08, |
| "logits/chosen": 4.303308568983775, |
| "logits/rejected": 4.412389834513493, |
| "logps/chosen": -339.0303047180176, |
| "logps/rejected": -317.9662094116211, |
| "loss": 0.5016836166381836, |
| "mean_token_accuracy": 0.7174497812986373, |
| "num_tokens": 39043108.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.1519692622125149, |
| "rewards/margins": 1.0284341365098952, |
| "rewards/rejected": -0.876464857161045, |
| "step": 2840 |
| }, |
| { |
| "entropy": 0.9783081382513046, |
| "epoch": 0.7338740826573967, |
| "grad_norm": 41.27275466918945, |
| "learning_rate": 8.749138643147558e-08, |
| "logits/chosen": 4.697188761436596, |
| "logits/rejected": 4.8487493974469675, |
| "logps/chosen": -305.1290481567383, |
| "logps/rejected": -299.2451332092285, |
| "loss": 0.5368721961975098, |
| "mean_token_accuracy": 0.7132978916168213, |
| "num_tokens": 39177011.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.1731885824352503, |
| "rewards/margins": 0.9777018561959266, |
| "rewards/rejected": -0.8045132834464311, |
| "step": 2850 |
| }, |
| { |
| "entropy": 0.992822477221489, |
| "epoch": 0.7364490794386507, |
| "grad_norm": 55.166175842285156, |
| "learning_rate": 8.591269781593574e-08, |
| "logits/chosen": 4.345998862675562, |
| "logits/rejected": 4.54603522894707, |
| "logps/chosen": -323.51059646606444, |
| "logps/rejected": -318.03050537109374, |
| "loss": 0.48357276916503905, |
| "mean_token_accuracy": 0.719045877456665, |
| "num_tokens": 39313378.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.19722165130078792, |
| "rewards/margins": 1.0652718126773835, |
| "rewards/rejected": -0.8680501654744148, |
| "step": 2860 |
| }, |
| { |
| "entropy": 0.9903046548366546, |
| "epoch": 0.7390240762199047, |
| "grad_norm": 28.31355857849121, |
| "learning_rate": 8.434542170491466e-08, |
| "logits/chosen": 4.410495406036079, |
| "logits/rejected": 4.481941922543696, |
| "logps/chosen": -304.29503479003904, |
| "logps/rejected": -301.4001434326172, |
| "loss": 0.5171587944030762, |
| "mean_token_accuracy": 0.7006939500570297, |
| "num_tokens": 39443882.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.21424660459160805, |
| "rewards/margins": 0.8981669448316097, |
| "rewards/rejected": -0.6839203275740147, |
| "step": 2870 |
| }, |
| { |
| "entropy": 1.0298338741064073, |
| "epoch": 0.7415990730011588, |
| "grad_norm": 67.0123519897461, |
| "learning_rate": 8.278966710468822e-08, |
| "logits/chosen": 4.201646701813575, |
| "logits/rejected": 4.241147471196167, |
| "logps/chosen": -345.51076583862306, |
| "logps/rejected": -325.53188705444336, |
| "loss": 0.5303487777709961, |
| "mean_token_accuracy": 0.7124982863664627, |
| "num_tokens": 39584784.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.08423602096736431, |
| "rewards/margins": 1.0374247565865518, |
| "rewards/rejected": -0.9531887352466584, |
| "step": 2880 |
| }, |
| { |
| "entropy": 0.9180865705013275, |
| "epoch": 0.7441740697824127, |
| "grad_norm": 61.85934066772461, |
| "learning_rate": 8.124554222019448e-08, |
| "logits/chosen": 4.503551553238841, |
| "logits/rejected": 4.840787797658568, |
| "logps/chosen": -289.37534484863284, |
| "logps/rejected": -303.46933517456057, |
| "loss": 0.54837646484375, |
| "mean_token_accuracy": 0.7242263734340668, |
| "num_tokens": 39725942.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.1193401824682951, |
| "rewards/margins": 0.92861771248281, |
| "rewards/rejected": -0.8092775233089924, |
| "step": 2890 |
| }, |
| { |
| "entropy": 1.0399951666593552, |
| "epoch": 0.7467490665636668, |
| "grad_norm": 81.25881958007812, |
| "learning_rate": 7.97131544475085e-08, |
| "logits/chosen": 4.212558677519399, |
| "logits/rejected": 4.5135824060273055, |
| "logps/chosen": -371.32031936645507, |
| "logps/rejected": -374.5592987060547, |
| "loss": 0.5198248863220215, |
| "mean_token_accuracy": 0.7017723619937897, |
| "num_tokens": 39881804.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.06453087776899338, |
| "rewards/margins": 0.9404610253870487, |
| "rewards/rejected": -0.8759301364421844, |
| "step": 2900 |
| }, |
| { |
| "entropy": 0.9375022858381271, |
| "epoch": 0.7493240633449209, |
| "grad_norm": 62.9113883972168, |
| "learning_rate": 7.819261036637284e-08, |
| "logits/chosen": 4.441437066640102, |
| "logits/rejected": 4.26147197062758, |
| "logps/chosen": -330.31504974365237, |
| "logps/rejected": -302.4228385925293, |
| "loss": 0.5019186019897461, |
| "mean_token_accuracy": 0.7240248680114746, |
| "num_tokens": 40022680.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.27599499225616453, |
| "rewards/margins": 1.134216284379363, |
| "rewards/rejected": -0.8582212992012501, |
| "step": 2910 |
| }, |
| { |
| "entropy": 1.0441882610321045, |
| "epoch": 0.7518990601261748, |
| "grad_norm": 54.067020416259766, |
| "learning_rate": 7.668401573278387e-08, |
| "logits/chosen": 4.317062533761897, |
| "logits/rejected": 4.409713854047424, |
| "logps/chosen": -324.02139205932616, |
| "logps/rejected": -311.38745193481446, |
| "loss": 0.47819151878356936, |
| "mean_token_accuracy": 0.6976284444332123, |
| "num_tokens": 40151504.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.3253614973276854, |
| "rewards/margins": 1.0317629069089889, |
| "rewards/rejected": -0.7064014218747616, |
| "step": 2920 |
| }, |
| { |
| "entropy": 0.9620579421520233, |
| "epoch": 0.7544740569074289, |
| "grad_norm": 57.29512023925781, |
| "learning_rate": 7.518747547163773e-08, |
| "logits/chosen": 4.529682488991996, |
| "logits/rejected": 4.549528768541835, |
| "logps/chosen": -341.7662750244141, |
| "logps/rejected": -341.3739807128906, |
| "loss": 0.4937107086181641, |
| "mean_token_accuracy": 0.726236167550087, |
| "num_tokens": 40294340.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.03524754866957665, |
| "rewards/margins": 1.0567084670066833, |
| "rewards/rejected": -1.0214609190821649, |
| "step": 2930 |
| }, |
| { |
| "entropy": 1.0326186656951903, |
| "epoch": 0.7570490536886829, |
| "grad_norm": 70.87602233886719, |
| "learning_rate": 7.370309366943114e-08, |
| "logits/chosen": 4.416342597151165, |
| "logits/rejected": 4.474957801985486, |
| "logps/chosen": -374.77448120117185, |
| "logps/rejected": -348.7484405517578, |
| "loss": 0.45115070343017577, |
| "mean_token_accuracy": 0.7073379307985306, |
| "num_tokens": 40442011.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.2300522394478321, |
| "rewards/margins": 1.0959902562201023, |
| "rewards/rejected": -0.8659380182623864, |
| "step": 2940 |
| }, |
| { |
| "entropy": 0.9630140721797943, |
| "epoch": 0.7596240504699369, |
| "grad_norm": 32.93770980834961, |
| "learning_rate": 7.22309735670231e-08, |
| "logits/chosen": 4.174306564433566, |
| "logits/rejected": 4.293997769817925, |
| "logps/chosen": -317.42327880859375, |
| "logps/rejected": -315.3322067260742, |
| "loss": 0.4436497211456299, |
| "mean_token_accuracy": 0.7245869845151901, |
| "num_tokens": 40576718.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.2645474158227444, |
| "rewards/margins": 1.1843299344182014, |
| "rewards/rejected": -0.9197825267910957, |
| "step": 2950 |
| }, |
| { |
| "entropy": 0.9748166382312775, |
| "epoch": 0.7621990472511909, |
| "grad_norm": 72.24275207519531, |
| "learning_rate": 7.077121755245428e-08, |
| "logits/chosen": 4.217123138479009, |
| "logits/rejected": 4.486150912960931, |
| "logps/chosen": -308.8130683898926, |
| "logps/rejected": -313.8214279174805, |
| "loss": 0.5102333545684814, |
| "mean_token_accuracy": 0.7062664836645126, |
| "num_tokens": 40711819.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": -0.007556497305631638, |
| "rewards/margins": 1.0142739318311214, |
| "rewards/rejected": -1.0218304254114627, |
| "step": 2960 |
| }, |
| { |
| "entropy": 1.053650790452957, |
| "epoch": 0.764774044032445, |
| "grad_norm": 42.517642974853516, |
| "learning_rate": 6.932392715382487e-08, |
| "logits/chosen": 4.506357549619956, |
| "logits/rejected": 4.419823614093227, |
| "logps/chosen": -355.1724250793457, |
| "logps/rejected": -311.9959014892578, |
| "loss": 0.4585103511810303, |
| "mean_token_accuracy": 0.6848949640989304, |
| "num_tokens": 40846130.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.08447664454579354, |
| "rewards/margins": 1.195004227757454, |
| "rewards/rejected": -1.1105275854468346, |
| "step": 2970 |
| }, |
| { |
| "entropy": 0.9970977663993835, |
| "epoch": 0.767349040813699, |
| "grad_norm": 48.33706283569336, |
| "learning_rate": 6.788920303223425e-08, |
| "logits/chosen": 4.363256771998095, |
| "logits/rejected": 4.447415007108594, |
| "logps/chosen": -345.1931518554687, |
| "logps/rejected": -314.6180160522461, |
| "loss": 0.46572027206420896, |
| "mean_token_accuracy": 0.7173113703727723, |
| "num_tokens": 40984083.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.11490394063293934, |
| "rewards/margins": 1.0985809415578842, |
| "rewards/rejected": -0.9836770124733448, |
| "step": 2980 |
| }, |
| { |
| "entropy": 0.9672432780265808, |
| "epoch": 0.769924037594953, |
| "grad_norm": 66.26432800292969, |
| "learning_rate": 6.646714497477948e-08, |
| "logits/chosen": 4.366952443799669, |
| "logits/rejected": 4.514437732158646, |
| "logps/chosen": -353.2722297668457, |
| "logps/rejected": -332.82758178710935, |
| "loss": 0.5087109565734863, |
| "mean_token_accuracy": 0.7158259689807892, |
| "num_tokens": 41129956.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.08387112971395254, |
| "rewards/margins": 0.9910045996308326, |
| "rewards/rejected": -0.9071334734559059, |
| "step": 2990 |
| }, |
| { |
| "entropy": 0.9817124366760254, |
| "epoch": 0.772499034376207, |
| "grad_norm": 91.40129089355469, |
| "learning_rate": 6.505785188761451e-08, |
| "logits/chosen": 4.227975146993117, |
| "logits/rejected": 4.198954829654662, |
| "logps/chosen": -368.0373992919922, |
| "logps/rejected": -342.33693771362306, |
| "loss": 0.5445797920227051, |
| "mean_token_accuracy": 0.7081363558769226, |
| "num_tokens": 41269766.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.050302238762378694, |
| "rewards/margins": 0.8330787874758243, |
| "rewards/rejected": -0.8833810284733772, |
| "step": 3000 |
| }, |
| { |
| "entropy": 1.0685946702957154, |
| "epoch": 0.7750740311574611, |
| "grad_norm": 56.52578353881836, |
| "learning_rate": 6.36614217890718e-08, |
| "logits/chosen": 4.127420726294206, |
| "logits/rejected": 4.333874667104852, |
| "logps/chosen": -348.59493865966795, |
| "logps/rejected": -331.4124214172363, |
| "loss": 0.5051251888275147, |
| "mean_token_accuracy": 0.7025615155696869, |
| "num_tokens": 41399201.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.06800611373037099, |
| "rewards/margins": 1.1953423842787743, |
| "rewards/rejected": -1.1273362621665002, |
| "step": 3010 |
| }, |
| { |
| "entropy": 1.0232778668403626, |
| "epoch": 0.777649027938715, |
| "grad_norm": 39.64238357543945, |
| "learning_rate": 6.227795180284465e-08, |
| "logits/chosen": 4.3056687184544575, |
| "logits/rejected": 4.124631304956351, |
| "logps/chosen": -343.99916610717776, |
| "logps/rejected": -295.5423843383789, |
| "loss": 0.5074767589569091, |
| "mean_token_accuracy": 0.6896465122699738, |
| "num_tokens": 41531613.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.15039641335606574, |
| "rewards/margins": 0.9849472507834435, |
| "rewards/rejected": -0.8345508441329003, |
| "step": 3020 |
| }, |
| { |
| "entropy": 0.9815282195806503, |
| "epoch": 0.7802240247199691, |
| "grad_norm": 52.85367202758789, |
| "learning_rate": 6.090753815123209e-08, |
| "logits/chosen": 4.558400550496125, |
| "logits/rejected": 4.555732151159805, |
| "logps/chosen": -327.8596466064453, |
| "logps/rejected": -314.9741668701172, |
| "loss": 0.46218223571777345, |
| "mean_token_accuracy": 0.716248345375061, |
| "num_tokens": 41666673.0, |
| "rewards/accuracies": 0.8, |
| "rewards/chosen": 0.35434846989810465, |
| "rewards/margins": 1.1373900040984153, |
| "rewards/rejected": -0.7830415509641171, |
| "step": 3030 |
| }, |
| { |
| "entropy": 0.9949847847223282, |
| "epoch": 0.7827990215012232, |
| "grad_norm": 53.299171447753906, |
| "learning_rate": 5.955027614844682e-08, |
| "logits/chosen": 4.394556627696557, |
| "logits/rejected": 4.386567171595721, |
| "logps/chosen": -299.4707862854004, |
| "logps/rejected": -303.38683853149416, |
| "loss": 0.518274974822998, |
| "mean_token_accuracy": 0.7210689902305603, |
| "num_tokens": 41803295.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.1871395196765661, |
| "rewards/margins": 1.0939487650990487, |
| "rewards/rejected": -0.9068092465400696, |
| "step": 3040 |
| }, |
| { |
| "entropy": 0.9936753064393997, |
| "epoch": 0.7853740182824771, |
| "grad_norm": 72.15467071533203, |
| "learning_rate": 5.820626019398534e-08, |
| "logits/chosen": 4.268774096198235, |
| "logits/rejected": 4.6194692725014495, |
| "logps/chosen": -323.48271560668945, |
| "logps/rejected": -352.22514190673826, |
| "loss": 0.4884005546569824, |
| "mean_token_accuracy": 0.7117483109235764, |
| "num_tokens": 41946617.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.21016546208411455, |
| "rewards/margins": 1.0751794874668121, |
| "rewards/rejected": -0.8650140359997749, |
| "step": 3050 |
| }, |
| { |
| "entropy": 1.0037529230117799, |
| "epoch": 0.7879490150637312, |
| "grad_norm": 70.5625, |
| "learning_rate": 5.687558376606291e-08, |
| "logits/chosen": 4.464402455004146, |
| "logits/rejected": 4.337021500618102, |
| "logps/chosen": -375.0096603393555, |
| "logps/rejected": -367.5603225708008, |
| "loss": 0.4842118263244629, |
| "mean_token_accuracy": 0.7070386677980423, |
| "num_tokens": 42094420.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.17200252264738083, |
| "rewards/margins": 1.0302217558026314, |
| "rewards/rejected": -0.8582192309200763, |
| "step": 3060 |
| }, |
| { |
| "entropy": 0.9708365321159362, |
| "epoch": 0.7905240118449852, |
| "grad_norm": 101.7674789428711, |
| "learning_rate": 5.555833941511198e-08, |
| "logits/chosen": 4.605958277092105, |
| "logits/rejected": 4.559032011572001, |
| "logps/chosen": -317.2640075683594, |
| "logps/rejected": -302.55060958862305, |
| "loss": 0.560026741027832, |
| "mean_token_accuracy": 0.7196670472621918, |
| "num_tokens": 42232440.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.15156788676977156, |
| "rewards/margins": 0.9224099405109882, |
| "rewards/rejected": -0.7708420552313328, |
| "step": 3070 |
| }, |
| { |
| "entropy": 0.9678906172513961, |
| "epoch": 0.7930990086262392, |
| "grad_norm": 58.81368637084961, |
| "learning_rate": 5.425461875734466e-08, |
| "logits/chosen": 4.442958832299348, |
| "logits/rejected": 4.563597377460239, |
| "logps/chosen": -315.42679290771486, |
| "logps/rejected": -327.9312545776367, |
| "loss": 0.4371212959289551, |
| "mean_token_accuracy": 0.7215264499187469, |
| "num_tokens": 42370464.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.28482050970196726, |
| "rewards/margins": 1.1923205159604549, |
| "rewards/rejected": -0.9075000010430813, |
| "step": 3080 |
| }, |
| { |
| "entropy": 1.0114638358354568, |
| "epoch": 0.7956740054074932, |
| "grad_norm": 58.771854400634766, |
| "learning_rate": 5.2964512468381157e-08, |
| "logits/chosen": 4.449208595583224, |
| "logits/rejected": 4.567686457399292, |
| "logps/chosen": -347.89423141479494, |
| "logps/rejected": -334.30847320556643, |
| "loss": 0.5054470062255859, |
| "mean_token_accuracy": 0.7074404180049896, |
| "num_tokens": 42518097.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.09104574248194694, |
| "rewards/margins": 1.058924862742424, |
| "rewards/rejected": -0.9678791239857674, |
| "step": 3090 |
| }, |
| { |
| "entropy": 0.9954713702201843, |
| "epoch": 0.7982490021887473, |
| "grad_norm": 55.89361572265625, |
| "learning_rate": 5.168811027694323e-08, |
| "logits/chosen": 4.395756512540025, |
| "logits/rejected": 4.416072690094935, |
| "logps/chosen": -280.47585067749026, |
| "logps/rejected": -279.00662994384766, |
| "loss": 0.4241953372955322, |
| "mean_token_accuracy": 0.7219798177480697, |
| "num_tokens": 42642299.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.0713214971125126, |
| "rewards/margins": 1.2660470068454743, |
| "rewards/rejected": -1.1947255536913872, |
| "step": 3100 |
| }, |
| { |
| "entropy": 0.9094302177429199, |
| "epoch": 0.8008239989700013, |
| "grad_norm": 50.474430084228516, |
| "learning_rate": 5.042550095861281e-08, |
| "logits/chosen": 4.322333511113185, |
| "logits/rejected": 4.444034928428492, |
| "logps/chosen": -305.94030380249023, |
| "logps/rejected": -325.72607574462893, |
| "loss": 0.5195977210998535, |
| "mean_token_accuracy": 0.7327635198831558, |
| "num_tokens": 42787119.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.0955929946154356, |
| "rewards/margins": 0.9432641051709652, |
| "rewards/rejected": -1.0388571172952652, |
| "step": 3110 |
| }, |
| { |
| "entropy": 0.9371308833360672, |
| "epoch": 0.8033989957512553, |
| "grad_norm": 53.76936721801758, |
| "learning_rate": 4.9176772329658445e-08, |
| "logits/chosen": 4.157149298615895, |
| "logits/rejected": 4.363022352815242, |
| "logps/chosen": -287.5912223815918, |
| "logps/rejected": -296.0829788208008, |
| "loss": 0.5081684112548828, |
| "mean_token_accuracy": 0.7216660618782044, |
| "num_tokens": 42928684.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.134003047272563, |
| "rewards/margins": 0.9698007084429264, |
| "rewards/rejected": -0.8357976607978344, |
| "step": 3120 |
| }, |
| { |
| "entropy": 0.9710374444723129, |
| "epoch": 0.8059739925325093, |
| "grad_norm": 52.81174087524414, |
| "learning_rate": 4.794201124092659e-08, |
| "logits/chosen": 4.387222672277018, |
| "logits/rejected": 4.36880727899687, |
| "logps/chosen": -339.0060348510742, |
| "logps/rejected": -332.5059280395508, |
| "loss": 0.4661799430847168, |
| "mean_token_accuracy": 0.7248915672302246, |
| "num_tokens": 43070782.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.14939685836434363, |
| "rewards/margins": 1.1781152054667472, |
| "rewards/rejected": -1.02871835231781, |
| "step": 3130 |
| }, |
| { |
| "entropy": 1.0249121218919754, |
| "epoch": 0.8085489893137634, |
| "grad_norm": 72.28135681152344, |
| "learning_rate": 4.672130357180168e-08, |
| "logits/chosen": 4.145989070533242, |
| "logits/rejected": 4.332538270159017, |
| "logps/chosen": -330.81416244506835, |
| "logps/rejected": -327.6256950378418, |
| "loss": 0.6214367389678955, |
| "mean_token_accuracy": 0.7014501124620438, |
| "num_tokens": 43203719.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": 0.002490393817424774, |
| "rewards/margins": 0.7461536779999733, |
| "rewards/rejected": -0.7436633050441742, |
| "step": 3140 |
| }, |
| { |
| "entropy": 0.9359357237815857, |
| "epoch": 0.8111239860950173, |
| "grad_norm": 59.6107063293457, |
| "learning_rate": 4.5514734224233076e-08, |
| "logits/chosen": 4.399457422597005, |
| "logits/rejected": 4.390425372401519, |
| "logps/chosen": -318.7063865661621, |
| "logps/rejected": -295.77928161621094, |
| "loss": 0.5429286956787109, |
| "mean_token_accuracy": 0.7206829756498336, |
| "num_tokens": 43342940.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": -0.011263814568519593, |
| "rewards/margins": 0.9415080212056637, |
| "rewards/rejected": -0.9527718558907509, |
| "step": 3150 |
| }, |
| { |
| "entropy": 0.9799446612596512, |
| "epoch": 0.8136989828762714, |
| "grad_norm": 54.60009002685547, |
| "learning_rate": 4.4322387116829454e-08, |
| "logits/chosen": 4.244228730951929, |
| "logits/rejected": 4.38494076500641, |
| "logps/chosen": -321.30857009887694, |
| "logps/rejected": -335.9203987121582, |
| "loss": 0.5400273323059082, |
| "mean_token_accuracy": 0.7213184833526611, |
| "num_tokens": 43477920.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.13054923564195633, |
| "rewards/margins": 1.0041513681411742, |
| "rewards/rejected": -0.8736021481454372, |
| "step": 3160 |
| }, |
| { |
| "entropy": 1.0019416391849518, |
| "epoch": 0.8162739796575255, |
| "grad_norm": 60.097293853759766, |
| "learning_rate": 4.314434517902271e-08, |
| "logits/chosen": 4.187407208641154, |
| "logits/rejected": 4.537707368797923, |
| "logps/chosen": -319.647412109375, |
| "logps/rejected": -298.61428756713866, |
| "loss": 0.5090551376342773, |
| "mean_token_accuracy": 0.7005850493907928, |
| "num_tokens": 43616356.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.032133683934807776, |
| "rewards/margins": 0.9810623221099377, |
| "rewards/rejected": -0.9489286404103041, |
| "step": 3170 |
| }, |
| { |
| "entropy": 0.9200732558965683, |
| "epoch": 0.8188489764387794, |
| "grad_norm": 58.14112091064453, |
| "learning_rate": 4.198069034530008e-08, |
| "logits/chosen": 4.217178553337115, |
| "logits/rejected": 4.304576324351252, |
| "logps/chosen": -338.6818000793457, |
| "logps/rejected": -300.8328262329102, |
| "loss": 0.540928840637207, |
| "mean_token_accuracy": 0.7179835706949234, |
| "num_tokens": 43764948.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.0418201744556427, |
| "rewards/margins": 0.8899469636380672, |
| "rewards/rejected": -0.8481268070638179, |
| "step": 3180 |
| }, |
| { |
| "entropy": 0.9348589479923248, |
| "epoch": 0.8214239732200335, |
| "grad_norm": 64.86296844482422, |
| "learning_rate": 4.0831503549505016e-08, |
| "logits/chosen": 4.202605550969819, |
| "logits/rejected": 4.265748399265876, |
| "logps/chosen": -319.32573852539065, |
| "logps/rejected": -280.73924789428713, |
| "loss": 0.5497491359710693, |
| "mean_token_accuracy": 0.7174855619668961, |
| "num_tokens": 43893288.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.1283643525093794, |
| "rewards/margins": 0.948226423561573, |
| "rewards/rejected": -0.8198620736598968, |
| "step": 3190 |
| }, |
| { |
| "entropy": 0.9861269295215607, |
| "epoch": 0.8239989700012875, |
| "grad_norm": 51.75646209716797, |
| "learning_rate": 3.969686471920852e-08, |
| "logits/chosen": 4.299380680822329, |
| "logits/rejected": 4.30685423706581, |
| "logps/chosen": -350.1937675476074, |
| "logps/rejected": -338.32622756958006, |
| "loss": 0.4989126682281494, |
| "mean_token_accuracy": 0.7143979758024216, |
| "num_tokens": 44043167.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.06968585439026356, |
| "rewards/margins": 1.104652463644743, |
| "rewards/rejected": -1.0349665828049184, |
| "step": 3200 |
| }, |
| { |
| "entropy": 0.9652820855379105, |
| "epoch": 0.8265739667825415, |
| "grad_norm": 85.18624114990234, |
| "learning_rate": 3.857685277015005e-08, |
| "logits/chosen": 4.324221564423723, |
| "logits/rejected": 4.543678736975258, |
| "logps/chosen": -343.1850891113281, |
| "logps/rejected": -327.8295722961426, |
| "loss": 0.5779080867767334, |
| "mean_token_accuracy": 0.7182841390371323, |
| "num_tokens": 44185012.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": -0.023977766185998915, |
| "rewards/margins": 0.8196044005453587, |
| "rewards/rejected": -0.8435821406543255, |
| "step": 3210 |
| }, |
| { |
| "entropy": 0.9091089338064193, |
| "epoch": 0.8291489635637955, |
| "grad_norm": 63.109886169433594, |
| "learning_rate": 3.747154560074861e-08, |
| "logits/chosen": 4.429191732784413, |
| "logits/rejected": 4.195509661221952, |
| "logps/chosen": -299.11571502685547, |
| "logps/rejected": -277.0503555297852, |
| "loss": 0.4867547512054443, |
| "mean_token_accuracy": 0.7231446325778961, |
| "num_tokens": 44326839.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.18592803850769996, |
| "rewards/margins": 1.041901146247983, |
| "rewards/rejected": -0.855973094701767, |
| "step": 3220 |
| }, |
| { |
| "entropy": 0.9785544514656067, |
| "epoch": 0.8317239603450496, |
| "grad_norm": 50.95145034790039, |
| "learning_rate": 3.638102008668517e-08, |
| "logits/chosen": 4.432440797187778, |
| "logits/rejected": 4.528259753053593, |
| "logps/chosen": -327.1904724121094, |
| "logps/rejected": -310.68684844970704, |
| "loss": 0.43732595443725586, |
| "mean_token_accuracy": 0.710511577129364, |
| "num_tokens": 44458796.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.2300154183059931, |
| "rewards/margins": 1.1821377247571945, |
| "rewards/rejected": -0.9521223038434983, |
| "step": 3230 |
| }, |
| { |
| "entropy": 0.9591111004352569, |
| "epoch": 0.8342989571263036, |
| "grad_norm": 63.192962646484375, |
| "learning_rate": 3.530535207555527e-08, |
| "logits/chosen": 4.2694179077771075, |
| "logits/rejected": 4.588908212903034, |
| "logps/chosen": -311.78385467529296, |
| "logps/rejected": -322.0230285644531, |
| "loss": 0.44493680000305175, |
| "mean_token_accuracy": 0.7172895401716233, |
| "num_tokens": 44596694.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.17520832270383835, |
| "rewards/margins": 1.1929794155061244, |
| "rewards/rejected": -1.017771103978157, |
| "step": 3240 |
| }, |
| { |
| "entropy": 0.9898055881261826, |
| "epoch": 0.8368739539075576, |
| "grad_norm": 80.0720443725586, |
| "learning_rate": 3.42446163815942e-08, |
| "logits/chosen": 4.414742148305107, |
| "logits/rejected": 4.522412258383963, |
| "logps/chosen": -355.3474433898926, |
| "logps/rejected": -336.7364868164062, |
| "loss": 0.4767888069152832, |
| "mean_token_accuracy": 0.7169474422931671, |
| "num_tokens": 44729978.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": -0.005838570743799209, |
| "rewards/margins": 1.0999968815594912, |
| "rewards/rejected": -1.1058354690670966, |
| "step": 3250 |
| }, |
| { |
| "entropy": 1.0163926750421524, |
| "epoch": 0.8394489506888116, |
| "grad_norm": 39.93063735961914, |
| "learning_rate": 3.319888678047364e-08, |
| "logits/chosen": 4.1840992037675155, |
| "logits/rejected": 4.442582675435789, |
| "logps/chosen": -325.13705291748045, |
| "logps/rejected": -338.8698677062988, |
| "loss": 0.4550458908081055, |
| "mean_token_accuracy": 0.7010392010211944, |
| "num_tokens": 44864417.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.14707549009472132, |
| "rewards/margins": 1.2349995456635952, |
| "rewards/rejected": -1.0879240721464156, |
| "step": 3260 |
| }, |
| { |
| "entropy": 1.0307422816753387, |
| "epoch": 0.8420239474700657, |
| "grad_norm": 54.57400131225586, |
| "learning_rate": 3.2168236004169873e-08, |
| "logits/chosen": 4.306451708066383, |
| "logits/rejected": 4.507960301818548, |
| "logps/chosen": -371.58898849487304, |
| "logps/rejected": -344.5499099731445, |
| "loss": 0.5361446380615235, |
| "mean_token_accuracy": 0.7136694967746735, |
| "num_tokens": 44997890.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": -0.03218022361397743, |
| "rewards/margins": 0.9815607778728008, |
| "rewards/rejected": -1.0137410029768943, |
| "step": 3270 |
| }, |
| { |
| "entropy": 1.021285566687584, |
| "epoch": 0.8445989442513196, |
| "grad_norm": 53.717735290527344, |
| "learning_rate": 3.115273573590574e-08, |
| "logits/chosen": 4.250017985967512, |
| "logits/rejected": 4.205670235926026, |
| "logps/chosen": -358.8350463867188, |
| "logps/rejected": -320.0092658996582, |
| "loss": 0.511791181564331, |
| "mean_token_accuracy": 0.6992344230413436, |
| "num_tokens": 45139221.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.07899544127285481, |
| "rewards/margins": 1.021796802431345, |
| "rewards/rejected": -0.9428013868629932, |
| "step": 3280 |
| }, |
| { |
| "entropy": 1.0297605782747268, |
| "epoch": 0.8471739410325737, |
| "grad_norm": 54.42903137207031, |
| "learning_rate": 3.015245660516472e-08, |
| "logits/chosen": 4.475107655950782, |
| "logits/rejected": 4.368668733903215, |
| "logps/chosen": -341.9279327392578, |
| "logps/rejected": -352.9533218383789, |
| "loss": 0.4986253261566162, |
| "mean_token_accuracy": 0.6980555474758148, |
| "num_tokens": 45283979.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.17783180978149177, |
| "rewards/margins": 0.9337885938584805, |
| "rewards/rejected": -0.7559567965567112, |
| "step": 3290 |
| }, |
| { |
| "entropy": 1.0129867225885392, |
| "epoch": 0.8497489378138278, |
| "grad_norm": 58.75669860839844, |
| "learning_rate": 2.91674681827786e-08, |
| "logits/chosen": 3.8199001294357067, |
| "logits/rejected": 4.099848671849001, |
| "logps/chosen": -340.3991340637207, |
| "logps/rejected": -338.3979187011719, |
| "loss": 0.49118432998657224, |
| "mean_token_accuracy": 0.7043730199337006, |
| "num_tokens": 45420691.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.15180539079010485, |
| "rewards/margins": 1.1536284662783145, |
| "rewards/rejected": -1.0018231131136417, |
| "step": 3300 |
| }, |
| { |
| "entropy": 1.0250534802675246, |
| "epoch": 0.8523239345950817, |
| "grad_norm": 58.28948211669922, |
| "learning_rate": 2.8197838976088823e-08, |
| "logits/chosen": 4.361926243219319, |
| "logits/rejected": 4.179079459291225, |
| "logps/chosen": -410.66667861938475, |
| "logps/rejected": -378.39041061401366, |
| "loss": 0.5394617080688476, |
| "mean_token_accuracy": 0.6966198533773422, |
| "num_tokens": 45580397.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.12192578278481961, |
| "rewards/margins": 1.0251854434609413, |
| "rewards/rejected": -0.9032596364617348, |
| "step": 3310 |
| }, |
| { |
| "entropy": 0.9336719363927841, |
| "epoch": 0.8548989313763358, |
| "grad_norm": 53.06043243408203, |
| "learning_rate": 2.7243636424181426e-08, |
| "logits/chosen": 4.435972725142554, |
| "logits/rejected": 4.655780388727168, |
| "logps/chosen": -311.94235076904295, |
| "logps/rejected": -322.9733139038086, |
| "loss": 0.4260276794433594, |
| "mean_token_accuracy": 0.7199991792440414, |
| "num_tokens": 45718068.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.15638988018035888, |
| "rewards/margins": 1.2638778954744339, |
| "rewards/rejected": -1.1074880246073007, |
| "step": 3320 |
| }, |
| { |
| "entropy": 1.085283598303795, |
| "epoch": 0.8574739281575898, |
| "grad_norm": 70.15311431884766, |
| "learning_rate": 2.6304926893196745e-08, |
| "logits/chosen": 4.27457033464077, |
| "logits/rejected": 4.373139219715083, |
| "logps/chosen": -358.29911346435546, |
| "logps/rejected": -335.08835525512694, |
| "loss": 0.5034091949462891, |
| "mean_token_accuracy": 0.6905516624450684, |
| "num_tokens": 45850667.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.16071342900395394, |
| "rewards/margins": 1.0567363321781158, |
| "rewards/rejected": -0.8960228696465492, |
| "step": 3330 |
| }, |
| { |
| "entropy": 1.0259439915418624, |
| "epoch": 0.8600489249388438, |
| "grad_norm": 50.04902267456055, |
| "learning_rate": 2.5381775671713816e-08, |
| "logits/chosen": 4.323888256404123, |
| "logits/rejected": 4.3945349140457, |
| "logps/chosen": -333.95324935913084, |
| "logps/rejected": -317.8748207092285, |
| "loss": 0.5272077560424805, |
| "mean_token_accuracy": 0.7033091694116592, |
| "num_tokens": 45984107.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": -0.007355622760951519, |
| "rewards/margins": 0.9154392205178737, |
| "rewards/rejected": -0.9227948397397995, |
| "step": 3340 |
| }, |
| { |
| "entropy": 0.9725571423768997, |
| "epoch": 0.8626239217200978, |
| "grad_norm": 57.20749282836914, |
| "learning_rate": 2.4474246966208812e-08, |
| "logits/chosen": 4.438619917633764, |
| "logits/rejected": 4.5021580303400475, |
| "logps/chosen": -344.06627960205077, |
| "logps/rejected": -323.3831077575684, |
| "loss": 0.4868340492248535, |
| "mean_token_accuracy": 0.7153410166501999, |
| "num_tokens": 46126452.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.028528289496898653, |
| "rewards/margins": 1.1265264317393302, |
| "rewards/rejected": -1.0979981660842895, |
| "step": 3350 |
| }, |
| { |
| "entropy": 0.9808278143405914, |
| "epoch": 0.8651989185013519, |
| "grad_norm": 58.57217788696289, |
| "learning_rate": 2.3582403896589968e-08, |
| "logits/chosen": 4.571170439503101, |
| "logits/rejected": 4.5146572611487485, |
| "logps/chosen": -348.4826530456543, |
| "logps/rejected": -341.46127395629884, |
| "loss": 0.513688039779663, |
| "mean_token_accuracy": 0.7054894149303437, |
| "num_tokens": 46271576.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.2376721866428852, |
| "rewards/margins": 1.028487491607666, |
| "rewards/rejected": -0.7908153183758259, |
| "step": 3360 |
| }, |
| { |
| "entropy": 0.9694880276918412, |
| "epoch": 0.867773915282606, |
| "grad_norm": 80.97119140625, |
| "learning_rate": 2.270630849180735e-08, |
| "logits/chosen": 4.648432265594115, |
| "logits/rejected": 4.546168698768557, |
| "logps/chosen": -330.9306983947754, |
| "logps/rejected": -308.4755485534668, |
| "loss": 0.49057564735412595, |
| "mean_token_accuracy": 0.7102002441883087, |
| "num_tokens": 46417059.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.2763052903115749, |
| "rewards/margins": 1.0597978465259075, |
| "rewards/rejected": -0.7834925360977649, |
| "step": 3370 |
| }, |
| { |
| "entropy": 1.0251123160123825, |
| "epoch": 0.8703489120638599, |
| "grad_norm": 60.645111083984375, |
| "learning_rate": 2.184602168553848e-08, |
| "logits/chosen": 4.418524502990852, |
| "logits/rejected": 4.563836301864508, |
| "logps/chosen": -349.13284454345705, |
| "logps/rejected": -311.8704147338867, |
| "loss": 0.564413595199585, |
| "mean_token_accuracy": 0.7005513310432434, |
| "num_tokens": 46548689.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.10500337593257428, |
| "rewards/margins": 0.8836007565259933, |
| "rewards/rejected": -0.9886041209101677, |
| "step": 3380 |
| }, |
| { |
| "entropy": 0.9626199662685394, |
| "epoch": 0.872923908845114, |
| "grad_norm": 52.24079513549805, |
| "learning_rate": 2.10016033119505e-08, |
| "logits/chosen": 4.275510975573473, |
| "logits/rejected": 4.560188272155659, |
| "logps/chosen": -325.59469375610354, |
| "logps/rejected": -311.76687850952146, |
| "loss": 0.4976241111755371, |
| "mean_token_accuracy": 0.7235046684741974, |
| "num_tokens": 46688467.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": -0.009001258015632629, |
| "rewards/margins": 1.0198740661144257, |
| "rewards/rejected": -1.0288753151893615, |
| "step": 3390 |
| }, |
| { |
| "entropy": 0.9624796003103256, |
| "epoch": 0.875498905626368, |
| "grad_norm": 60.422061920166016, |
| "learning_rate": 2.0173112101538575e-08, |
| "logits/chosen": 4.165599870005346, |
| "logits/rejected": 4.334379304055487, |
| "logps/chosen": -355.1679443359375, |
| "logps/rejected": -329.28182525634764, |
| "loss": 0.524689769744873, |
| "mean_token_accuracy": 0.7226759999990463, |
| "num_tokens": 46824018.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.01133049912750721, |
| "rewards/margins": 0.9631065726280212, |
| "rewards/rejected": -0.9517760708928108, |
| "step": 3400 |
| }, |
| { |
| "entropy": 0.9720148354768753, |
| "epoch": 0.878073902407622, |
| "grad_norm": 51.23833084106445, |
| "learning_rate": 1.936060567704101e-08, |
| "logits/chosen": 4.417957401809131, |
| "logits/rejected": 4.420553470939984, |
| "logps/chosen": -347.849609375, |
| "logps/rejected": -325.4596824645996, |
| "loss": 0.5499835968017578, |
| "mean_token_accuracy": 0.7164855569601059, |
| "num_tokens": 46962450.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.06175786927342415, |
| "rewards/margins": 1.0169452384114266, |
| "rewards/rejected": -0.9551873594522476, |
| "step": 3410 |
| }, |
| { |
| "entropy": 0.9205372929573059, |
| "epoch": 0.880648899188876, |
| "grad_norm": 71.48031616210938, |
| "learning_rate": 1.8564140549431784e-08, |
| "logits/chosen": 4.543263375398614, |
| "logits/rejected": 4.34733416576674, |
| "logps/chosen": -323.281729888916, |
| "logps/rejected": -320.5871925354004, |
| "loss": 0.5773813247680664, |
| "mean_token_accuracy": 0.7166509211063385, |
| "num_tokens": 47108489.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.012704336643218994, |
| "rewards/margins": 0.8792961195111275, |
| "rewards/rejected": -0.8920004740357399, |
| "step": 3420 |
| }, |
| { |
| "entropy": 1.0096068501472473, |
| "epoch": 0.8832238959701301, |
| "grad_norm": 39.13214874267578, |
| "learning_rate": 1.778377211398968e-08, |
| "logits/chosen": 4.290272215409019, |
| "logits/rejected": 4.223242211919704, |
| "logps/chosen": -360.93024291992185, |
| "logps/rejected": -334.8608901977539, |
| "loss": 0.43523283004760743, |
| "mean_token_accuracy": 0.7145869106054306, |
| "num_tokens": 47245415.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.17010074853897095, |
| "rewards/margins": 1.2077522069215774, |
| "rewards/rejected": -1.0376514531672, |
| "step": 3430 |
| }, |
| { |
| "entropy": 0.9463884651660919, |
| "epoch": 0.885798892751384, |
| "grad_norm": 58.91367721557617, |
| "learning_rate": 1.7019554646445834e-08, |
| "logits/chosen": 4.213345909251523, |
| "logits/rejected": 4.4840705751237895, |
| "logps/chosen": -325.1591438293457, |
| "logps/rejected": -360.0714385986328, |
| "loss": 0.4720927238464355, |
| "mean_token_accuracy": 0.7402132153511047, |
| "num_tokens": 47386884.0, |
| "rewards/accuracies": 0.8, |
| "rewards/chosen": 0.17252297177910805, |
| "rewards/margins": 1.1690418511629104, |
| "rewards/rejected": -0.9965188939124345, |
| "step": 3440 |
| }, |
| { |
| "entropy": 0.9820081353187561, |
| "epoch": 0.8883738895326381, |
| "grad_norm": 70.7621078491211, |
| "learning_rate": 1.6271541299208685e-08, |
| "logits/chosen": 4.105262020171894, |
| "logits/rejected": 4.634721768465438, |
| "logps/chosen": -344.67479553222654, |
| "logps/rejected": -343.77735366821287, |
| "loss": 0.45592031478881834, |
| "mean_token_accuracy": 0.7207370430231095, |
| "num_tokens": 47530905.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.15923903696238995, |
| "rewards/margins": 1.182755758613348, |
| "rewards/rejected": -1.0235167093575002, |
| "step": 3450 |
| }, |
| { |
| "entropy": 1.001122486591339, |
| "epoch": 0.8909488863138921, |
| "grad_norm": 60.54716873168945, |
| "learning_rate": 1.5539784097667124e-08, |
| "logits/chosen": 4.40741366964353, |
| "logits/rejected": 4.270748510304506, |
| "logps/chosen": -306.87539291381836, |
| "logps/rejected": -299.2044052124023, |
| "loss": 0.5292521476745605, |
| "mean_token_accuracy": 0.7106813669204712, |
| "num_tokens": 47659337.0, |
| "rewards/accuracies": 0.70625, |
| "rewards/chosen": 0.09808586463332176, |
| "rewards/margins": 0.9288386881351471, |
| "rewards/rejected": -0.8307528175413609, |
| "step": 3460 |
| }, |
| { |
| "entropy": 0.9732944667339325, |
| "epoch": 0.8935238830951461, |
| "grad_norm": 48.38511657714844, |
| "learning_rate": 1.4824333936572037e-08, |
| "logits/chosen": 4.468958133684264, |
| "logits/rejected": 4.732916791050794, |
| "logps/chosen": -304.51147232055666, |
| "logps/rejected": -310.21005630493164, |
| "loss": 0.5139917373657227, |
| "mean_token_accuracy": 0.6996112436056137, |
| "num_tokens": 47782251.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.17072734758257865, |
| "rewards/margins": 1.0207956433296204, |
| "rewards/rejected": -0.850068312138319, |
| "step": 3470 |
| }, |
| { |
| "entropy": 0.9817721247673035, |
| "epoch": 0.8960988798764001, |
| "grad_norm": 69.75738525390625, |
| "learning_rate": 1.41252405764965e-08, |
| "logits/chosen": 4.303265819710651, |
| "logits/rejected": 4.364314668182876, |
| "logps/chosen": -354.1113571166992, |
| "logps/rejected": -362.1471939086914, |
| "loss": 0.494260311126709, |
| "mean_token_accuracy": 0.7172562003135681, |
| "num_tokens": 47931522.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.0932394809089601, |
| "rewards/margins": 1.0632715754210949, |
| "rewards/rejected": -0.9700321093201637, |
| "step": 3480 |
| }, |
| { |
| "entropy": 0.9759364396333694, |
| "epoch": 0.8986738766576542, |
| "grad_norm": 54.06985855102539, |
| "learning_rate": 1.3442552640374943e-08, |
| "logits/chosen": 4.454683544464034, |
| "logits/rejected": 4.838835310257229, |
| "logps/chosen": -325.03923416137695, |
| "logps/rejected": -336.8667266845703, |
| "loss": 0.46573600769042967, |
| "mean_token_accuracy": 0.7211115062236786, |
| "num_tokens": 48064681.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.0038886144757270813, |
| "rewards/margins": 1.223842778801918, |
| "rewards/rejected": -1.219954188168049, |
| "step": 3490 |
| }, |
| { |
| "entropy": 0.9556342899799347, |
| "epoch": 0.9012488734389082, |
| "grad_norm": 60.828941345214844, |
| "learning_rate": 1.2776317610121296e-08, |
| "logits/chosen": 4.4151058022555665, |
| "logits/rejected": 4.379737999278982, |
| "logps/chosen": -344.0136268615723, |
| "logps/rejected": -304.6720855712891, |
| "loss": 0.5218825817108155, |
| "mean_token_accuracy": 0.7084827989339828, |
| "num_tokens": 48196983.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.16754340901970863, |
| "rewards/margins": 1.1194391429424286, |
| "rewards/rejected": -0.9518957525491715, |
| "step": 3500 |
| }, |
| { |
| "entropy": 1.016376093029976, |
| "epoch": 0.9038238702201622, |
| "grad_norm": 38.34528350830078, |
| "learning_rate": 1.2126581823326487e-08, |
| "logits/chosen": 4.517608908474623, |
| "logits/rejected": 4.287675331926841, |
| "logps/chosen": -353.27395095825193, |
| "logps/rejected": -344.9620536804199, |
| "loss": 0.47306151390075685, |
| "mean_token_accuracy": 0.7100744724273682, |
| "num_tokens": 48334195.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.15619846247136593, |
| "rewards/margins": 1.1464665360748767, |
| "rewards/rejected": -0.9902680486440658, |
| "step": 3510 |
| }, |
| { |
| "entropy": 0.9969549298286438, |
| "epoch": 0.9063988670014163, |
| "grad_norm": 76.98277282714844, |
| "learning_rate": 1.1493390470035752e-08, |
| "logits/chosen": 4.47879155288048, |
| "logits/rejected": 4.561442948216003, |
| "logps/chosen": -340.20881576538085, |
| "logps/rejected": -316.81405181884764, |
| "loss": 0.5304606437683106, |
| "mean_token_accuracy": 0.7083680987358093, |
| "num_tokens": 48464269.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.08661839216947556, |
| "rewards/margins": 0.9748988553881646, |
| "rewards/rejected": -0.8882804535329342, |
| "step": 3520 |
| }, |
| { |
| "entropy": 0.9985833793878556, |
| "epoch": 0.9089738637826703, |
| "grad_norm": 57.09160614013672, |
| "learning_rate": 1.0876787589605502e-08, |
| "logits/chosen": 4.438816131902436, |
| "logits/rejected": 4.630601574751163, |
| "logps/chosen": -326.38246154785156, |
| "logps/rejected": -330.35162963867185, |
| "loss": 0.5265772819519043, |
| "mean_token_accuracy": 0.7096292585134506, |
| "num_tokens": 48597852.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.12409729398787021, |
| "rewards/margins": 0.9895732261240482, |
| "rewards/rejected": -0.865475931763649, |
| "step": 3530 |
| }, |
| { |
| "entropy": 0.9311624348163605, |
| "epoch": 0.9115488605639243, |
| "grad_norm": 31.453582763671875, |
| "learning_rate": 1.0276816067640277e-08, |
| "logits/chosen": 4.354476711818544, |
| "logits/rejected": 4.563276009580368, |
| "logps/chosen": -297.76134796142577, |
| "logps/rejected": -302.59604721069337, |
| "loss": 0.4661102294921875, |
| "mean_token_accuracy": 0.7178372591733932, |
| "num_tokens": 48733326.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.053084198385477066, |
| "rewards/margins": 1.0801889322698117, |
| "rewards/rejected": -1.0271047309041024, |
| "step": 3540 |
| }, |
| { |
| "entropy": 1.0048164397478103, |
| "epoch": 0.9141238573451783, |
| "grad_norm": 54.727439880371094, |
| "learning_rate": 9.693517633010157e-09, |
| "logits/chosen": 4.331816697040276, |
| "logits/rejected": 4.619338770355471, |
| "logps/chosen": -323.12854995727537, |
| "logps/rejected": -349.59267578125, |
| "loss": 0.45178980827331544, |
| "mean_token_accuracy": 0.7198001354932785, |
| "num_tokens": 48879817.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.1579263649880886, |
| "rewards/margins": 1.2303193509578705, |
| "rewards/rejected": -1.0723929926753044, |
| "step": 3550 |
| }, |
| { |
| "entropy": 1.0108634173870086, |
| "epoch": 0.9166988541264324, |
| "grad_norm": 55.493743896484375, |
| "learning_rate": 9.126932854948255e-09, |
| "logits/chosen": 4.160852975325641, |
| "logits/rejected": 4.183551900899551, |
| "logps/chosen": -344.4482650756836, |
| "logps/rejected": -323.087296295166, |
| "loss": 0.5426519393920899, |
| "mean_token_accuracy": 0.6971057683229447, |
| "num_tokens": 49017367.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.11907950565218925, |
| "rewards/margins": 0.898277985304594, |
| "rewards/rejected": -0.7791984729468823, |
| "step": 3560 |
| }, |
| { |
| "entropy": 0.9394759550690651, |
| "epoch": 0.9192738509076863, |
| "grad_norm": 66.28225708007812, |
| "learning_rate": 8.57710114022922e-09, |
| "logits/chosen": 4.451378819490435, |
| "logits/rejected": 4.237397623162968, |
| "logps/chosen": -331.27843322753904, |
| "logps/rejected": -334.40370483398436, |
| "loss": 0.4559895992279053, |
| "mean_token_accuracy": 0.7293369919061661, |
| "num_tokens": 49174005.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": 0.27012583091855047, |
| "rewards/margins": 1.190965287387371, |
| "rewards/rejected": -0.92083944901824, |
| "step": 3570 |
| }, |
| { |
| "entropy": 0.976487186551094, |
| "epoch": 0.9218488476889404, |
| "grad_norm": 84.45362091064453, |
| "learning_rate": 8.044060730428431e-09, |
| "logits/chosen": 4.495778418241548, |
| "logits/rejected": 4.430111787511347, |
| "logps/chosen": -330.96579513549807, |
| "logps/rejected": -310.9513671875, |
| "loss": 0.5034812927246094, |
| "mean_token_accuracy": 0.7043195247650147, |
| "num_tokens": 49309238.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.06998340710997582, |
| "rewards/margins": 0.9363695189356804, |
| "rewards/rejected": -0.8663861125707626, |
| "step": 3580 |
| }, |
| { |
| "entropy": 0.9788063317537308, |
| "epoch": 0.9244238444701944, |
| "grad_norm": 42.2854118347168, |
| "learning_rate": 7.527848699262151e-09, |
| "logits/chosen": 4.376404988570172, |
| "logits/rejected": 4.37388951762519, |
| "logps/chosen": -334.96368713378905, |
| "logps/rejected": -303.0596351623535, |
| "loss": 0.5014265060424805, |
| "mean_token_accuracy": 0.7028478592634201, |
| "num_tokens": 49444791.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.06081862226128578, |
| "rewards/margins": 1.1488137885928154, |
| "rewards/rejected": -1.0879951477050782, |
| "step": 3590 |
| }, |
| { |
| "entropy": 0.9703697711229324, |
| "epoch": 0.9269988412514485, |
| "grad_norm": 63.97004318237305, |
| "learning_rate": 7.028500950009036e-09, |
| "logits/chosen": 4.33694045197637, |
| "logits/rejected": 4.370983334664355, |
| "logps/chosen": -337.7773941040039, |
| "logps/rejected": -343.25316162109374, |
| "loss": 0.47454023361206055, |
| "mean_token_accuracy": 0.7206103503704071, |
| "num_tokens": 49588105.0, |
| "rewards/accuracies": 0.7625, |
| "rewards/chosen": 0.12851075902581216, |
| "rewards/margins": 1.0958582788705826, |
| "rewards/rejected": -0.9673475250601768, |
| "step": 3600 |
| }, |
| { |
| "entropy": 0.9552771568298339, |
| "epoch": 0.9295738380327024, |
| "grad_norm": 74.13584899902344, |
| "learning_rate": 6.546052213013076e-09, |
| "logits/chosen": 4.549494020329994, |
| "logits/rejected": 4.766378175997125, |
| "logps/chosen": -322.0286994934082, |
| "logps/rejected": -304.0044807434082, |
| "loss": 0.493131160736084, |
| "mean_token_accuracy": 0.7168387413024903, |
| "num_tokens": 49725681.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.12400502972304821, |
| "rewards/margins": 0.9972663760185242, |
| "rewards/rejected": -0.873261358588934, |
| "step": 3610 |
| }, |
| { |
| "entropy": 0.9899664372205734, |
| "epoch": 0.9321488348139565, |
| "grad_norm": 73.20947265625, |
| "learning_rate": 6.08053604326797e-09, |
| "logits/chosen": 4.3790757163409, |
| "logits/rejected": 4.53567903067453, |
| "logps/chosen": -368.3445587158203, |
| "logps/rejected": -335.86344680786135, |
| "loss": 0.5364912986755371, |
| "mean_token_accuracy": 0.7091458439826965, |
| "num_tokens": 49867742.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.019294209219515322, |
| "rewards/margins": 0.9568890571594239, |
| "rewards/rejected": -0.9375948514789343, |
| "step": 3620 |
| }, |
| { |
| "entropy": 0.9530099391937256, |
| "epoch": 0.9347238315952106, |
| "grad_norm": 48.23020935058594, |
| "learning_rate": 5.631984818083329e-09, |
| "logits/chosen": 4.296055677285347, |
| "logits/rejected": 4.332563173146445, |
| "logps/chosen": -330.5657104492187, |
| "logps/rejected": -337.90441665649416, |
| "loss": 0.502886962890625, |
| "mean_token_accuracy": 0.7261953383684159, |
| "num_tokens": 50020610.0, |
| "rewards/accuracies": 0.73125, |
| "rewards/chosen": 0.011554064974188805, |
| "rewards/margins": 0.980589747056365, |
| "rewards/rejected": -0.9690356459468603, |
| "step": 3630 |
| }, |
| { |
| "entropy": 1.016004478931427, |
| "epoch": 0.9372988283764645, |
| "grad_norm": 26.548263549804688, |
| "learning_rate": 5.2004297348328675e-09, |
| "logits/chosen": 4.448092568149213, |
| "logits/rejected": 4.512511965425527, |
| "logps/chosen": -342.29409942626955, |
| "logps/rejected": -327.07233200073244, |
| "loss": 0.5192561626434327, |
| "mean_token_accuracy": 0.6933602273464203, |
| "num_tokens": 50167159.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": -0.028628817386925222, |
| "rewards/margins": 0.975167042016983, |
| "rewards/rejected": -1.0037958763539792, |
| "step": 3640 |
| }, |
| { |
| "entropy": 1.037110298871994, |
| "epoch": 0.9398738251577186, |
| "grad_norm": 49.208045959472656, |
| "learning_rate": 4.785900808784521e-09, |
| "logits/chosen": 4.18404633139661, |
| "logits/rejected": 4.366859724127451, |
| "logps/chosen": -368.90018463134766, |
| "logps/rejected": -326.96028289794924, |
| "loss": 0.5454651832580566, |
| "mean_token_accuracy": 0.6996820390224456, |
| "num_tokens": 50305415.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.1265437588095665, |
| "rewards/margins": 0.8717910453677178, |
| "rewards/rejected": -0.7452472664415837, |
| "step": 3650 |
| }, |
| { |
| "entropy": 0.9307460784912109, |
| "epoch": 0.9424488219389726, |
| "grad_norm": 57.491458892822266, |
| "learning_rate": 4.38842687101279e-09, |
| "logits/chosen": 4.498533801183553, |
| "logits/rejected": 4.700642188877437, |
| "logps/chosen": -334.06893768310545, |
| "logps/rejected": -319.5280303955078, |
| "loss": 0.5305843830108643, |
| "mean_token_accuracy": 0.7245637029409409, |
| "num_tokens": 50440896.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.10404398180544376, |
| "rewards/margins": 1.0015343960374594, |
| "rewards/rejected": -0.8974904328584671, |
| "step": 3660 |
| }, |
| { |
| "entropy": 0.9172339409589767, |
| "epoch": 0.9450238187202266, |
| "grad_norm": 49.77804183959961, |
| "learning_rate": 4.008035566393675e-09, |
| "logits/chosen": 4.25483115385353, |
| "logits/rejected": 4.390901937743865, |
| "logps/chosen": -317.6829383850098, |
| "logps/rejected": -300.2950981140137, |
| "loss": 0.5132992744445801, |
| "mean_token_accuracy": 0.721070921421051, |
| "num_tokens": 50582442.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.06545577980577946, |
| "rewards/margins": 0.9651219345629215, |
| "rewards/rejected": -0.8996661596000195, |
| "step": 3670 |
| }, |
| { |
| "entropy": 0.9238002628087998, |
| "epoch": 0.9475988155014806, |
| "grad_norm": 56.991695404052734, |
| "learning_rate": 3.6447533516816843e-09, |
| "logits/chosen": 4.512138319031409, |
| "logits/rejected": 4.6444846471796835, |
| "logps/chosen": -300.3995414733887, |
| "logps/rejected": -277.4478889465332, |
| "loss": 0.5094088077545166, |
| "mean_token_accuracy": 0.7250993430614472, |
| "num_tokens": 50718730.0, |
| "rewards/accuracies": 0.75625, |
| "rewards/chosen": 0.17520229630172252, |
| "rewards/margins": 0.9895223274827003, |
| "rewards/rejected": -0.8143200561404228, |
| "step": 3680 |
| }, |
| { |
| "entropy": 0.9735976099967957, |
| "epoch": 0.9501738122827347, |
| "grad_norm": 52.92755889892578, |
| "learning_rate": 3.298605493669976e-09, |
| "logits/chosen": 4.401222899972965, |
| "logits/rejected": 4.386323304332022, |
| "logps/chosen": -317.9237892150879, |
| "logps/rejected": -314.15701446533205, |
| "loss": 0.5146839141845703, |
| "mean_token_accuracy": 0.7224712401628495, |
| "num_tokens": 50853694.0, |
| "rewards/accuracies": 0.7375, |
| "rewards/chosen": 0.12512148544192314, |
| "rewards/margins": 1.0661357514560224, |
| "rewards/rejected": -0.941014252603054, |
| "step": 3690 |
| }, |
| { |
| "entropy": 1.0029818892478943, |
| "epoch": 0.9527488090639886, |
| "grad_norm": 47.47979736328125, |
| "learning_rate": 2.969616067432873e-09, |
| "logits/chosen": 4.289798602571775, |
| "logits/rejected": 4.290125592710071, |
| "logps/chosen": -332.7917869567871, |
| "logps/rejected": -309.6506996154785, |
| "loss": 0.5282728195190429, |
| "mean_token_accuracy": 0.7061104476451874, |
| "num_tokens": 50973757.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.04582103714346886, |
| "rewards/margins": 1.0365157380700112, |
| "rewards/rejected": -0.990694697573781, |
| "step": 3700 |
| }, |
| { |
| "entropy": 0.948711308836937, |
| "epoch": 0.9553238058452427, |
| "grad_norm": 59.510738372802734, |
| "learning_rate": 2.65780795465137e-09, |
| "logits/chosen": 4.399845970240033, |
| "logits/rejected": 4.513711278582631, |
| "logps/chosen": -344.8980438232422, |
| "logps/rejected": -307.91698532104493, |
| "loss": 0.5410758018493652, |
| "mean_token_accuracy": 0.7102200150489807, |
| "num_tokens": 51118168.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": -0.2040230341255665, |
| "rewards/margins": 0.9633007615804672, |
| "rewards/rejected": -1.1673237793147564, |
| "step": 3710 |
| }, |
| { |
| "entropy": 0.9567266225814819, |
| "epoch": 0.9578988026264967, |
| "grad_norm": 65.30461120605469, |
| "learning_rate": 2.3632028420218496e-09, |
| "logits/chosen": 4.403461993684977, |
| "logits/rejected": 4.558946817995839, |
| "logps/chosen": -352.7233604431152, |
| "logps/rejected": -305.87541580200195, |
| "loss": 0.5773183345794678, |
| "mean_token_accuracy": 0.7121285676956177, |
| "num_tokens": 51256978.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": 0.035476422682404515, |
| "rewards/margins": 0.8842684037983417, |
| "rewards/rejected": -0.8487919729202986, |
| "step": 3720 |
| }, |
| { |
| "entropy": 1.0510099142789842, |
| "epoch": 0.9604737994077508, |
| "grad_norm": 26.84271240234375, |
| "learning_rate": 2.0858212197476245e-09, |
| "logits/chosen": 4.372463580494778, |
| "logits/rejected": 4.265729635275705, |
| "logps/chosen": -324.15354919433594, |
| "logps/rejected": -302.15199890136716, |
| "loss": 0.49859132766723635, |
| "mean_token_accuracy": 0.7022114306688308, |
| "num_tokens": 51385761.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": -0.035042554885149, |
| "rewards/margins": 1.1020322252064942, |
| "rewards/rejected": -1.1370747864246369, |
| "step": 3730 |
| }, |
| { |
| "entropy": 0.994208711385727, |
| "epoch": 0.9630487961890047, |
| "grad_norm": 46.86901092529297, |
| "learning_rate": 1.8256823801137689e-09, |
| "logits/chosen": 4.404450478702302, |
| "logits/rejected": 4.57243674071521, |
| "logps/chosen": -300.2155632019043, |
| "logps/rejected": -299.6177520751953, |
| "loss": 0.46028990745544435, |
| "mean_token_accuracy": 0.7044767260551452, |
| "num_tokens": 51516979.0, |
| "rewards/accuracies": 0.7875, |
| "rewards/chosen": 0.23469365537166595, |
| "rewards/margins": 1.0552187602967025, |
| "rewards/rejected": -0.8205250978469849, |
| "step": 3740 |
| }, |
| { |
| "entropy": 0.9790441781282425, |
| "epoch": 0.9656237929702588, |
| "grad_norm": 51.8381233215332, |
| "learning_rate": 1.582804416145417e-09, |
| "logits/chosen": 4.391877838478205, |
| "logits/rejected": 4.334116422860771, |
| "logps/chosen": -325.6937026977539, |
| "logps/rejected": -335.8347755432129, |
| "loss": 0.5244534492492676, |
| "mean_token_accuracy": 0.7070640385150909, |
| "num_tokens": 51647982.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.10406715907156468, |
| "rewards/margins": 1.0737947836518287, |
| "rewards/rejected": -0.9697276167571545, |
| "step": 3750 |
| }, |
| { |
| "entropy": 1.053725364804268, |
| "epoch": 0.9681987897515129, |
| "grad_norm": 39.69020462036133, |
| "learning_rate": 1.3572042203493784e-09, |
| "logits/chosen": 4.288184999106501, |
| "logits/rejected": 4.723794248653045, |
| "logps/chosen": -301.32601737976074, |
| "logps/rejected": -288.46766204833983, |
| "loss": 0.5159613609313964, |
| "mean_token_accuracy": 0.7011086165904998, |
| "num_tokens": 51773767.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -0.0047888275235891346, |
| "rewards/margins": 0.9299458310008049, |
| "rewards/rejected": -0.9347346603870392, |
| "step": 3760 |
| }, |
| { |
| "entropy": 0.9579876095056534, |
| "epoch": 0.9707737865327668, |
| "grad_norm": 59.244327545166016, |
| "learning_rate": 1.1488974835390796e-09, |
| "logits/chosen": 4.175309925483745, |
| "logits/rejected": 4.1302942537954745, |
| "logps/chosen": -332.2213508605957, |
| "logps/rejected": -316.0274543762207, |
| "loss": 0.5204493045806885, |
| "mean_token_accuracy": 0.6993423879146576, |
| "num_tokens": 51915427.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.01833348087966442, |
| "rewards/margins": 1.0724851697683335, |
| "rewards/rejected": -1.0541516847908496, |
| "step": 3770 |
| }, |
| { |
| "entropy": 0.9610406279563903, |
| "epoch": 0.9733487833140209, |
| "grad_norm": 46.04349136352539, |
| "learning_rate": 9.57898693743464e-10, |
| "logits/chosen": 4.1796459032978195, |
| "logits/rejected": 4.298540422448867, |
| "logps/chosen": -337.83121490478516, |
| "logps/rejected": -350.1724571228027, |
| "loss": 0.4208075523376465, |
| "mean_token_accuracy": 0.7275057882070541, |
| "num_tokens": 52055506.0, |
| "rewards/accuracies": 0.76875, |
| "rewards/chosen": 0.03156646024435759, |
| "rewards/margins": 1.250686925649643, |
| "rewards/rejected": -1.2191204607486725, |
| "step": 3780 |
| }, |
| { |
| "entropy": 0.9695363134145737, |
| "epoch": 0.9759237800952749, |
| "grad_norm": 62.886592864990234, |
| "learning_rate": 7.842211351991312e-10, |
| "logits/chosen": 4.404529722887416, |
| "logits/rejected": 4.35932479335686, |
| "logps/chosen": -335.45074768066405, |
| "logps/rejected": -331.40769653320314, |
| "loss": 0.48799610137939453, |
| "mean_token_accuracy": 0.7236330389976502, |
| "num_tokens": 52199784.0, |
| "rewards/accuracies": 0.775, |
| "rewards/chosen": -0.024450248479843138, |
| "rewards/margins": 1.0116318702697753, |
| "rewards/rejected": -1.0360821306705474, |
| "step": 3790 |
| }, |
| { |
| "entropy": 1.002945750951767, |
| "epoch": 0.9784987768765289, |
| "grad_norm": 55.1344108581543, |
| "learning_rate": 6.278768874265206e-10, |
| "logits/chosen": 4.345139753849429, |
| "logits/rejected": 4.478437201822584, |
| "logps/chosen": -299.3524765014648, |
| "logps/rejected": -308.5686767578125, |
| "loss": 0.5122725009918213, |
| "mean_token_accuracy": 0.7084631592035293, |
| "num_tokens": 52332543.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.05897114500403404, |
| "rewards/margins": 1.132245208323002, |
| "rewards/rejected": -1.0732740998268127, |
| "step": 3800 |
| }, |
| { |
| "entropy": 0.9791542381048203, |
| "epoch": 0.9810737736577829, |
| "grad_norm": 74.19060516357422, |
| "learning_rate": 4.888768243898056e-10, |
| "logits/chosen": 4.233951982196901, |
| "logits/rejected": 4.480447268588881, |
| "logps/chosen": -290.825382232666, |
| "logps/rejected": -293.841047668457, |
| "loss": 0.5422379493713378, |
| "mean_token_accuracy": 0.7104681402444839, |
| "num_tokens": 52467596.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.043087200820446016, |
| "rewards/margins": 1.0256297551095486, |
| "rewards/rejected": -0.9825425572693348, |
| "step": 3810 |
| }, |
| { |
| "entropy": 0.9377125799655914, |
| "epoch": 0.983648770439037, |
| "grad_norm": 62.60303497314453, |
| "learning_rate": 3.6723061374036025e-10, |
| "logits/chosen": 4.597417417132064, |
| "logits/rejected": 4.760054064041833, |
| "logps/chosen": -293.8340728759766, |
| "logps/rejected": -323.3628387451172, |
| "loss": 0.6000535011291503, |
| "mean_token_accuracy": 0.7310346335172653, |
| "num_tokens": 52603104.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": -0.01746976636350155, |
| "rewards/margins": 0.8278767816722393, |
| "rewards/rejected": -0.8453465715050698, |
| "step": 3820 |
| }, |
| { |
| "entropy": 0.9405255407094956, |
| "epoch": 0.9862237672202909, |
| "grad_norm": 69.44036102294922, |
| "learning_rate": 2.629467161446852e-10, |
| "logits/chosen": 4.497785698829776, |
| "logits/rejected": 4.675584302140631, |
| "logps/chosen": -315.19013671875, |
| "logps/rejected": -322.6870307922363, |
| "loss": 0.5892306327819824, |
| "mean_token_accuracy": 0.7260134309530258, |
| "num_tokens": 52745037.0, |
| "rewards/accuracies": 0.69375, |
| "rewards/chosen": 0.04895561598241329, |
| "rewards/margins": 0.8657831616699696, |
| "rewards/rejected": -0.8168275523930788, |
| "step": 3830 |
| }, |
| { |
| "entropy": 0.9618756115436554, |
| "epoch": 0.988798764001545, |
| "grad_norm": 76.69972229003906, |
| "learning_rate": 1.7603238469571234e-10, |
| "logits/chosen": 4.24194299584001, |
| "logits/rejected": 4.270564125145848, |
| "logps/chosen": -352.28253479003905, |
| "logps/rejected": -338.86647338867186, |
| "loss": 0.5200666427612305, |
| "mean_token_accuracy": 0.7175153642892838, |
| "num_tokens": 52888306.0, |
| "rewards/accuracies": 0.725, |
| "rewards/chosen": 0.06362154483795165, |
| "rewards/margins": 1.0577692627906798, |
| "rewards/rejected": -0.994147739931941, |
| "step": 3840 |
| }, |
| { |
| "entropy": 0.9540875136852265, |
| "epoch": 0.991373760782799, |
| "grad_norm": 72.23999786376953, |
| "learning_rate": 1.0649366440837471e-10, |
| "logits/chosen": 4.449182385583951, |
| "logits/rejected": 4.436637852350745, |
| "logps/chosen": -391.4891944885254, |
| "logps/rejected": -323.86412811279297, |
| "loss": 0.5273003578186035, |
| "mean_token_accuracy": 0.7144930273294449, |
| "num_tokens": 53031726.0, |
| "rewards/accuracies": 0.74375, |
| "rewards/chosen": 0.18170544244349002, |
| "rewards/margins": 1.0253052242100238, |
| "rewards/rejected": -0.8435997664928436, |
| "step": 3850 |
| }, |
| { |
| "entropy": 0.9514907240867615, |
| "epoch": 0.9939487575640531, |
| "grad_norm": 53.7625617980957, |
| "learning_rate": 5.433539179930391e-11, |
| "logits/chosen": 4.684322845812066, |
| "logits/rejected": 4.2598464409633285, |
| "logps/chosen": -322.4320526123047, |
| "logps/rejected": -303.11545791625974, |
| "loss": 0.5720946311950683, |
| "mean_token_accuracy": 0.7100042164325714, |
| "num_tokens": 53161230.0, |
| "rewards/accuracies": 0.7, |
| "rewards/chosen": -0.11039275005459785, |
| "rewards/margins": 0.9645901948213578, |
| "rewards/rejected": -1.0749829187989235, |
| "step": 3860 |
| }, |
| { |
| "entropy": 0.9525843232870101, |
| "epoch": 0.996523754345307, |
| "grad_norm": 60.94110870361328, |
| "learning_rate": 1.9561194550293815e-11, |
| "logits/chosen": 4.319067716941498, |
| "logits/rejected": 4.458056856677681, |
| "logps/chosen": -330.46614227294924, |
| "logps/rejected": -335.6721809387207, |
| "loss": 0.4807633399963379, |
| "mean_token_accuracy": 0.7251676887273788, |
| "num_tokens": 53304079.0, |
| "rewards/accuracies": 0.79375, |
| "rewards/chosen": 0.16729226494207977, |
| "rewards/margins": 1.125135461986065, |
| "rewards/rejected": -0.9578431934118271, |
| "step": 3870 |
| }, |
| { |
| "entropy": 0.9905207127332687, |
| "epoch": 0.9990987511265611, |
| "grad_norm": 51.55181121826172, |
| "learning_rate": 2.1734912560855688e-12, |
| "logits/chosen": 4.26814043785083, |
| "logits/rejected": 4.257808624133493, |
| "logps/chosen": -362.62685546875, |
| "logps/rejected": -311.6571701049805, |
| "loss": 0.563578462600708, |
| "mean_token_accuracy": 0.7050950795412063, |
| "num_tokens": 53435910.0, |
| "rewards/accuracies": 0.675, |
| "rewards/chosen": -0.00768631212413311, |
| "rewards/margins": 0.872562188655138, |
| "rewards/rejected": -0.8802485037595034, |
| "step": 3880 |
| }, |
| { |
| "entropy": 0.9510993446622577, |
| "epoch": 1.0, |
| "logits/chosen": 4.121521583799508, |
| "logits/rejected": 3.973326885699311, |
| "logps/chosen": -317.1263449532645, |
| "logps/rejected": -297.800528390067, |
| "mean_token_accuracy": 0.7126511675970895, |
| "num_tokens": 53480717.0, |
| "rewards/accuracies": 0.8392857142857143, |
| "rewards/chosen": 0.34361560216971804, |
| "rewards/margins": 1.3156322070530482, |
| "rewards/rejected": -0.9720166155270168, |
| "step": 3884, |
| "total_flos": 1.6967093886910464e+17, |
| "train_loss": 0.538786339802796, |
| "train_runtime": 81210.4656, |
| "train_samples_per_second": 0.765, |
| "train_steps_per_second": 0.048 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 3884, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": false, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.6967093886910464e+17, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
