| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.988679245283019, |
| "eval_steps": 500, |
| "global_step": 396, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.01, |
| "grad_norm": 8.418300067215691, |
| "learning_rate": 1.25e-08, |
| "logps/chosen": -39.02219009399414, |
| "logps/rejected": -45.12399673461914, |
| "loss": 0.6931, |
| "losses/dpo": 0.6931471824645996, |
| "losses/sft": 1.552122950553894, |
| "losses/total": 0.6931471824645996, |
| "ref_logps/chosen": -39.02219009399414, |
| "ref_logps/rejected": -45.12399673461914, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.02, |
| "grad_norm": 7.6721075942113535, |
| "learning_rate": 2.5e-08, |
| "logps/chosen": -37.21428680419922, |
| "logps/rejected": -44.4819221496582, |
| "loss": 0.6931, |
| "losses/dpo": 0.6931471824645996, |
| "losses/sft": 1.6663763523101807, |
| "losses/total": 0.6931471824645996, |
| "ref_logps/chosen": -37.21428680419922, |
| "ref_logps/rejected": -44.4819221496582, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.02, |
| "grad_norm": 7.948340327346078, |
| "learning_rate": 3.75e-08, |
| "logps/chosen": -41.46142578125, |
| "logps/rejected": -52.18663024902344, |
| "loss": 0.6926, |
| "losses/dpo": 0.6867616176605225, |
| "losses/sft": 1.7890703678131104, |
| "losses/total": 0.6867616176605225, |
| "ref_logps/chosen": -41.46522903442383, |
| "ref_logps/rejected": -52.1768798828125, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": 0.0003804098814725876, |
| "rewards/margins": 0.0013556077610701323, |
| "rewards/rejected": -0.0009751979378052056, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.03, |
| "grad_norm": 8.038993728778431, |
| "learning_rate": 5e-08, |
| "logps/chosen": -39.45478057861328, |
| "logps/rejected": -45.85334014892578, |
| "loss": 0.6936, |
| "losses/dpo": 0.6930198073387146, |
| "losses/sft": 1.6549196243286133, |
| "losses/total": 0.6930198073387146, |
| "ref_logps/chosen": -39.42698287963867, |
| "ref_logps/rejected": -45.83390426635742, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": -0.002779680071398616, |
| "rewards/margins": -0.0008358716731891036, |
| "rewards/rejected": -0.001943808514624834, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.04, |
| "grad_norm": 7.749366133023054, |
| "learning_rate": 6.25e-08, |
| "logps/chosen": -44.29286193847656, |
| "logps/rejected": -51.08875274658203, |
| "loss": 0.695, |
| "losses/dpo": 0.6971508264541626, |
| "losses/sft": 1.3629276752471924, |
| "losses/total": 0.6971508264541626, |
| "ref_logps/chosen": -44.301361083984375, |
| "ref_logps/rejected": -51.132286071777344, |
| "rewards/accuracies": 0.4453125, |
| "rewards/chosen": 0.0008498989045619965, |
| "rewards/margins": -0.003503247397020459, |
| "rewards/rejected": 0.004353146068751812, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.05, |
| "grad_norm": 7.545034174633802, |
| "learning_rate": 7.5e-08, |
| "logps/chosen": -37.192138671875, |
| "logps/rejected": -44.56536865234375, |
| "loss": 0.693, |
| "losses/dpo": 0.6901252269744873, |
| "losses/sft": 1.235260248184204, |
| "losses/total": 0.6901252269744873, |
| "ref_logps/chosen": -37.197486877441406, |
| "ref_logps/rejected": -44.56662368774414, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.000535178929567337, |
| "rewards/margins": 0.000409391475841403, |
| "rewards/rejected": 0.00012578748282976449, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.05, |
| "grad_norm": 7.645026557219469, |
| "learning_rate": 8.75e-08, |
| "logps/chosen": -40.067909240722656, |
| "logps/rejected": -46.251487731933594, |
| "loss": 0.6941, |
| "losses/dpo": 0.695063054561615, |
| "losses/sft": 1.8211989402770996, |
| "losses/total": 0.695063054561615, |
| "ref_logps/chosen": -40.05988311767578, |
| "ref_logps/rejected": -46.26015090942383, |
| "rewards/accuracies": 0.453125, |
| "rewards/chosen": -0.0008021063404157758, |
| "rewards/margins": -0.001668928423896432, |
| "rewards/rejected": 0.000866822199895978, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.06, |
| "grad_norm": 8.091132576285203, |
| "learning_rate": 1e-07, |
| "logps/chosen": -44.73344802856445, |
| "logps/rejected": -46.818824768066406, |
| "loss": 0.6949, |
| "losses/dpo": 0.6943204402923584, |
| "losses/sft": 1.470657229423523, |
| "losses/total": 0.6943204402923584, |
| "ref_logps/chosen": -44.7131233215332, |
| "ref_logps/rejected": -46.83186340332031, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.0020323917269706726, |
| "rewards/margins": -0.0033366940915584564, |
| "rewards/rejected": 0.0013043024810031056, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.07, |
| "grad_norm": 7.987567478058529, |
| "learning_rate": 1.125e-07, |
| "logps/chosen": -40.069969177246094, |
| "logps/rejected": -50.12396240234375, |
| "loss": 0.6944, |
| "losses/dpo": 0.6880265474319458, |
| "losses/sft": 1.1660749912261963, |
| "losses/total": 0.6880265474319458, |
| "ref_logps/chosen": -40.05424118041992, |
| "ref_logps/rejected": -50.13201904296875, |
| "rewards/accuracies": 0.4765625, |
| "rewards/chosen": -0.0015729822916910052, |
| "rewards/margins": -0.002378995530307293, |
| "rewards/rejected": 0.000806013063993305, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 7.458190702935405, |
| "learning_rate": 1.25e-07, |
| "logps/chosen": -36.18950271606445, |
| "logps/rejected": -45.11402130126953, |
| "loss": 0.694, |
| "losses/dpo": 0.6928867697715759, |
| "losses/sft": 1.605255365371704, |
| "losses/total": 0.6928867697715759, |
| "ref_logps/chosen": -36.18606185913086, |
| "ref_logps/rejected": -45.12499237060547, |
| "rewards/accuracies": 0.4765625, |
| "rewards/chosen": -0.00034411592059768736, |
| "rewards/margins": -0.0014411872252821922, |
| "rewards/rejected": 0.0010970717994496226, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 7.498431737676318, |
| "learning_rate": 1.375e-07, |
| "logps/chosen": -38.895912170410156, |
| "logps/rejected": -44.2772216796875, |
| "loss": 0.6922, |
| "losses/dpo": 0.6932664513587952, |
| "losses/sft": 1.4097728729248047, |
| "losses/total": 0.6932664513587952, |
| "ref_logps/chosen": -38.896759033203125, |
| "ref_logps/rejected": -44.25825881958008, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 8.490856271237135e-05, |
| "rewards/margins": 0.0019812812097370625, |
| "rewards/rejected": -0.001896372647024691, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.09, |
| "grad_norm": 7.699329532824058, |
| "learning_rate": 1.5e-07, |
| "logps/chosen": -41.140281677246094, |
| "logps/rejected": -45.357364654541016, |
| "loss": 0.6914, |
| "losses/dpo": 0.6933699250221252, |
| "losses/sft": 1.6783134937286377, |
| "losses/total": 0.6933699250221252, |
| "ref_logps/chosen": -41.16625213623047, |
| "ref_logps/rejected": -45.34600830078125, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.002597447484731674, |
| "rewards/margins": 0.0037334603257477283, |
| "rewards/rejected": -0.0011360126081854105, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.1, |
| "grad_norm": 7.69326977444187, |
| "learning_rate": 1.625e-07, |
| "logps/chosen": -40.31540298461914, |
| "logps/rejected": -50.180397033691406, |
| "loss": 0.6925, |
| "losses/dpo": 0.6910371780395508, |
| "losses/sft": 1.366438865661621, |
| "losses/total": 0.6910371780395508, |
| "ref_logps/chosen": -40.30924987792969, |
| "ref_logps/rejected": -50.16073989868164, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": -0.0006153000867925584, |
| "rewards/margins": 0.0013502361252903938, |
| "rewards/rejected": -0.0019655367359519005, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.11, |
| "grad_norm": 8.114372207024019, |
| "learning_rate": 1.75e-07, |
| "logps/chosen": -37.29108428955078, |
| "logps/rejected": -44.525848388671875, |
| "loss": 0.6911, |
| "losses/dpo": 0.6899442076683044, |
| "losses/sft": 1.4768216609954834, |
| "losses/total": 0.6899442076683044, |
| "ref_logps/chosen": -37.311187744140625, |
| "ref_logps/rejected": -44.50410079956055, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": 0.0020105522125959396, |
| "rewards/margins": 0.004185608588159084, |
| "rewards/rejected": -0.0021750556770712137, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.11, |
| "grad_norm": 7.835230950103234, |
| "learning_rate": 1.875e-07, |
| "logps/chosen": -38.33734893798828, |
| "logps/rejected": -43.93443298339844, |
| "loss": 0.6932, |
| "losses/dpo": 0.6937546133995056, |
| "losses/sft": 1.4263617992401123, |
| "losses/total": 0.6937546133995056, |
| "ref_logps/chosen": -38.31421661376953, |
| "ref_logps/rejected": -43.90996170043945, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.0023131906054913998, |
| "rewards/margins": 0.00013422727352008224, |
| "rewards/rejected": -0.002447417937219143, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 7.58949645380321, |
| "learning_rate": 2e-07, |
| "logps/chosen": -40.26841735839844, |
| "logps/rejected": -43.40159225463867, |
| "loss": 0.6934, |
| "losses/dpo": 0.7032474279403687, |
| "losses/sft": 1.5701673030853271, |
| "losses/total": 0.7032474279403687, |
| "ref_logps/chosen": -40.24923324584961, |
| "ref_logps/rejected": -43.38625717163086, |
| "rewards/accuracies": 0.4921875, |
| "rewards/chosen": -0.0019177356734871864, |
| "rewards/margins": -0.0003846373874694109, |
| "rewards/rejected": -0.0015330985188484192, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.13, |
| "grad_norm": 7.829467009021115, |
| "learning_rate": 2.1249999999999998e-07, |
| "logps/chosen": -41.63703536987305, |
| "logps/rejected": -46.70919418334961, |
| "loss": 0.6952, |
| "losses/dpo": 0.6850873231887817, |
| "losses/sft": 1.4479947090148926, |
| "losses/total": 0.6850873231887817, |
| "ref_logps/chosen": -41.58295822143555, |
| "ref_logps/rejected": -46.69389343261719, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.005406979937106371, |
| "rewards/margins": -0.003876863745972514, |
| "rewards/rejected": -0.0015301161911338568, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.14, |
| "grad_norm": 7.485609028305383, |
| "learning_rate": 2.25e-07, |
| "logps/chosen": -40.4469108581543, |
| "logps/rejected": -44.94635009765625, |
| "loss": 0.6927, |
| "losses/dpo": 0.6907854080200195, |
| "losses/sft": 1.1833800077438354, |
| "losses/total": 0.6907854080200195, |
| "ref_logps/chosen": -40.407257080078125, |
| "ref_logps/rejected": -44.894813537597656, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.003965577110648155, |
| "rewards/margins": 0.0011882353574037552, |
| "rewards/rejected": -0.005153812933713198, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.14, |
| "grad_norm": 7.404661140565325, |
| "learning_rate": 2.3749999999999998e-07, |
| "logps/chosen": -35.739524841308594, |
| "logps/rejected": -46.330265045166016, |
| "loss": 0.6923, |
| "losses/dpo": 0.6935074329376221, |
| "losses/sft": 1.8608835935592651, |
| "losses/total": 0.6935074329376221, |
| "ref_logps/chosen": -35.708274841308594, |
| "ref_logps/rejected": -46.27949523925781, |
| "rewards/accuracies": 0.5078125, |
| "rewards/chosen": -0.003125070594251156, |
| "rewards/margins": 0.001951692276634276, |
| "rewards/rejected": -0.00507676275447011, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.15, |
| "grad_norm": 7.514252016049334, |
| "learning_rate": 2.5e-07, |
| "logps/chosen": -40.884029388427734, |
| "logps/rejected": -47.1005859375, |
| "loss": 0.6939, |
| "losses/dpo": 0.6919451355934143, |
| "losses/sft": 1.3290549516677856, |
| "losses/total": 0.6919451355934143, |
| "ref_logps/chosen": -40.82601547241211, |
| "ref_logps/rejected": -47.054588317871094, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": -0.0058018057607114315, |
| "rewards/margins": -0.001202343963086605, |
| "rewards/rejected": -0.004599461797624826, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 11.774765376903847, |
| "learning_rate": 2.625e-07, |
| "logps/chosen": -40.38345718383789, |
| "logps/rejected": -51.66474914550781, |
| "loss": 0.6922, |
| "losses/dpo": 0.6949824690818787, |
| "losses/sft": 1.548210620880127, |
| "losses/total": 0.6949824690818787, |
| "ref_logps/chosen": -40.31751251220703, |
| "ref_logps/rejected": -51.57673645019531, |
| "rewards/accuracies": 0.515625, |
| "rewards/chosen": -0.006594239268451929, |
| "rewards/margins": 0.002207120880484581, |
| "rewards/rejected": -0.008801360614597797, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.17, |
| "grad_norm": 7.542951860866026, |
| "learning_rate": 2.75e-07, |
| "logps/chosen": -38.18585205078125, |
| "logps/rejected": -47.13993835449219, |
| "loss": 0.6924, |
| "losses/dpo": 0.6899946928024292, |
| "losses/sft": 1.1579391956329346, |
| "losses/total": 0.6899946928024292, |
| "ref_logps/chosen": -38.091922760009766, |
| "ref_logps/rejected": -47.029541015625, |
| "rewards/accuracies": 0.4765625, |
| "rewards/chosen": -0.009392979554831982, |
| "rewards/margins": 0.0016468917019665241, |
| "rewards/rejected": -0.011039872653782368, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.17, |
| "grad_norm": 7.876385596657873, |
| "learning_rate": 2.8749999999999995e-07, |
| "logps/chosen": -38.30439376831055, |
| "logps/rejected": -49.61843490600586, |
| "loss": 0.6895, |
| "losses/dpo": 0.6851339936256409, |
| "losses/sft": 1.5843318700790405, |
| "losses/total": 0.6851339936256409, |
| "ref_logps/chosen": -38.242801666259766, |
| "ref_logps/rejected": -49.4824104309082, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.006159077398478985, |
| "rewards/margins": 0.007443387992680073, |
| "rewards/rejected": -0.013602466322481632, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.18, |
| "grad_norm": 7.668314504955077, |
| "learning_rate": 3e-07, |
| "logps/chosen": -40.48552322387695, |
| "logps/rejected": -46.48503494262695, |
| "loss": 0.6895, |
| "losses/dpo": 0.6920894980430603, |
| "losses/sft": 1.4189947843551636, |
| "losses/total": 0.6920894980430603, |
| "ref_logps/chosen": -40.395687103271484, |
| "ref_logps/rejected": -46.31945037841797, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.008983338251709938, |
| "rewards/margins": 0.007574939634650946, |
| "rewards/rejected": -0.01655827835202217, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.19, |
| "grad_norm": 7.344435357199023, |
| "learning_rate": 3.1249999999999997e-07, |
| "logps/chosen": -39.81501388549805, |
| "logps/rejected": -45.954071044921875, |
| "loss": 0.6914, |
| "losses/dpo": 0.6927582621574402, |
| "losses/sft": 1.6129871606826782, |
| "losses/total": 0.6927582621574402, |
| "ref_logps/chosen": -39.700050354003906, |
| "ref_logps/rejected": -45.80275344848633, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.011496355757117271, |
| "rewards/margins": 0.0036351331509649754, |
| "rewards/rejected": -0.01513148844242096, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 7.977738556938539, |
| "learning_rate": 3.25e-07, |
| "logps/chosen": -39.41866683959961, |
| "logps/rejected": -47.94341278076172, |
| "loss": 0.6919, |
| "losses/dpo": 0.6823984384536743, |
| "losses/sft": 1.1218098402023315, |
| "losses/total": 0.6823984384536743, |
| "ref_logps/chosen": -39.244632720947266, |
| "ref_logps/rejected": -47.742191314697266, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.017403149977326393, |
| "rewards/margins": 0.002718748524785042, |
| "rewards/rejected": -0.020121898502111435, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 7.137045291395353, |
| "learning_rate": 3.375e-07, |
| "logps/chosen": -36.09528350830078, |
| "logps/rejected": -43.786441802978516, |
| "loss": 0.6934, |
| "losses/dpo": 0.7064226269721985, |
| "losses/sft": 1.185333251953125, |
| "losses/total": 0.7064226269721985, |
| "ref_logps/chosen": -35.91387939453125, |
| "ref_logps/rejected": -43.607757568359375, |
| "rewards/accuracies": 0.5390625, |
| "rewards/chosen": -0.01814102753996849, |
| "rewards/margins": -0.0002719040203373879, |
| "rewards/rejected": -0.017869124189019203, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.21, |
| "grad_norm": 7.379899822037054, |
| "learning_rate": 3.5e-07, |
| "logps/chosen": -44.72355270385742, |
| "logps/rejected": -47.34676742553711, |
| "loss": 0.6909, |
| "losses/dpo": 0.6887847185134888, |
| "losses/sft": 1.6178021430969238, |
| "losses/total": 0.6887847185134888, |
| "ref_logps/chosen": -44.496578216552734, |
| "ref_logps/rejected": -47.07260513305664, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.02269744500517845, |
| "rewards/margins": 0.004718274809420109, |
| "rewards/rejected": -0.027415720745921135, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.22, |
| "grad_norm": 8.0485668325091, |
| "learning_rate": 3.6249999999999997e-07, |
| "logps/chosen": -41.700931549072266, |
| "logps/rejected": -50.03131103515625, |
| "loss": 0.6893, |
| "losses/dpo": 0.6823500394821167, |
| "losses/sft": 1.4876271486282349, |
| "losses/total": 0.6823500394821167, |
| "ref_logps/chosen": -41.42716979980469, |
| "ref_logps/rejected": -49.678436279296875, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": -0.027376368641853333, |
| "rewards/margins": 0.007911860011518002, |
| "rewards/rejected": -0.03528822585940361, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.23, |
| "grad_norm": 7.828639871809106, |
| "learning_rate": 3.75e-07, |
| "logps/chosen": -41.79233932495117, |
| "logps/rejected": -48.521629333496094, |
| "loss": 0.6884, |
| "losses/dpo": 0.6861717700958252, |
| "losses/sft": 1.3226033449172974, |
| "losses/total": 0.6861717700958252, |
| "ref_logps/chosen": -41.542991638183594, |
| "ref_logps/rejected": -48.17402648925781, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.024935012683272362, |
| "rewards/margins": 0.009825671091675758, |
| "rewards/rejected": -0.03476068750023842, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.23, |
| "grad_norm": 8.365578824729102, |
| "learning_rate": 3.875e-07, |
| "logps/chosen": -41.142433166503906, |
| "logps/rejected": -48.93161392211914, |
| "loss": 0.6954, |
| "losses/dpo": 0.6871266961097717, |
| "losses/sft": 1.0435420274734497, |
| "losses/total": 0.6871266961097717, |
| "ref_logps/chosen": -40.73477554321289, |
| "ref_logps/rejected": -48.56393814086914, |
| "rewards/accuracies": 0.484375, |
| "rewards/chosen": -0.04076562076807022, |
| "rewards/margins": -0.00399819714948535, |
| "rewards/rejected": -0.03676741570234299, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 7.2403699625390585, |
| "learning_rate": 4e-07, |
| "logps/chosen": -37.81801223754883, |
| "logps/rejected": -46.747371673583984, |
| "loss": 0.6899, |
| "losses/dpo": 0.696927011013031, |
| "losses/sft": 1.7572060823440552, |
| "losses/total": 0.696927011013031, |
| "ref_logps/chosen": -37.4744758605957, |
| "ref_logps/rejected": -46.330135345458984, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.03435356542468071, |
| "rewards/margins": 0.007370149716734886, |
| "rewards/rejected": -0.04172371327877045, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.25, |
| "grad_norm": 7.310694198303384, |
| "learning_rate": 4.1249999999999997e-07, |
| "logps/chosen": -35.633243560791016, |
| "logps/rejected": -41.00613021850586, |
| "loss": 0.6852, |
| "losses/dpo": 0.6834661960601807, |
| "losses/sft": 1.3767448663711548, |
| "losses/total": 0.6834661960601807, |
| "ref_logps/chosen": -35.318153381347656, |
| "ref_logps/rejected": -40.5264892578125, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.03150848299264908, |
| "rewards/margins": 0.016455503180623055, |
| "rewards/rejected": -0.04796398803591728, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.26, |
| "grad_norm": 7.510894622614614, |
| "learning_rate": 4.2499999999999995e-07, |
| "logps/chosen": -42.22370529174805, |
| "logps/rejected": -48.45228576660156, |
| "loss": 0.688, |
| "losses/dpo": 0.6784626841545105, |
| "losses/sft": 1.7890020608901978, |
| "losses/total": 0.6784626841545105, |
| "ref_logps/chosen": -41.76036834716797, |
| "ref_logps/rejected": -47.876914978027344, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.046334076672792435, |
| "rewards/margins": 0.011203275993466377, |
| "rewards/rejected": -0.05753735080361366, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.26, |
| "grad_norm": 7.524260365013066, |
| "learning_rate": 4.375e-07, |
| "logps/chosen": -41.117164611816406, |
| "logps/rejected": -47.30539321899414, |
| "loss": 0.6874, |
| "losses/dpo": 0.6890352368354797, |
| "losses/sft": 1.9127196073532104, |
| "losses/total": 0.6890352368354797, |
| "ref_logps/chosen": -40.611358642578125, |
| "ref_logps/rejected": -46.675148010253906, |
| "rewards/accuracies": 0.6171875, |
| "rewards/chosen": -0.0505804568529129, |
| "rewards/margins": 0.01244389358907938, |
| "rewards/rejected": -0.0630243569612503, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.27, |
| "grad_norm": 7.835502878067996, |
| "learning_rate": 4.5e-07, |
| "logps/chosen": -42.31553649902344, |
| "logps/rejected": -48.77557373046875, |
| "loss": 0.6846, |
| "losses/dpo": 0.6801737546920776, |
| "losses/sft": 1.0419285297393799, |
| "losses/total": 0.6801737546920776, |
| "ref_logps/chosen": -41.76920700073242, |
| "ref_logps/rejected": -48.04461669921875, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.054632995277643204, |
| "rewards/margins": 0.0184622872620821, |
| "rewards/rejected": -0.07309528440237045, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.28, |
| "grad_norm": 10.089436620525504, |
| "learning_rate": 4.625e-07, |
| "logps/chosen": -40.23670196533203, |
| "logps/rejected": -44.72167205810547, |
| "loss": 0.6856, |
| "losses/dpo": 0.6820752620697021, |
| "losses/sft": 1.6919959783554077, |
| "losses/total": 0.6820752620697021, |
| "ref_logps/chosen": -39.70112991333008, |
| "ref_logps/rejected": -44.0257682800293, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.05355698987841606, |
| "rewards/margins": 0.016033286228775978, |
| "rewards/rejected": -0.06959027796983719, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.29, |
| "grad_norm": 7.4705752998877974, |
| "learning_rate": 4.7499999999999995e-07, |
| "logps/chosen": -40.608951568603516, |
| "logps/rejected": -46.77935791015625, |
| "loss": 0.6846, |
| "losses/dpo": 0.7083909511566162, |
| "losses/sft": 1.3596407175064087, |
| "losses/total": 0.7083909511566162, |
| "ref_logps/chosen": -40.02342987060547, |
| "ref_logps/rejected": -46.008670806884766, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.05855226144194603, |
| "rewards/margins": 0.018516112118959427, |
| "rewards/rejected": -0.07706836611032486, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.29, |
| "grad_norm": 7.4834962101051445, |
| "learning_rate": 4.875e-07, |
| "logps/chosen": -38.924591064453125, |
| "logps/rejected": -44.15880584716797, |
| "loss": 0.6806, |
| "losses/dpo": 0.6712931394577026, |
| "losses/sft": 1.4741321802139282, |
| "losses/total": 0.6712931394577026, |
| "ref_logps/chosen": -38.32358932495117, |
| "ref_logps/rejected": -43.294708251953125, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.060100097209215164, |
| "rewards/margins": 0.02630985900759697, |
| "rewards/rejected": -0.08640995621681213, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.3, |
| "grad_norm": 7.593289226892247, |
| "learning_rate": 5e-07, |
| "logps/chosen": -36.14179611206055, |
| "logps/rejected": -43.69697952270508, |
| "loss": 0.6893, |
| "losses/dpo": 0.6905455589294434, |
| "losses/sft": 1.8340303897857666, |
| "losses/total": 0.6905455589294434, |
| "ref_logps/chosen": -35.46891784667969, |
| "ref_logps/rejected": -42.935638427734375, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.06728792935609818, |
| "rewards/margins": 0.008846651762723923, |
| "rewards/rejected": -0.0761345773935318, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.31, |
| "grad_norm": 7.184019422431451, |
| "learning_rate": 4.985955056179775e-07, |
| "logps/chosen": -36.04156494140625, |
| "logps/rejected": -44.501773834228516, |
| "loss": 0.6765, |
| "losses/dpo": 0.6632527112960815, |
| "losses/sft": 1.562534213066101, |
| "losses/total": 0.6632527112960815, |
| "ref_logps/chosen": -35.38131332397461, |
| "ref_logps/rejected": -43.47880935668945, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.06602565199136734, |
| "rewards/margins": 0.036271147429943085, |
| "rewards/rejected": -0.10229679197072983, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 7.817004450242549, |
| "learning_rate": 4.97191011235955e-07, |
| "logps/chosen": -40.78254699707031, |
| "logps/rejected": -48.181861877441406, |
| "loss": 0.6803, |
| "losses/dpo": 0.7099467515945435, |
| "losses/sft": 1.8783167600631714, |
| "losses/total": 0.7099467515945435, |
| "ref_logps/chosen": -40.004154205322266, |
| "ref_logps/rejected": -47.115760803222656, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.07783940434455872, |
| "rewards/margins": 0.028770849108695984, |
| "rewards/rejected": -0.1066102534532547, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 7.433654746121009, |
| "learning_rate": 4.957865168539325e-07, |
| "logps/chosen": -40.71480941772461, |
| "logps/rejected": -47.88724136352539, |
| "loss": 0.6803, |
| "losses/dpo": 0.7072566151618958, |
| "losses/sft": 1.6432607173919678, |
| "losses/total": 0.7072566151618958, |
| "ref_logps/chosen": -39.739933013916016, |
| "ref_logps/rejected": -46.61668014526367, |
| "rewards/accuracies": 0.6171875, |
| "rewards/chosen": -0.09748795628547668, |
| "rewards/margins": 0.029567349702119827, |
| "rewards/rejected": -0.1270553022623062, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.33, |
| "grad_norm": 7.64577773243097, |
| "learning_rate": 4.943820224719101e-07, |
| "logps/chosen": -36.590328216552734, |
| "logps/rejected": -45.61329650878906, |
| "loss": 0.6824, |
| "losses/dpo": 0.6843121647834778, |
| "losses/sft": 2.019310235977173, |
| "losses/total": 0.6843121647834778, |
| "ref_logps/chosen": -35.54472732543945, |
| "ref_logps/rejected": -44.32211685180664, |
| "rewards/accuracies": 0.6171875, |
| "rewards/chosen": -0.10456002503633499, |
| "rewards/margins": 0.024558255448937416, |
| "rewards/rejected": -0.12911829352378845, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.34, |
| "grad_norm": 7.451674659941506, |
| "learning_rate": 4.929775280898877e-07, |
| "logps/chosen": -39.82986068725586, |
| "logps/rejected": -44.68933868408203, |
| "loss": 0.6767, |
| "losses/dpo": 0.6984357833862305, |
| "losses/sft": 1.321048617362976, |
| "losses/total": 0.6984357833862305, |
| "ref_logps/chosen": -38.77911376953125, |
| "ref_logps/rejected": -43.251163482666016, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.10507487505674362, |
| "rewards/margins": 0.038742441684007645, |
| "rewards/rejected": -0.14381732046604156, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.35, |
| "grad_norm": 7.481580488082584, |
| "learning_rate": 4.915730337078651e-07, |
| "logps/chosen": -40.547637939453125, |
| "logps/rejected": -48.37934112548828, |
| "loss": 0.6784, |
| "losses/dpo": 0.682788610458374, |
| "losses/sft": 1.305440902709961, |
| "losses/total": 0.682788610458374, |
| "ref_logps/chosen": -39.39177703857422, |
| "ref_logps/rejected": -46.888671875, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.11558566987514496, |
| "rewards/margins": 0.03348149359226227, |
| "rewards/rejected": -0.14906716346740723, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.35, |
| "grad_norm": 7.503690704863454, |
| "learning_rate": 4.901685393258427e-07, |
| "logps/chosen": -43.58844757080078, |
| "logps/rejected": -46.27735137939453, |
| "loss": 0.6796, |
| "losses/dpo": 0.6725805997848511, |
| "losses/sft": 1.7845996618270874, |
| "losses/total": 0.6725805997848511, |
| "ref_logps/chosen": -42.20465850830078, |
| "ref_logps/rejected": -44.57255554199219, |
| "rewards/accuracies": 0.5546875, |
| "rewards/chosen": -0.13837924599647522, |
| "rewards/margins": 0.03210053965449333, |
| "rewards/rejected": -0.17047978937625885, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 7.6005277154002275, |
| "learning_rate": 4.887640449438202e-07, |
| "logps/chosen": -40.19657897949219, |
| "logps/rejected": -46.2965087890625, |
| "loss": 0.6806, |
| "losses/dpo": 0.6915363669395447, |
| "losses/sft": 1.4134502410888672, |
| "losses/total": 0.6915363669395447, |
| "ref_logps/chosen": -38.93235778808594, |
| "ref_logps/rejected": -44.731178283691406, |
| "rewards/accuracies": 0.546875, |
| "rewards/chosen": -0.12642225623130798, |
| "rewards/margins": 0.030110429972410202, |
| "rewards/rejected": -0.15653270483016968, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.37, |
| "grad_norm": 7.342034157176263, |
| "learning_rate": 4.873595505617978e-07, |
| "logps/chosen": -35.246055603027344, |
| "logps/rejected": -45.013092041015625, |
| "loss": 0.671, |
| "losses/dpo": 0.6273987293243408, |
| "losses/sft": 1.2200208902359009, |
| "losses/total": 0.6273987293243408, |
| "ref_logps/chosen": -33.989261627197266, |
| "ref_logps/rejected": -43.26504898071289, |
| "rewards/accuracies": 0.6171875, |
| "rewards/chosen": -0.12567944824695587, |
| "rewards/margins": 0.04912441223859787, |
| "rewards/rejected": -0.17480388283729553, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.38, |
| "grad_norm": 7.768856876362336, |
| "learning_rate": 4.859550561797752e-07, |
| "logps/chosen": -41.647483825683594, |
| "logps/rejected": -49.59557342529297, |
| "loss": 0.6668, |
| "losses/dpo": 0.6410457491874695, |
| "losses/sft": 2.0004844665527344, |
| "losses/total": 0.6410457491874695, |
| "ref_logps/chosen": -40.150108337402344, |
| "ref_logps/rejected": -47.4974365234375, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.1497381180524826, |
| "rewards/margins": 0.06007564440369606, |
| "rewards/rejected": -0.20981375873088837, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.38, |
| "grad_norm": 7.482808682633612, |
| "learning_rate": 4.845505617977528e-07, |
| "logps/chosen": -42.28547668457031, |
| "logps/rejected": -46.57417297363281, |
| "loss": 0.6686, |
| "losses/dpo": 0.6821735501289368, |
| "losses/sft": 1.643945336341858, |
| "losses/total": 0.6821735501289368, |
| "ref_logps/chosen": -40.781097412109375, |
| "ref_logps/rejected": -44.47871017456055, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.15043821930885315, |
| "rewards/margins": 0.05910744518041611, |
| "rewards/rejected": -0.20954564213752747, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.39, |
| "grad_norm": 7.444552223013138, |
| "learning_rate": 4.831460674157303e-07, |
| "logps/chosen": -38.653770446777344, |
| "logps/rejected": -47.96025848388672, |
| "loss": 0.6696, |
| "losses/dpo": 0.653758704662323, |
| "losses/sft": 1.9075889587402344, |
| "losses/total": 0.653758704662323, |
| "ref_logps/chosen": -37.04439163208008, |
| "ref_logps/rejected": -45.76567840576172, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.16093730926513672, |
| "rewards/margins": 0.05852021649479866, |
| "rewards/rejected": -0.2194575071334839, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 7.051622049892774, |
| "learning_rate": 4.817415730337078e-07, |
| "logps/chosen": -36.511940002441406, |
| "logps/rejected": -42.634193420410156, |
| "loss": 0.672, |
| "losses/dpo": 0.6566299200057983, |
| "losses/sft": 1.6063774824142456, |
| "losses/total": 0.6566299200057983, |
| "ref_logps/chosen": -34.99435806274414, |
| "ref_logps/rejected": -40.612342834472656, |
| "rewards/accuracies": 0.6171875, |
| "rewards/chosen": -0.15175840258598328, |
| "rewards/margins": 0.05042674392461777, |
| "rewards/rejected": -0.20218515396118164, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.41, |
| "grad_norm": 7.752968590362967, |
| "learning_rate": 4.803370786516854e-07, |
| "logps/chosen": -43.374481201171875, |
| "logps/rejected": -46.1808967590332, |
| "loss": 0.664, |
| "losses/dpo": 0.5915548801422119, |
| "losses/sft": 1.5764846801757812, |
| "losses/total": 0.5915548801422119, |
| "ref_logps/chosen": -41.619239807128906, |
| "ref_logps/rejected": -43.757720947265625, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.17552456259727478, |
| "rewards/margins": 0.06679282337427139, |
| "rewards/rejected": -0.24231737852096558, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.42, |
| "grad_norm": 7.375135165111918, |
| "learning_rate": 4.789325842696629e-07, |
| "logps/chosen": -40.775726318359375, |
| "logps/rejected": -45.556365966796875, |
| "loss": 0.6783, |
| "losses/dpo": 0.6768916845321655, |
| "losses/sft": 1.3732706308364868, |
| "losses/total": 0.6768916845321655, |
| "ref_logps/chosen": -39.01034927368164, |
| "ref_logps/rejected": -43.436553955078125, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.17653760313987732, |
| "rewards/margins": 0.03544352203607559, |
| "rewards/rejected": -0.21198111772537231, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.42, |
| "grad_norm": 7.238231049231885, |
| "learning_rate": 4.775280898876405e-07, |
| "logps/chosen": -39.11316680908203, |
| "logps/rejected": -45.04530334472656, |
| "loss": 0.6642, |
| "losses/dpo": 0.6528148651123047, |
| "losses/sft": 1.159528136253357, |
| "losses/total": 0.6528148651123047, |
| "ref_logps/chosen": -37.2640266418457, |
| "ref_logps/rejected": -42.495338439941406, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.18491369485855103, |
| "rewards/margins": 0.07008323073387146, |
| "rewards/rejected": -0.2549969553947449, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.43, |
| "grad_norm": 7.652181625277677, |
| "learning_rate": 4.7612359550561797e-07, |
| "logps/chosen": -43.244380950927734, |
| "logps/rejected": -49.415409088134766, |
| "loss": 0.6679, |
| "losses/dpo": 0.6176864504814148, |
| "losses/sft": 1.796196460723877, |
| "losses/total": 0.6176864504814148, |
| "ref_logps/chosen": -41.167381286621094, |
| "ref_logps/rejected": -46.70917892456055, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.20770025253295898, |
| "rewards/margins": 0.06292243301868439, |
| "rewards/rejected": -0.2706226706504822, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.44, |
| "grad_norm": 8.012193852457372, |
| "learning_rate": 4.747191011235955e-07, |
| "logps/chosen": -38.95054626464844, |
| "logps/rejected": -45.45573043823242, |
| "loss": 0.6545, |
| "losses/dpo": 0.721019983291626, |
| "losses/sft": 1.6278411149978638, |
| "losses/total": 0.721019983291626, |
| "ref_logps/chosen": -37.17967987060547, |
| "ref_logps/rejected": -42.77497100830078, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.17708644270896912, |
| "rewards/margins": 0.09098967909812927, |
| "rewards/rejected": -0.2680761218070984, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.45, |
| "grad_norm": 7.745152630727936, |
| "learning_rate": 4.7331460674157303e-07, |
| "logps/chosen": -41.84577560424805, |
| "logps/rejected": -54.23434066772461, |
| "loss": 0.654, |
| "losses/dpo": 0.601816713809967, |
| "losses/sft": 1.5886242389678955, |
| "losses/total": 0.601816713809967, |
| "ref_logps/chosen": -39.931434631347656, |
| "ref_logps/rejected": -51.34343719482422, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.19143418967723846, |
| "rewards/margins": 0.09765592962503433, |
| "rewards/rejected": -0.2890901267528534, |
| "step": 59 |
| }, |
| { |
| "epoch": 0.45, |
| "grad_norm": 8.104292528879256, |
| "learning_rate": 4.7191011235955054e-07, |
| "logps/chosen": -40.5402717590332, |
| "logps/rejected": -48.11115264892578, |
| "loss": 0.6612, |
| "losses/dpo": 0.6307883858680725, |
| "losses/sft": 1.6475903987884521, |
| "losses/total": 0.6307883858680725, |
| "ref_logps/chosen": -38.59111022949219, |
| "ref_logps/rejected": -45.39031219482422, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.19491644203662872, |
| "rewards/margins": 0.07716768234968185, |
| "rewards/rejected": -0.27208411693573, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.46, |
| "grad_norm": 8.24076971848322, |
| "learning_rate": 4.705056179775281e-07, |
| "logps/chosen": -41.55039978027344, |
| "logps/rejected": -51.43959426879883, |
| "loss": 0.6677, |
| "losses/dpo": 0.7012457251548767, |
| "losses/sft": 2.175475597381592, |
| "losses/total": 0.7012457251548767, |
| "ref_logps/chosen": -39.29528045654297, |
| "ref_logps/rejected": -48.50824737548828, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.22551202774047852, |
| "rewards/margins": 0.06762254983186722, |
| "rewards/rejected": -0.2931345999240875, |
| "step": 61 |
| }, |
| { |
| "epoch": 0.47, |
| "grad_norm": 8.35481489409821, |
| "learning_rate": 4.691011235955056e-07, |
| "logps/chosen": -43.0926399230957, |
| "logps/rejected": -47.44728088378906, |
| "loss": 0.6462, |
| "losses/dpo": 0.6053961515426636, |
| "losses/sft": 1.459052324295044, |
| "losses/total": 0.6053961515426636, |
| "ref_logps/chosen": -40.96197509765625, |
| "ref_logps/rejected": -44.2166633605957, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.2130661904811859, |
| "rewards/margins": 0.1099955290555954, |
| "rewards/rejected": -0.3230617344379425, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 7.797774561384425, |
| "learning_rate": 4.6769662921348315e-07, |
| "logps/chosen": -38.055320739746094, |
| "logps/rejected": -47.66813659667969, |
| "loss": 0.652, |
| "losses/dpo": 0.632691502571106, |
| "losses/sft": 1.4375559091567993, |
| "losses/total": 0.632691502571106, |
| "ref_logps/chosen": -36.05341720581055, |
| "ref_logps/rejected": -44.64914321899414, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.20018979907035828, |
| "rewards/margins": 0.10170910507440567, |
| "rewards/rejected": -0.30189892649650574, |
| "step": 63 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 7.28638958987082, |
| "learning_rate": 4.662921348314606e-07, |
| "logps/chosen": -39.81006622314453, |
| "logps/rejected": -46.78810501098633, |
| "loss": 0.662, |
| "losses/dpo": 0.6003807783126831, |
| "losses/sft": 1.3374682664871216, |
| "losses/total": 0.6003807783126831, |
| "ref_logps/chosen": -37.332698822021484, |
| "ref_logps/rejected": -43.507659912109375, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.247736856341362, |
| "rewards/margins": 0.0803074836730957, |
| "rewards/rejected": -0.3280443251132965, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.49, |
| "grad_norm": 7.798097069372596, |
| "learning_rate": 4.6488764044943816e-07, |
| "logps/chosen": -45.76461410522461, |
| "logps/rejected": -50.199825286865234, |
| "loss": 0.6814, |
| "losses/dpo": 0.6105685234069824, |
| "losses/sft": 1.8878819942474365, |
| "losses/total": 0.6105685234069824, |
| "ref_logps/chosen": -42.697021484375, |
| "ref_logps/rejected": -46.67133331298828, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.30675944685935974, |
| "rewards/margins": 0.046089351177215576, |
| "rewards/rejected": -0.35284876823425293, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.5, |
| "grad_norm": 7.825603469401775, |
| "learning_rate": 4.634831460674157e-07, |
| "logps/chosen": -43.35639953613281, |
| "logps/rejected": -53.741355895996094, |
| "loss": 0.6436, |
| "losses/dpo": 0.6293699741363525, |
| "losses/sft": 1.4026882648468018, |
| "losses/total": 0.6293699741363525, |
| "ref_logps/chosen": -40.74187469482422, |
| "ref_logps/rejected": -49.880897521972656, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.2614526152610779, |
| "rewards/margins": 0.12459328025579453, |
| "rewards/rejected": -0.3860458731651306, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.51, |
| "grad_norm": 7.635556989764748, |
| "learning_rate": 4.620786516853932e-07, |
| "logps/chosen": -40.68864440917969, |
| "logps/rejected": -46.91835021972656, |
| "loss": 0.6539, |
| "losses/dpo": 0.7432792782783508, |
| "losses/sft": 1.5986056327819824, |
| "losses/total": 0.7432792782783508, |
| "ref_logps/chosen": -38.21084213256836, |
| "ref_logps/rejected": -43.418067932128906, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.24778038263320923, |
| "rewards/margins": 0.10224790126085281, |
| "rewards/rejected": -0.35002827644348145, |
| "step": 67 |
| }, |
| { |
| "epoch": 0.51, |
| "grad_norm": 7.6861933707333305, |
| "learning_rate": 4.606741573033708e-07, |
| "logps/chosen": -42.49998092651367, |
| "logps/rejected": -49.80910873413086, |
| "loss": 0.6241, |
| "losses/dpo": 0.5949782133102417, |
| "losses/sft": 1.2951277494430542, |
| "losses/total": 0.5949782133102417, |
| "ref_logps/chosen": -40.09019088745117, |
| "ref_logps/rejected": -45.69980239868164, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.24097900092601776, |
| "rewards/margins": 0.16995173692703247, |
| "rewards/rejected": -0.4109307527542114, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.52, |
| "grad_norm": 7.5671933847566555, |
| "learning_rate": 4.592696629213483e-07, |
| "logps/chosen": -42.11725616455078, |
| "logps/rejected": -52.82745361328125, |
| "loss": 0.6438, |
| "losses/dpo": 0.6235805749893188, |
| "losses/sft": 1.4768122434616089, |
| "losses/total": 0.6235805749893188, |
| "ref_logps/chosen": -39.234642028808594, |
| "ref_logps/rejected": -48.657501220703125, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.2882614731788635, |
| "rewards/margins": 0.12873350083827972, |
| "rewards/rejected": -0.41699495911598206, |
| "step": 69 |
| }, |
| { |
| "epoch": 0.53, |
| "grad_norm": 7.380869348497186, |
| "learning_rate": 4.5786516853932584e-07, |
| "logps/chosen": -40.33063507080078, |
| "logps/rejected": -47.34225845336914, |
| "loss": 0.6519, |
| "losses/dpo": 0.6829323768615723, |
| "losses/sft": 1.6434234380722046, |
| "losses/total": 0.6829323768615723, |
| "ref_logps/chosen": -37.294776916503906, |
| "ref_logps/rejected": -43.10985565185547, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.30358612537384033, |
| "rewards/margins": 0.11965445429086685, |
| "rewards/rejected": -0.4232405722141266, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.54, |
| "grad_norm": 7.505680485493994, |
| "learning_rate": 4.5646067415730334e-07, |
| "logps/chosen": -40.74094009399414, |
| "logps/rejected": -49.552616119384766, |
| "loss": 0.6423, |
| "losses/dpo": 0.6549758315086365, |
| "losses/sft": 1.5461335182189941, |
| "losses/total": 0.6549758315086365, |
| "ref_logps/chosen": -37.58620071411133, |
| "ref_logps/rejected": -45.091209411621094, |
| "rewards/accuracies": 0.6328125, |
| "rewards/chosen": -0.3154744505882263, |
| "rewards/margins": 0.1306663304567337, |
| "rewards/rejected": -0.4461407959461212, |
| "step": 71 |
| }, |
| { |
| "epoch": 0.54, |
| "grad_norm": 7.805459051678804, |
| "learning_rate": 4.550561797752809e-07, |
| "logps/chosen": -42.94220733642578, |
| "logps/rejected": -54.635963439941406, |
| "loss": 0.6384, |
| "losses/dpo": 0.589752733707428, |
| "losses/sft": 1.55972421169281, |
| "losses/total": 0.589752733707428, |
| "ref_logps/chosen": -39.39100646972656, |
| "ref_logps/rejected": -49.64848709106445, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.35512006282806396, |
| "rewards/margins": 0.1436270773410797, |
| "rewards/rejected": -0.49874716997146606, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.55, |
| "grad_norm": 8.048352395094511, |
| "learning_rate": 4.536516853932584e-07, |
| "logps/chosen": -41.640235900878906, |
| "logps/rejected": -53.20794677734375, |
| "loss": 0.621, |
| "losses/dpo": 0.6062641143798828, |
| "losses/sft": 1.0970079898834229, |
| "losses/total": 0.6062641143798828, |
| "ref_logps/chosen": -38.45201873779297, |
| "ref_logps/rejected": -48.13432312011719, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.3188212513923645, |
| "rewards/margins": 0.18854106962680817, |
| "rewards/rejected": -0.5073623061180115, |
| "step": 73 |
| }, |
| { |
| "epoch": 0.56, |
| "grad_norm": 12.072882817554818, |
| "learning_rate": 4.522471910112359e-07, |
| "logps/chosen": -43.404823303222656, |
| "logps/rejected": -50.17894744873047, |
| "loss": 0.6527, |
| "losses/dpo": 0.5960186719894409, |
| "losses/sft": 1.444412112236023, |
| "losses/total": 0.5960186719894409, |
| "ref_logps/chosen": -39.74496841430664, |
| "ref_logps/rejected": -45.32371139526367, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.3659852147102356, |
| "rewards/margins": 0.11953801661729813, |
| "rewards/rejected": -0.4855232238769531, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.57, |
| "grad_norm": 7.650328731492573, |
| "learning_rate": 4.5084269662921347e-07, |
| "logps/chosen": -43.5896110534668, |
| "logps/rejected": -51.086971282958984, |
| "loss": 0.6409, |
| "losses/dpo": 0.6057982444763184, |
| "losses/sft": 1.4658453464508057, |
| "losses/total": 0.6057982444763184, |
| "ref_logps/chosen": -39.98381423950195, |
| "ref_logps/rejected": -45.86697769165039, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.3605796992778778, |
| "rewards/margins": 0.16141945123672485, |
| "rewards/rejected": -0.521999180316925, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.57, |
| "grad_norm": 7.945574535694735, |
| "learning_rate": 4.4943820224719097e-07, |
| "logps/chosen": -44.07162094116211, |
| "logps/rejected": -50.66339874267578, |
| "loss": 0.6556, |
| "losses/dpo": 0.6170323491096497, |
| "losses/sft": 1.8739807605743408, |
| "losses/total": 0.6170323491096497, |
| "ref_logps/chosen": -40.06593322753906, |
| "ref_logps/rejected": -45.41100311279297, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.4005686044692993, |
| "rewards/margins": 0.12467078864574432, |
| "rewards/rejected": -0.5252394080162048, |
| "step": 76 |
| }, |
| { |
| "epoch": 0.58, |
| "grad_norm": 9.606133876539282, |
| "learning_rate": 4.4803370786516853e-07, |
| "logps/chosen": -42.20341110229492, |
| "logps/rejected": -50.49468231201172, |
| "loss": 0.6503, |
| "losses/dpo": 0.586646556854248, |
| "losses/sft": 1.5989439487457275, |
| "losses/total": 0.586646556854248, |
| "ref_logps/chosen": -38.3419075012207, |
| "ref_logps/rejected": -45.38350296020508, |
| "rewards/accuracies": 0.6328125, |
| "rewards/chosen": -0.3861507773399353, |
| "rewards/margins": 0.12496703863143921, |
| "rewards/rejected": -0.5111178159713745, |
| "step": 77 |
| }, |
| { |
| "epoch": 0.59, |
| "grad_norm": 8.32429224169226, |
| "learning_rate": 4.4662921348314603e-07, |
| "logps/chosen": -42.94104766845703, |
| "logps/rejected": -54.27581787109375, |
| "loss": 0.6335, |
| "losses/dpo": 0.629318356513977, |
| "losses/sft": 1.5925976037979126, |
| "losses/total": 0.629318356513977, |
| "ref_logps/chosen": -38.90857696533203, |
| "ref_logps/rejected": -48.47710037231445, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.40324676036834717, |
| "rewards/margins": 0.1766246110200882, |
| "rewards/rejected": -0.5798712968826294, |
| "step": 78 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 7.898667752271484, |
| "learning_rate": 4.452247191011236e-07, |
| "logps/chosen": -42.00642395019531, |
| "logps/rejected": -51.58437728881836, |
| "loss": 0.6326, |
| "losses/dpo": 0.6813949346542358, |
| "losses/sft": 1.5958709716796875, |
| "losses/total": 0.6813949346542358, |
| "ref_logps/chosen": -38.2513427734375, |
| "ref_logps/rejected": -46.194374084472656, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.3755083978176117, |
| "rewards/margins": 0.16349197924137115, |
| "rewards/rejected": -0.5390004515647888, |
| "step": 79 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 8.4984309671094, |
| "learning_rate": 4.438202247191011e-07, |
| "logps/chosen": -47.29401397705078, |
| "logps/rejected": -54.29883575439453, |
| "loss": 0.642, |
| "losses/dpo": 0.6930491924285889, |
| "losses/sft": 1.8281772136688232, |
| "losses/total": 0.6930491924285889, |
| "ref_logps/chosen": -42.54518127441406, |
| "ref_logps/rejected": -47.80082321166992, |
| "rewards/accuracies": 0.578125, |
| "rewards/chosen": -0.47488299012184143, |
| "rewards/margins": 0.17491832375526428, |
| "rewards/rejected": -0.6498013138771057, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.61, |
| "grad_norm": 8.385277776330648, |
| "learning_rate": 4.4241573033707865e-07, |
| "logps/chosen": -48.636085510253906, |
| "logps/rejected": -54.089210510253906, |
| "loss": 0.6105, |
| "losses/dpo": 0.5880630612373352, |
| "losses/sft": 1.647892951965332, |
| "losses/total": 0.5880630612373352, |
| "ref_logps/chosen": -43.907169342041016, |
| "ref_logps/rejected": -47.025569915771484, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.472891628742218, |
| "rewards/margins": 0.2334723323583603, |
| "rewards/rejected": -0.7063639760017395, |
| "step": 81 |
| }, |
| { |
| "epoch": 0.62, |
| "grad_norm": 10.98551780733234, |
| "learning_rate": 4.410112359550562e-07, |
| "logps/chosen": -44.03840637207031, |
| "logps/rejected": -52.65192413330078, |
| "loss": 0.622, |
| "losses/dpo": 0.5322688817977905, |
| "losses/sft": 1.9210578203201294, |
| "losses/total": 0.5322688817977905, |
| "ref_logps/chosen": -39.61579132080078, |
| "ref_logps/rejected": -46.18141174316406, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4422611594200134, |
| "rewards/margins": 0.2047904133796692, |
| "rewards/rejected": -0.6470515727996826, |
| "step": 82 |
| }, |
| { |
| "epoch": 0.63, |
| "grad_norm": 7.7041080998180425, |
| "learning_rate": 4.3960674157303366e-07, |
| "logps/chosen": -44.86250305175781, |
| "logps/rejected": -52.133052825927734, |
| "loss": 0.6265, |
| "losses/dpo": 0.5972993969917297, |
| "losses/sft": 1.4703764915466309, |
| "losses/total": 0.5972993969917297, |
| "ref_logps/chosen": -40.2207145690918, |
| "ref_logps/rejected": -45.56148910522461, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.46417874097824097, |
| "rewards/margins": 0.192977637052536, |
| "rewards/rejected": -0.6571563482284546, |
| "step": 83 |
| }, |
| { |
| "epoch": 0.63, |
| "grad_norm": 7.893501393776045, |
| "learning_rate": 4.382022471910112e-07, |
| "logps/chosen": -45.97453689575195, |
| "logps/rejected": -55.243316650390625, |
| "loss": 0.6289, |
| "losses/dpo": 0.5659444332122803, |
| "losses/sft": 1.6541783809661865, |
| "losses/total": 0.5659444332122803, |
| "ref_logps/chosen": -40.886837005615234, |
| "ref_logps/rejected": -48.23194885253906, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.5087698698043823, |
| "rewards/margins": 0.19236721098423004, |
| "rewards/rejected": -0.7011370062828064, |
| "step": 84 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 9.509139423826628, |
| "learning_rate": 4.367977528089887e-07, |
| "logps/chosen": -44.12831115722656, |
| "logps/rejected": -54.7608642578125, |
| "loss": 0.6193, |
| "losses/dpo": 0.5432471036911011, |
| "losses/sft": 1.8381226062774658, |
| "losses/total": 0.5432471036911011, |
| "ref_logps/chosen": -39.457069396972656, |
| "ref_logps/rejected": -47.7266731262207, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.46712398529052734, |
| "rewards/margins": 0.23629523813724518, |
| "rewards/rejected": -0.7034192681312561, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.65, |
| "grad_norm": 7.887107642093141, |
| "learning_rate": 4.353932584269663e-07, |
| "logps/chosen": -45.145904541015625, |
| "logps/rejected": -55.40123748779297, |
| "loss": 0.6092, |
| "losses/dpo": 0.5780594944953918, |
| "losses/sft": 1.9163440465927124, |
| "losses/total": 0.5780594944953918, |
| "ref_logps/chosen": -40.329139709472656, |
| "ref_logps/rejected": -48.183074951171875, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.4816761910915375, |
| "rewards/margins": 0.24014019966125488, |
| "rewards/rejected": -0.7218164205551147, |
| "step": 86 |
| }, |
| { |
| "epoch": 0.66, |
| "grad_norm": 8.39204893477319, |
| "learning_rate": 4.339887640449438e-07, |
| "logps/chosen": -46.145851135253906, |
| "logps/rejected": -53.961036682128906, |
| "loss": 0.6403, |
| "losses/dpo": 0.60181725025177, |
| "losses/sft": 1.625700831413269, |
| "losses/total": 0.60181725025177, |
| "ref_logps/chosen": -40.71726608276367, |
| "ref_logps/rejected": -46.57926940917969, |
| "rewards/accuracies": 0.5859375, |
| "rewards/chosen": -0.5428579449653625, |
| "rewards/margins": 0.19531863927841187, |
| "rewards/rejected": -0.7381765842437744, |
| "step": 87 |
| }, |
| { |
| "epoch": 0.66, |
| "grad_norm": 8.63628479914341, |
| "learning_rate": 4.3258426966292134e-07, |
| "logps/chosen": -48.23160171508789, |
| "logps/rejected": -53.496604919433594, |
| "loss": 0.6382, |
| "losses/dpo": 0.5790094137191772, |
| "losses/sft": 1.4581667184829712, |
| "losses/total": 0.5790094137191772, |
| "ref_logps/chosen": -42.78254699707031, |
| "ref_logps/rejected": -46.28538131713867, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.5449056029319763, |
| "rewards/margins": 0.1762169450521469, |
| "rewards/rejected": -0.7211225628852844, |
| "step": 88 |
| }, |
| { |
| "epoch": 0.67, |
| "grad_norm": 7.682264878764254, |
| "learning_rate": 4.311797752808989e-07, |
| "logps/chosen": -44.9177360534668, |
| "logps/rejected": -57.826866149902344, |
| "loss": 0.5789, |
| "losses/dpo": 0.5269919037818909, |
| "losses/sft": 1.9525985717773438, |
| "losses/total": 0.5269919037818909, |
| "ref_logps/chosen": -39.326385498046875, |
| "ref_logps/rejected": -48.92866134643555, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.5591354370117188, |
| "rewards/margins": 0.33068495988845825, |
| "rewards/rejected": -0.889820396900177, |
| "step": 89 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 9.993714538070773, |
| "learning_rate": 4.297752808988764e-07, |
| "logps/chosen": -47.74348068237305, |
| "logps/rejected": -51.28035354614258, |
| "loss": 0.6687, |
| "losses/dpo": 0.6379462480545044, |
| "losses/sft": 1.61128830909729, |
| "losses/total": 0.6379462480545044, |
| "ref_logps/chosen": -41.72521209716797, |
| "ref_logps/rejected": -44.09041976928711, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.6018266677856445, |
| "rewards/margins": 0.11716663837432861, |
| "rewards/rejected": -0.7189933061599731, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.69, |
| "grad_norm": 9.79439579593073, |
| "learning_rate": 4.2837078651685396e-07, |
| "logps/chosen": -46.20167541503906, |
| "logps/rejected": -54.01323318481445, |
| "loss": 0.6199, |
| "losses/dpo": 0.4756242632865906, |
| "losses/sft": 1.6830320358276367, |
| "losses/total": 0.4756242632865906, |
| "ref_logps/chosen": -40.16909408569336, |
| "ref_logps/rejected": -45.69634246826172, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.6032581329345703, |
| "rewards/margins": 0.22843076288700104, |
| "rewards/rejected": -0.8316888809204102, |
| "step": 91 |
| }, |
| { |
| "epoch": 0.69, |
| "grad_norm": 8.115443534609298, |
| "learning_rate": 4.269662921348314e-07, |
| "logps/chosen": -49.6710319519043, |
| "logps/rejected": -60.84608840942383, |
| "loss": 0.5948, |
| "losses/dpo": 0.6356014013290405, |
| "losses/sft": 1.7809040546417236, |
| "losses/total": 0.6356014013290405, |
| "ref_logps/chosen": -43.4056282043457, |
| "ref_logps/rejected": -51.69749069213867, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.6265405416488647, |
| "rewards/margins": 0.2883196473121643, |
| "rewards/rejected": -0.914860188961029, |
| "step": 92 |
| }, |
| { |
| "epoch": 0.7, |
| "grad_norm": 8.161770613063224, |
| "learning_rate": 4.2556179775280896e-07, |
| "logps/chosen": -44.44715881347656, |
| "logps/rejected": -52.76496124267578, |
| "loss": 0.6499, |
| "losses/dpo": 0.5320106744766235, |
| "losses/sft": 1.5193849802017212, |
| "losses/total": 0.5320106744766235, |
| "ref_logps/chosen": -38.260562896728516, |
| "ref_logps/rejected": -44.902198791503906, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.618659496307373, |
| "rewards/margins": 0.1676165610551834, |
| "rewards/rejected": -0.78627610206604, |
| "step": 93 |
| }, |
| { |
| "epoch": 0.71, |
| "grad_norm": 11.63146009410558, |
| "learning_rate": 4.2415730337078647e-07, |
| "logps/chosen": -48.569419860839844, |
| "logps/rejected": -55.14238357543945, |
| "loss": 0.6309, |
| "losses/dpo": 0.5968553423881531, |
| "losses/sft": 1.5720221996307373, |
| "losses/total": 0.5968553423881531, |
| "ref_logps/chosen": -42.18726348876953, |
| "ref_logps/rejected": -46.23759460449219, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.6382158994674683, |
| "rewards/margins": 0.2522626221179962, |
| "rewards/rejected": -0.8904784917831421, |
| "step": 94 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 7.826574210546138, |
| "learning_rate": 4.22752808988764e-07, |
| "logps/chosen": -48.970035552978516, |
| "logps/rejected": -56.224632263183594, |
| "loss": 0.6209, |
| "losses/dpo": 0.566143810749054, |
| "losses/sft": 1.7626792192459106, |
| "losses/total": 0.566143810749054, |
| "ref_logps/chosen": -42.288108825683594, |
| "ref_logps/rejected": -47.209964752197266, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.6681923270225525, |
| "rewards/margins": 0.23327398300170898, |
| "rewards/rejected": -0.9014662504196167, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 7.984390488888068, |
| "learning_rate": 4.2134831460674153e-07, |
| "logps/chosen": -44.690216064453125, |
| "logps/rejected": -57.38431930541992, |
| "loss": 0.5916, |
| "losses/dpo": 0.5278609395027161, |
| "losses/sft": 1.7001551389694214, |
| "losses/total": 0.5278609395027161, |
| "ref_logps/chosen": -39.018001556396484, |
| "ref_logps/rejected": -48.73064422607422, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.567221999168396, |
| "rewards/margins": 0.2981455326080322, |
| "rewards/rejected": -0.8653674125671387, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.73, |
| "grad_norm": 8.140376307221391, |
| "learning_rate": 4.199438202247191e-07, |
| "logps/chosen": -45.94129180908203, |
| "logps/rejected": -57.70640563964844, |
| "loss": 0.5843, |
| "losses/dpo": 0.6323425769805908, |
| "losses/sft": 1.9729546308517456, |
| "losses/total": 0.6323425769805908, |
| "ref_logps/chosen": -39.79209899902344, |
| "ref_logps/rejected": -47.78965377807617, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.6149196624755859, |
| "rewards/margins": 0.37675485014915466, |
| "rewards/rejected": -0.991674542427063, |
| "step": 97 |
| }, |
| { |
| "epoch": 0.74, |
| "grad_norm": 8.940775697669213, |
| "learning_rate": 4.1853932584269664e-07, |
| "logps/chosen": -49.44526672363281, |
| "logps/rejected": -54.83664321899414, |
| "loss": 0.6672, |
| "losses/dpo": 0.6795445084571838, |
| "losses/sft": 1.7624062299728394, |
| "losses/total": 0.6795445084571838, |
| "ref_logps/chosen": -42.365447998046875, |
| "ref_logps/rejected": -46.211761474609375, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.7079817652702332, |
| "rewards/margins": 0.15450690686702728, |
| "rewards/rejected": -0.8624885678291321, |
| "step": 98 |
| }, |
| { |
| "epoch": 0.75, |
| "grad_norm": 7.425234946687916, |
| "learning_rate": 4.1713483146067415e-07, |
| "logps/chosen": -43.5301513671875, |
| "logps/rejected": -52.020164489746094, |
| "loss": 0.5957, |
| "losses/dpo": 0.6246699690818787, |
| "losses/sft": 1.7049494981765747, |
| "losses/total": 0.6246699690818787, |
| "ref_logps/chosen": -37.423370361328125, |
| "ref_logps/rejected": -42.76348876953125, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -0.6106777191162109, |
| "rewards/margins": 0.31499022245407104, |
| "rewards/rejected": -0.9256680607795715, |
| "step": 99 |
| }, |
| { |
| "epoch": 0.75, |
| "grad_norm": 9.179756029554161, |
| "learning_rate": 4.157303370786517e-07, |
| "logps/chosen": -49.9049072265625, |
| "logps/rejected": -51.58677673339844, |
| "loss": 0.6817, |
| "losses/dpo": 0.4553123712539673, |
| "losses/sft": 1.8781102895736694, |
| "losses/total": 0.4553123712539673, |
| "ref_logps/chosen": -42.09438705444336, |
| "ref_logps/rejected": -42.1667594909668, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.7810521721839905, |
| "rewards/margins": 0.16094914078712463, |
| "rewards/rejected": -0.9420013427734375, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.76, |
| "grad_norm": 8.48875168411848, |
| "learning_rate": 4.1432584269662915e-07, |
| "logps/chosen": -43.814697265625, |
| "logps/rejected": -56.10200881958008, |
| "loss": 0.5895, |
| "losses/dpo": 0.6046161651611328, |
| "losses/sft": 1.7643048763275146, |
| "losses/total": 0.6046161651611328, |
| "ref_logps/chosen": -37.67271423339844, |
| "ref_logps/rejected": -46.54473114013672, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.6141979694366455, |
| "rewards/margins": 0.3415302038192749, |
| "rewards/rejected": -0.9557281732559204, |
| "step": 101 |
| }, |
| { |
| "epoch": 0.77, |
| "grad_norm": 8.913022100408304, |
| "learning_rate": 4.129213483146067e-07, |
| "logps/chosen": -43.33885955810547, |
| "logps/rejected": -49.23841094970703, |
| "loss": 0.6399, |
| "losses/dpo": 0.5780912637710571, |
| "losses/sft": 2.1572117805480957, |
| "losses/total": 0.5780912637710571, |
| "ref_logps/chosen": -37.323486328125, |
| "ref_logps/rejected": -41.119110107421875, |
| "rewards/accuracies": 0.609375, |
| "rewards/chosen": -0.6015373468399048, |
| "rewards/margins": 0.21039217710494995, |
| "rewards/rejected": -0.8119295239448547, |
| "step": 102 |
| }, |
| { |
| "epoch": 0.78, |
| "grad_norm": 8.206141020123177, |
| "learning_rate": 4.115168539325842e-07, |
| "logps/chosen": -48.0225830078125, |
| "logps/rejected": -54.95545196533203, |
| "loss": 0.6109, |
| "losses/dpo": 0.5648887753486633, |
| "losses/sft": 1.740958571434021, |
| "losses/total": 0.5648887753486633, |
| "ref_logps/chosen": -41.10425567626953, |
| "ref_logps/rejected": -45.241817474365234, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.6918322443962097, |
| "rewards/margins": 0.27953118085861206, |
| "rewards/rejected": -0.9713634848594666, |
| "step": 103 |
| }, |
| { |
| "epoch": 0.78, |
| "grad_norm": 7.999449117068295, |
| "learning_rate": 4.1011235955056177e-07, |
| "logps/chosen": -48.33061981201172, |
| "logps/rejected": -56.14019775390625, |
| "loss": 0.6036, |
| "losses/dpo": 0.5798739194869995, |
| "losses/sft": 1.6369647979736328, |
| "losses/total": 0.5798739194869995, |
| "ref_logps/chosen": -40.94407653808594, |
| "ref_logps/rejected": -45.9286003112793, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.7386540174484253, |
| "rewards/margins": 0.28250569105148315, |
| "rewards/rejected": -1.0211596488952637, |
| "step": 104 |
| }, |
| { |
| "epoch": 0.79, |
| "grad_norm": 7.961208032843563, |
| "learning_rate": 4.0870786516853933e-07, |
| "logps/chosen": -43.01828384399414, |
| "logps/rejected": -55.2956657409668, |
| "loss": 0.5814, |
| "losses/dpo": 0.5777114629745483, |
| "losses/sft": 2.0679547786712646, |
| "losses/total": 0.5777114629745483, |
| "ref_logps/chosen": -36.25102233886719, |
| "ref_logps/rejected": -45.063175201416016, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.6767261624336243, |
| "rewards/margins": 0.3465230464935303, |
| "rewards/rejected": -1.0232491493225098, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 8.162855072558866, |
| "learning_rate": 4.0730337078651683e-07, |
| "logps/chosen": -38.68614959716797, |
| "logps/rejected": -51.58649444580078, |
| "loss": 0.6392, |
| "losses/dpo": 0.6045973896980286, |
| "losses/sft": 1.7759897708892822, |
| "losses/total": 0.6045973896980286, |
| "ref_logps/chosen": -32.37638854980469, |
| "ref_logps/rejected": -42.94057083129883, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.6309766173362732, |
| "rewards/margins": 0.2336157262325287, |
| "rewards/rejected": -0.8645923733711243, |
| "step": 106 |
| }, |
| { |
| "epoch": 0.81, |
| "grad_norm": 10.312576606575677, |
| "learning_rate": 4.058988764044944e-07, |
| "logps/chosen": -50.928245544433594, |
| "logps/rejected": -59.36943054199219, |
| "loss": 0.654, |
| "losses/dpo": 0.735203206539154, |
| "losses/sft": 2.190847158432007, |
| "losses/total": 0.735203206539154, |
| "ref_logps/chosen": -42.11771774291992, |
| "ref_logps/rejected": -48.740692138671875, |
| "rewards/accuracies": 0.6015625, |
| "rewards/chosen": -0.8810529112815857, |
| "rewards/margins": 0.18182089924812317, |
| "rewards/rejected": -1.0628738403320312, |
| "step": 107 |
| }, |
| { |
| "epoch": 0.82, |
| "grad_norm": 8.648201545330428, |
| "learning_rate": 4.044943820224719e-07, |
| "logps/chosen": -47.14788818359375, |
| "logps/rejected": -54.653385162353516, |
| "loss": 0.63, |
| "losses/dpo": 0.6501352787017822, |
| "losses/sft": 2.0834176540374756, |
| "losses/total": 0.6501352787017822, |
| "ref_logps/chosen": -39.38991928100586, |
| "ref_logps/rejected": -44.46839904785156, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.7757968902587891, |
| "rewards/margins": 0.24270157516002655, |
| "rewards/rejected": -1.0184985399246216, |
| "step": 108 |
| }, |
| { |
| "epoch": 0.82, |
| "grad_norm": 9.8578017889453, |
| "learning_rate": 4.0308988764044945e-07, |
| "logps/chosen": -48.94048309326172, |
| "logps/rejected": -56.453369140625, |
| "loss": 0.6293, |
| "losses/dpo": 0.7049952149391174, |
| "losses/sft": 2.280228614807129, |
| "losses/total": 0.7049952149391174, |
| "ref_logps/chosen": -41.544471740722656, |
| "ref_logps/rejected": -46.083526611328125, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.7396014332771301, |
| "rewards/margins": 0.29738324880599976, |
| "rewards/rejected": -1.0369845628738403, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.83, |
| "grad_norm": 8.394061989951249, |
| "learning_rate": 4.0168539325842696e-07, |
| "logps/chosen": -45.53207015991211, |
| "logps/rejected": -54.01716613769531, |
| "loss": 0.6652, |
| "losses/dpo": 0.5625556111335754, |
| "losses/sft": 2.00128436088562, |
| "losses/total": 0.5625556111335754, |
| "ref_logps/chosen": -37.465904235839844, |
| "ref_logps/rejected": -44.16294860839844, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.8066164255142212, |
| "rewards/margins": 0.17880576848983765, |
| "rewards/rejected": -0.9854221940040588, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.84, |
| "grad_norm": 8.76540952525436, |
| "learning_rate": 4.0028089887640446e-07, |
| "logps/chosen": -46.494407653808594, |
| "logps/rejected": -56.70625686645508, |
| "loss": 0.6355, |
| "losses/dpo": 0.816941499710083, |
| "losses/sft": 2.149186372756958, |
| "losses/total": 0.816941499710083, |
| "ref_logps/chosen": -38.6019401550293, |
| "ref_logps/rejected": -46.42439270019531, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.7892469167709351, |
| "rewards/margins": 0.23893946409225464, |
| "rewards/rejected": -1.028186321258545, |
| "step": 111 |
| }, |
| { |
| "epoch": 0.85, |
| "grad_norm": 8.706675493262694, |
| "learning_rate": 3.9887640449438196e-07, |
| "logps/chosen": -48.493709564208984, |
| "logps/rejected": -58.21279525756836, |
| "loss": 0.6039, |
| "losses/dpo": 0.49378710985183716, |
| "losses/sft": 1.3456647396087646, |
| "losses/total": 0.49378710985183716, |
| "ref_logps/chosen": -41.08473205566406, |
| "ref_logps/rejected": -47.72603225708008, |
| "rewards/accuracies": 0.6640625, |
| "rewards/chosen": -0.7408972978591919, |
| "rewards/margins": 0.30777889490127563, |
| "rewards/rejected": -1.0486761331558228, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.85, |
| "grad_norm": 8.390537463814029, |
| "learning_rate": 3.974719101123595e-07, |
| "logps/chosen": -45.94505310058594, |
| "logps/rejected": -59.44303894042969, |
| "loss": 0.5863, |
| "losses/dpo": 0.7109102606773376, |
| "losses/sft": 1.734868049621582, |
| "losses/total": 0.7109102606773376, |
| "ref_logps/chosen": -39.46669387817383, |
| "ref_logps/rejected": -49.24740219116211, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.6478357911109924, |
| "rewards/margins": 0.37172842025756836, |
| "rewards/rejected": -1.0195642709732056, |
| "step": 113 |
| }, |
| { |
| "epoch": 0.86, |
| "grad_norm": 7.999605487961454, |
| "learning_rate": 3.960674157303371e-07, |
| "logps/chosen": -43.57410430908203, |
| "logps/rejected": -53.34558868408203, |
| "loss": 0.6172, |
| "losses/dpo": 0.5053269863128662, |
| "losses/sft": 1.450685739517212, |
| "losses/total": 0.5053269863128662, |
| "ref_logps/chosen": -36.833072662353516, |
| "ref_logps/rejected": -43.94281768798828, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.6741028428077698, |
| "rewards/margins": 0.26617470383644104, |
| "rewards/rejected": -0.9402774572372437, |
| "step": 114 |
| }, |
| { |
| "epoch": 0.87, |
| "grad_norm": 9.415075126299298, |
| "learning_rate": 3.946629213483146e-07, |
| "logps/chosen": -46.565555572509766, |
| "logps/rejected": -57.010372161865234, |
| "loss": 0.5955, |
| "losses/dpo": 0.5628423690795898, |
| "losses/sft": 1.7239865064620972, |
| "losses/total": 0.5628423690795898, |
| "ref_logps/chosen": -39.13492202758789, |
| "ref_logps/rejected": -46.0317497253418, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.743063747882843, |
| "rewards/margins": 0.3547991216182709, |
| "rewards/rejected": -1.097862958908081, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.88, |
| "grad_norm": 9.600908058178481, |
| "learning_rate": 3.9325842696629214e-07, |
| "logps/chosen": -50.78204345703125, |
| "logps/rejected": -56.04452896118164, |
| "loss": 0.6852, |
| "losses/dpo": 0.635813295841217, |
| "losses/sft": 1.73310124874115, |
| "losses/total": 0.635813295841217, |
| "ref_logps/chosen": -42.01323318481445, |
| "ref_logps/rejected": -45.830718994140625, |
| "rewards/accuracies": 0.6171875, |
| "rewards/chosen": -0.8768815994262695, |
| "rewards/margins": 0.14449933171272278, |
| "rewards/rejected": -1.02138090133667, |
| "step": 116 |
| }, |
| { |
| "epoch": 0.88, |
| "grad_norm": 8.373342286084757, |
| "learning_rate": 3.9185393258426964e-07, |
| "logps/chosen": -47.82256317138672, |
| "logps/rejected": -54.956302642822266, |
| "loss": 0.6136, |
| "losses/dpo": 0.6849408149719238, |
| "losses/sft": 2.210094451904297, |
| "losses/total": 0.6849408149719238, |
| "ref_logps/chosen": -40.209930419921875, |
| "ref_logps/rejected": -44.30906295776367, |
| "rewards/accuracies": 0.6484375, |
| "rewards/chosen": -0.7612636089324951, |
| "rewards/margins": 0.3034607172012329, |
| "rewards/rejected": -1.0647242069244385, |
| "step": 117 |
| }, |
| { |
| "epoch": 0.89, |
| "grad_norm": 9.637403138990082, |
| "learning_rate": 3.904494382022472e-07, |
| "logps/chosen": -48.02635955810547, |
| "logps/rejected": -57.71215057373047, |
| "loss": 0.5835, |
| "losses/dpo": 0.6933724880218506, |
| "losses/sft": 1.4713902473449707, |
| "losses/total": 0.6933724880218506, |
| "ref_logps/chosen": -40.86788558959961, |
| "ref_logps/rejected": -47.20375442504883, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -0.7158471345901489, |
| "rewards/margins": 0.33499258756637573, |
| "rewards/rejected": -1.0508397817611694, |
| "step": 118 |
| }, |
| { |
| "epoch": 0.9, |
| "grad_norm": 8.531064508491891, |
| "learning_rate": 3.890449438202247e-07, |
| "logps/chosen": -46.2841796875, |
| "logps/rejected": -54.74563980102539, |
| "loss": 0.6334, |
| "losses/dpo": 0.702929675579071, |
| "losses/sft": 1.360397219657898, |
| "losses/total": 0.702929675579071, |
| "ref_logps/chosen": -39.4974365234375, |
| "ref_logps/rejected": -45.235111236572266, |
| "rewards/accuracies": 0.640625, |
| "rewards/chosen": -0.6786739826202393, |
| "rewards/margins": 0.2723783850669861, |
| "rewards/rejected": -0.9510524272918701, |
| "step": 119 |
| }, |
| { |
| "epoch": 0.91, |
| "grad_norm": 41.566258002848386, |
| "learning_rate": 3.876404494382022e-07, |
| "logps/chosen": -44.022640228271484, |
| "logps/rejected": -55.926021575927734, |
| "loss": 0.596, |
| "losses/dpo": 0.5612522959709167, |
| "losses/sft": 1.8725919723510742, |
| "losses/total": 0.5612522959709167, |
| "ref_logps/chosen": -37.003326416015625, |
| "ref_logps/rejected": -45.59555435180664, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.701931357383728, |
| "rewards/margins": 0.3311149477958679, |
| "rewards/rejected": -1.0330464839935303, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.91, |
| "grad_norm": 8.107636440875833, |
| "learning_rate": 3.8623595505617977e-07, |
| "logps/chosen": -46.37071990966797, |
| "logps/rejected": -57.88187026977539, |
| "loss": 0.5981, |
| "losses/dpo": 0.5683261156082153, |
| "losses/sft": 1.2367005348205566, |
| "losses/total": 0.5683261156082153, |
| "ref_logps/chosen": -40.31574630737305, |
| "ref_logps/rejected": -48.33371353149414, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.6054975986480713, |
| "rewards/margins": 0.3493175208568573, |
| "rewards/rejected": -0.9548150300979614, |
| "step": 121 |
| }, |
| { |
| "epoch": 0.92, |
| "grad_norm": 7.976666948784577, |
| "learning_rate": 3.8483146067415727e-07, |
| "logps/chosen": -49.71910095214844, |
| "logps/rejected": -61.38713073730469, |
| "loss": 0.5628, |
| "losses/dpo": 0.6633545756340027, |
| "losses/sft": 2.0874011516571045, |
| "losses/total": 0.6633545756340027, |
| "ref_logps/chosen": -42.128177642822266, |
| "ref_logps/rejected": -49.5493278503418, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.7590923309326172, |
| "rewards/margins": 0.42468804121017456, |
| "rewards/rejected": -1.1837804317474365, |
| "step": 122 |
| }, |
| { |
| "epoch": 0.93, |
| "grad_norm": 8.273120167281697, |
| "learning_rate": 3.834269662921348e-07, |
| "logps/chosen": -44.52275848388672, |
| "logps/rejected": -56.69093322753906, |
| "loss": 0.5973, |
| "losses/dpo": 0.7617586851119995, |
| "losses/sft": 1.4152177572250366, |
| "losses/total": 0.7617586851119995, |
| "ref_logps/chosen": -38.226234436035156, |
| "ref_logps/rejected": -47.10552215576172, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.6296522617340088, |
| "rewards/margins": 0.32888925075531006, |
| "rewards/rejected": -0.9585415124893188, |
| "step": 123 |
| }, |
| { |
| "epoch": 0.94, |
| "grad_norm": 8.426019615665346, |
| "learning_rate": 3.8202247191011233e-07, |
| "logps/chosen": -45.17867660522461, |
| "logps/rejected": -52.477989196777344, |
| "loss": 0.5956, |
| "losses/dpo": 0.8184994459152222, |
| "losses/sft": 2.1767871379852295, |
| "losses/total": 0.8184994459152222, |
| "ref_logps/chosen": -39.22052001953125, |
| "ref_logps/rejected": -42.9237060546875, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.5958150625228882, |
| "rewards/margins": 0.35961273312568665, |
| "rewards/rejected": -0.9554278254508972, |
| "step": 124 |
| }, |
| { |
| "epoch": 0.94, |
| "grad_norm": 9.445037206868065, |
| "learning_rate": 3.806179775280899e-07, |
| "logps/chosen": -49.57510757446289, |
| "logps/rejected": -57.41535568237305, |
| "loss": 0.626, |
| "losses/dpo": 0.5964499711990356, |
| "losses/sft": 1.574311375617981, |
| "losses/total": 0.5964499711990356, |
| "ref_logps/chosen": -42.10932159423828, |
| "ref_logps/rejected": -47.498348236083984, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.746578574180603, |
| "rewards/margins": 0.24512259662151337, |
| "rewards/rejected": -0.9917011260986328, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.95, |
| "grad_norm": 8.142660343260747, |
| "learning_rate": 3.792134831460674e-07, |
| "logps/chosen": -49.39699935913086, |
| "logps/rejected": -60.31464385986328, |
| "loss": 0.5557, |
| "losses/dpo": 0.4981670677661896, |
| "losses/sft": 1.7636176347732544, |
| "losses/total": 0.4981670677661896, |
| "ref_logps/chosen": -42.645172119140625, |
| "ref_logps/rejected": -49.29505157470703, |
| "rewards/accuracies": 0.7578125, |
| "rewards/chosen": -0.675183117389679, |
| "rewards/margins": 0.4267764091491699, |
| "rewards/rejected": -1.1019595861434937, |
| "step": 126 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 10.371790207828976, |
| "learning_rate": 3.7780898876404495e-07, |
| "logps/chosen": -49.649539947509766, |
| "logps/rejected": -55.797752380371094, |
| "loss": 0.6236, |
| "losses/dpo": 0.6386290788650513, |
| "losses/sft": 1.9548349380493164, |
| "losses/total": 0.6386290788650513, |
| "ref_logps/chosen": -43.093387603759766, |
| "ref_logps/rejected": -46.55876541137695, |
| "rewards/accuracies": 0.6796875, |
| "rewards/chosen": -0.6556151509284973, |
| "rewards/margins": 0.2682836949825287, |
| "rewards/rejected": -0.9238989353179932, |
| "step": 127 |
| }, |
| { |
| "epoch": 0.97, |
| "grad_norm": 8.551344033897433, |
| "learning_rate": 3.7640449438202245e-07, |
| "logps/chosen": -47.41435623168945, |
| "logps/rejected": -53.9222526550293, |
| "loss": 0.6097, |
| "losses/dpo": 0.5466079711914062, |
| "losses/sft": 1.6678849458694458, |
| "losses/total": 0.5466079711914062, |
| "ref_logps/chosen": -40.969482421875, |
| "ref_logps/rejected": -44.308204650878906, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.6444875001907349, |
| "rewards/margins": 0.3169165849685669, |
| "rewards/rejected": -0.9614041447639465, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.97, |
| "grad_norm": 8.48986980564796, |
| "learning_rate": 3.75e-07, |
| "logps/chosen": -46.43178176879883, |
| "logps/rejected": -54.83501434326172, |
| "loss": 0.6116, |
| "losses/dpo": 0.7334883213043213, |
| "losses/sft": 2.06964111328125, |
| "losses/total": 0.7334883213043213, |
| "ref_logps/chosen": -39.70704650878906, |
| "ref_logps/rejected": -45.3802490234375, |
| "rewards/accuracies": 0.703125, |
| "rewards/chosen": -0.6724739074707031, |
| "rewards/margins": 0.2730027139186859, |
| "rewards/rejected": -0.9454765915870667, |
| "step": 129 |
| }, |
| { |
| "epoch": 0.98, |
| "grad_norm": 7.931029049263691, |
| "learning_rate": 3.735955056179775e-07, |
| "logps/chosen": -41.7554817199707, |
| "logps/rejected": -51.33645248413086, |
| "loss": 0.6118, |
| "losses/dpo": 0.6140174865722656, |
| "losses/sft": 1.9485254287719727, |
| "losses/total": 0.6140174865722656, |
| "ref_logps/chosen": -36.31658172607422, |
| "ref_logps/rejected": -43.00047302246094, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.5438905954360962, |
| "rewards/margins": 0.28970640897750854, |
| "rewards/rejected": -0.8335970640182495, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.99, |
| "grad_norm": 7.961225244135859, |
| "learning_rate": 3.72191011235955e-07, |
| "logps/chosen": -45.26293182373047, |
| "logps/rejected": -53.40849304199219, |
| "loss": 0.5975, |
| "losses/dpo": 0.6238459348678589, |
| "losses/sft": 1.9288058280944824, |
| "losses/total": 0.6238459348678589, |
| "ref_logps/chosen": -39.287940979003906, |
| "ref_logps/rejected": -43.95112991333008, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.597498893737793, |
| "rewards/margins": 0.34823763370513916, |
| "rewards/rejected": -0.9457363486289978, |
| "step": 131 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 8.463248361122965, |
| "learning_rate": 3.707865168539326e-07, |
| "logps/chosen": -46.658084869384766, |
| "logps/rejected": -55.08595275878906, |
| "loss": 0.6124, |
| "losses/dpo": 0.5780600309371948, |
| "losses/sft": 2.4361934661865234, |
| "losses/total": 0.5780600309371948, |
| "ref_logps/chosen": -40.17093276977539, |
| "ref_logps/rejected": -45.625953674316406, |
| "rewards/accuracies": 0.671875, |
| "rewards/chosen": -0.6487153172492981, |
| "rewards/margins": 0.29728472232818604, |
| "rewards/rejected": -0.9460000395774841, |
| "step": 132 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 7.966461556458486, |
| "learning_rate": 3.693820224719101e-07, |
| "logps/chosen": -43.571929931640625, |
| "logps/rejected": -56.445987701416016, |
| "loss": 0.5996, |
| "losses/dpo": 0.6271636486053467, |
| "losses/sft": 1.7036837339401245, |
| "losses/total": 0.6271636486053467, |
| "ref_logps/chosen": -37.748313903808594, |
| "ref_logps/rejected": -46.95930862426758, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.5823614001274109, |
| "rewards/margins": 0.366305947303772, |
| "rewards/rejected": -0.9486674070358276, |
| "step": 133 |
| }, |
| { |
| "epoch": 1.01, |
| "grad_norm": 8.685706334967172, |
| "learning_rate": 3.6797752808988764e-07, |
| "logps/chosen": -45.6722412109375, |
| "logps/rejected": -58.07893371582031, |
| "loss": 0.5569, |
| "losses/dpo": 0.44357961416244507, |
| "losses/sft": 1.9906073808670044, |
| "losses/total": 0.44357961416244507, |
| "ref_logps/chosen": -39.666465759277344, |
| "ref_logps/rejected": -47.67109298706055, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.6005774736404419, |
| "rewards/margins": 0.44020622968673706, |
| "rewards/rejected": -1.0407837629318237, |
| "step": 134 |
| }, |
| { |
| "epoch": 1.02, |
| "grad_norm": 7.5465440969699475, |
| "learning_rate": 3.6657303370786514e-07, |
| "logps/chosen": -44.68495178222656, |
| "logps/rejected": -59.658729553222656, |
| "loss": 0.5418, |
| "losses/dpo": 0.6047704815864563, |
| "losses/sft": 1.8062564134597778, |
| "losses/total": 0.6047704815864563, |
| "ref_logps/chosen": -38.53544616699219, |
| "ref_logps/rejected": -48.757301330566406, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.6149506568908691, |
| "rewards/margins": 0.47519204020500183, |
| "rewards/rejected": -1.0901426076889038, |
| "step": 135 |
| }, |
| { |
| "epoch": 1.03, |
| "grad_norm": 8.040538818716184, |
| "learning_rate": 3.651685393258427e-07, |
| "logps/chosen": -49.753868103027344, |
| "logps/rejected": -55.77690124511719, |
| "loss": 0.5676, |
| "losses/dpo": 0.6332641243934631, |
| "losses/sft": 1.9821722507476807, |
| "losses/total": 0.6332641243934631, |
| "ref_logps/chosen": -43.2289924621582, |
| "ref_logps/rejected": -44.969783782958984, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.6524879932403564, |
| "rewards/margins": 0.4282234311103821, |
| "rewards/rejected": -1.0807113647460938, |
| "step": 136 |
| }, |
| { |
| "epoch": 1.03, |
| "grad_norm": 8.176454529563474, |
| "learning_rate": 3.637640449438202e-07, |
| "logps/chosen": -44.740989685058594, |
| "logps/rejected": -56.03924560546875, |
| "loss": 0.5426, |
| "losses/dpo": 0.5114879608154297, |
| "losses/sft": 1.7858185768127441, |
| "losses/total": 0.5114879608154297, |
| "ref_logps/chosen": -39.14986038208008, |
| "ref_logps/rejected": -45.894187927246094, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.5591133236885071, |
| "rewards/margins": 0.45539283752441406, |
| "rewards/rejected": -1.014506220817566, |
| "step": 137 |
| }, |
| { |
| "epoch": 1.04, |
| "grad_norm": 7.378583429433305, |
| "learning_rate": 3.6235955056179776e-07, |
| "logps/chosen": -43.69382095336914, |
| "logps/rejected": -58.01210021972656, |
| "loss": 0.5576, |
| "losses/dpo": 0.5225633978843689, |
| "losses/sft": 1.5681252479553223, |
| "losses/total": 0.5225633978843689, |
| "ref_logps/chosen": -37.7261848449707, |
| "ref_logps/rejected": -47.4239616394043, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.5967639684677124, |
| "rewards/margins": 0.46204984188079834, |
| "rewards/rejected": -1.0588138103485107, |
| "step": 138 |
| }, |
| { |
| "epoch": 1.05, |
| "grad_norm": 6.804949706358365, |
| "learning_rate": 3.6095505617977526e-07, |
| "logps/chosen": -41.86065673828125, |
| "logps/rejected": -56.17615509033203, |
| "loss": 0.541, |
| "losses/dpo": 0.5155816674232483, |
| "losses/sft": 1.66976797580719, |
| "losses/total": 0.5155816674232483, |
| "ref_logps/chosen": -36.93909454345703, |
| "ref_logps/rejected": -46.33450698852539, |
| "rewards/accuracies": 0.7578125, |
| "rewards/chosen": -0.49215659499168396, |
| "rewards/margins": 0.492008239030838, |
| "rewards/rejected": -0.9841648936271667, |
| "step": 139 |
| }, |
| { |
| "epoch": 1.06, |
| "grad_norm": 9.08939326139503, |
| "learning_rate": 3.5955056179775277e-07, |
| "logps/chosen": -50.215492248535156, |
| "logps/rejected": -61.04364013671875, |
| "loss": 0.4877, |
| "losses/dpo": 0.4505589008331299, |
| "losses/sft": 2.045948028564453, |
| "losses/total": 0.4505589008331299, |
| "ref_logps/chosen": -44.16120147705078, |
| "ref_logps/rejected": -48.857383728027344, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -0.6054291725158691, |
| "rewards/margins": 0.6131964921951294, |
| "rewards/rejected": -1.218625545501709, |
| "step": 140 |
| }, |
| { |
| "epoch": 1.06, |
| "grad_norm": 7.204665546592892, |
| "learning_rate": 3.581460674157303e-07, |
| "logps/chosen": -44.662261962890625, |
| "logps/rejected": -56.869659423828125, |
| "loss": 0.5186, |
| "losses/dpo": 0.4171184301376343, |
| "losses/sft": 1.8761239051818848, |
| "losses/total": 0.4171184301376343, |
| "ref_logps/chosen": -38.65668487548828, |
| "ref_logps/rejected": -45.534423828125, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.6005581617355347, |
| "rewards/margins": 0.5329651832580566, |
| "rewards/rejected": -1.1335232257843018, |
| "step": 141 |
| }, |
| { |
| "epoch": 1.07, |
| "grad_norm": 7.887278895914157, |
| "learning_rate": 3.5674157303370783e-07, |
| "logps/chosen": -42.77589416503906, |
| "logps/rejected": -51.52098083496094, |
| "loss": 0.5407, |
| "losses/dpo": 0.35035043954849243, |
| "losses/sft": 1.5323569774627686, |
| "losses/total": 0.35035043954849243, |
| "ref_logps/chosen": -37.22514343261719, |
| "ref_logps/rejected": -41.38079833984375, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.5550752282142639, |
| "rewards/margins": 0.45894336700439453, |
| "rewards/rejected": -1.0140186548233032, |
| "step": 142 |
| }, |
| { |
| "epoch": 1.08, |
| "grad_norm": 7.8842349887311505, |
| "learning_rate": 3.553370786516854e-07, |
| "logps/chosen": -48.38291549682617, |
| "logps/rejected": -60.9052619934082, |
| "loss": 0.5223, |
| "losses/dpo": 0.3222103714942932, |
| "losses/sft": 1.7116018533706665, |
| "losses/total": 0.3222103714942932, |
| "ref_logps/chosen": -41.83588409423828, |
| "ref_logps/rejected": -48.54487228393555, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.6547029614448547, |
| "rewards/margins": 0.5813360810279846, |
| "rewards/rejected": -1.2360389232635498, |
| "step": 143 |
| }, |
| { |
| "epoch": 1.09, |
| "grad_norm": 8.55836602139343, |
| "learning_rate": 3.539325842696629e-07, |
| "logps/chosen": -43.38884353637695, |
| "logps/rejected": -58.84518051147461, |
| "loss": 0.5957, |
| "losses/dpo": 0.458510160446167, |
| "losses/sft": 1.5300884246826172, |
| "losses/total": 0.458510160446167, |
| "ref_logps/chosen": -36.08906555175781, |
| "ref_logps/rejected": -47.359981536865234, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.7299779653549194, |
| "rewards/margins": 0.41854166984558105, |
| "rewards/rejected": -1.14851975440979, |
| "step": 144 |
| }, |
| { |
| "epoch": 1.09, |
| "grad_norm": 7.848515280846644, |
| "learning_rate": 3.5252808988764045e-07, |
| "logps/chosen": -44.214759826660156, |
| "logps/rejected": -55.23337936401367, |
| "loss": 0.5469, |
| "losses/dpo": 0.47486573457717896, |
| "losses/sft": 1.7108842134475708, |
| "losses/total": 0.47486573457717896, |
| "ref_logps/chosen": -38.41852569580078, |
| "ref_logps/rejected": -44.74144744873047, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.5796229243278503, |
| "rewards/margins": 0.4695700407028198, |
| "rewards/rejected": -1.049193024635315, |
| "step": 145 |
| }, |
| { |
| "epoch": 1.1, |
| "grad_norm": 7.658781592826533, |
| "learning_rate": 3.51123595505618e-07, |
| "logps/chosen": -44.951683044433594, |
| "logps/rejected": -60.23070526123047, |
| "loss": 0.5266, |
| "losses/dpo": 0.5872490406036377, |
| "losses/sft": 1.8830925226211548, |
| "losses/total": 0.5872490406036377, |
| "ref_logps/chosen": -38.69530487060547, |
| "ref_logps/rejected": -48.41132354736328, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.6256377696990967, |
| "rewards/margins": 0.5562998056411743, |
| "rewards/rejected": -1.1819374561309814, |
| "step": 146 |
| }, |
| { |
| "epoch": 1.11, |
| "grad_norm": 8.54544198888784, |
| "learning_rate": 3.497191011235955e-07, |
| "logps/chosen": -48.944068908691406, |
| "logps/rejected": -63.39553451538086, |
| "loss": 0.533, |
| "losses/dpo": 0.40039166808128357, |
| "losses/sft": 1.862857460975647, |
| "losses/total": 0.40039166808128357, |
| "ref_logps/chosen": -42.56263732910156, |
| "ref_logps/rejected": -50.94089126586914, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.6381431818008423, |
| "rewards/margins": 0.6073207259178162, |
| "rewards/rejected": -1.2454639673233032, |
| "step": 147 |
| }, |
| { |
| "epoch": 1.12, |
| "grad_norm": 7.389416730585067, |
| "learning_rate": 3.48314606741573e-07, |
| "logps/chosen": -38.91257858276367, |
| "logps/rejected": -49.71195602416992, |
| "loss": 0.5426, |
| "losses/dpo": 0.46995991468429565, |
| "losses/sft": 1.7020084857940674, |
| "losses/total": 0.46995991468429565, |
| "ref_logps/chosen": -33.769771575927734, |
| "ref_logps/rejected": -40.01531219482422, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.5142804384231567, |
| "rewards/margins": 0.45538395643234253, |
| "rewards/rejected": -0.969664454460144, |
| "step": 148 |
| }, |
| { |
| "epoch": 1.12, |
| "grad_norm": 7.674169098254074, |
| "learning_rate": 3.469101123595505e-07, |
| "logps/chosen": -51.57361602783203, |
| "logps/rejected": -61.665931701660156, |
| "loss": 0.4713, |
| "losses/dpo": 0.4245557487010956, |
| "losses/sft": 1.5430463552474976, |
| "losses/total": 0.4245557487010956, |
| "ref_logps/chosen": -44.85792541503906, |
| "ref_logps/rejected": -48.205955505371094, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.6715684533119202, |
| "rewards/margins": 0.6744291186332703, |
| "rewards/rejected": -1.3459975719451904, |
| "step": 149 |
| }, |
| { |
| "epoch": 1.13, |
| "grad_norm": 22.33237134875282, |
| "learning_rate": 3.4550561797752807e-07, |
| "logps/chosen": -47.884517669677734, |
| "logps/rejected": -58.32150650024414, |
| "loss": 0.5577, |
| "losses/dpo": 0.5011469721794128, |
| "losses/sft": 1.6741063594818115, |
| "losses/total": 0.5011469721794128, |
| "ref_logps/chosen": -40.72835159301758, |
| "ref_logps/rejected": -46.43974685668945, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -0.7156162261962891, |
| "rewards/margins": 0.47256001830101013, |
| "rewards/rejected": -1.1881763935089111, |
| "step": 150 |
| }, |
| { |
| "epoch": 1.14, |
| "grad_norm": 7.494096272004472, |
| "learning_rate": 3.441011235955056e-07, |
| "logps/chosen": -47.11280059814453, |
| "logps/rejected": -60.22679901123047, |
| "loss": 0.4938, |
| "losses/dpo": 0.5828070044517517, |
| "losses/sft": 1.9693294763565063, |
| "losses/total": 0.5828070044517517, |
| "ref_logps/chosen": -41.27467727661133, |
| "ref_logps/rejected": -47.46904754638672, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.5838119983673096, |
| "rewards/margins": 0.691962480545044, |
| "rewards/rejected": -1.2757744789123535, |
| "step": 151 |
| }, |
| { |
| "epoch": 1.15, |
| "grad_norm": 7.516820355286503, |
| "learning_rate": 3.4269662921348313e-07, |
| "logps/chosen": -40.18495559692383, |
| "logps/rejected": -54.60972595214844, |
| "loss": 0.5232, |
| "losses/dpo": 0.5568748116493225, |
| "losses/sft": 1.7009055614471436, |
| "losses/total": 0.5568748116493225, |
| "ref_logps/chosen": -33.816017150878906, |
| "ref_logps/rejected": -43.013160705566406, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.6368939280509949, |
| "rewards/margins": 0.5227622389793396, |
| "rewards/rejected": -1.159656047821045, |
| "step": 152 |
| }, |
| { |
| "epoch": 1.15, |
| "grad_norm": 7.502946402300403, |
| "learning_rate": 3.4129213483146064e-07, |
| "logps/chosen": -42.238433837890625, |
| "logps/rejected": -53.268348693847656, |
| "loss": 0.543, |
| "losses/dpo": 0.5467118620872498, |
| "losses/sft": 1.2100037336349487, |
| "losses/total": 0.5467118620872498, |
| "ref_logps/chosen": -35.477867126464844, |
| "ref_logps/rejected": -41.79436111450195, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.676056981086731, |
| "rewards/margins": 0.47134220600128174, |
| "rewards/rejected": -1.1473990678787231, |
| "step": 153 |
| }, |
| { |
| "epoch": 1.16, |
| "grad_norm": 7.772041407679626, |
| "learning_rate": 3.398876404494382e-07, |
| "logps/chosen": -44.26182174682617, |
| "logps/rejected": -64.4139633178711, |
| "loss": 0.4692, |
| "losses/dpo": 0.4075261354446411, |
| "losses/sft": 1.9415578842163086, |
| "losses/total": 0.4075261354446411, |
| "ref_logps/chosen": -37.64110565185547, |
| "ref_logps/rejected": -50.549678802490234, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.6620715260505676, |
| "rewards/margins": 0.7243567109107971, |
| "rewards/rejected": -1.3864283561706543, |
| "step": 154 |
| }, |
| { |
| "epoch": 1.17, |
| "grad_norm": 9.887168758150576, |
| "learning_rate": 3.3848314606741575e-07, |
| "logps/chosen": -46.856407165527344, |
| "logps/rejected": -60.13237380981445, |
| "loss": 0.5388, |
| "losses/dpo": 0.4830757975578308, |
| "losses/sft": 1.654982328414917, |
| "losses/total": 0.4830757975578308, |
| "ref_logps/chosen": -39.95355987548828, |
| "ref_logps/rejected": -47.57228088378906, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.6902844905853271, |
| "rewards/margins": 0.5657243132591248, |
| "rewards/rejected": -1.2560089826583862, |
| "step": 155 |
| }, |
| { |
| "epoch": 1.18, |
| "grad_norm": 8.0774394495718, |
| "learning_rate": 3.3707865168539325e-07, |
| "logps/chosen": -46.11971664428711, |
| "logps/rejected": -58.2076301574707, |
| "loss": 0.5571, |
| "losses/dpo": 0.6668601036071777, |
| "losses/sft": 2.1451334953308105, |
| "losses/total": 0.6668601036071777, |
| "ref_logps/chosen": -38.718963623046875, |
| "ref_logps/rejected": -45.97993850708008, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -0.7400756478309631, |
| "rewards/margins": 0.48269355297088623, |
| "rewards/rejected": -1.2227692604064941, |
| "step": 156 |
| }, |
| { |
| "epoch": 1.18, |
| "grad_norm": 7.6995260059404735, |
| "learning_rate": 3.356741573033708e-07, |
| "logps/chosen": -42.61933898925781, |
| "logps/rejected": -53.914058685302734, |
| "loss": 0.5563, |
| "losses/dpo": 0.5214600563049316, |
| "losses/sft": 1.6869933605194092, |
| "losses/total": 0.5214600563049316, |
| "ref_logps/chosen": -35.26099395751953, |
| "ref_logps/rejected": -41.69981002807617, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.7358340620994568, |
| "rewards/margins": 0.4855908751487732, |
| "rewards/rejected": -1.2214250564575195, |
| "step": 157 |
| }, |
| { |
| "epoch": 1.19, |
| "grad_norm": 10.378363778579734, |
| "learning_rate": 3.3426966292134826e-07, |
| "logps/chosen": -46.86024856567383, |
| "logps/rejected": -61.01960754394531, |
| "loss": 0.5047, |
| "losses/dpo": 0.35230398178100586, |
| "losses/sft": 2.1814329624176025, |
| "losses/total": 0.35230398178100586, |
| "ref_logps/chosen": -40.29644775390625, |
| "ref_logps/rejected": -48.00575256347656, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.6563804149627686, |
| "rewards/margins": 0.6450048685073853, |
| "rewards/rejected": -1.3013852834701538, |
| "step": 158 |
| }, |
| { |
| "epoch": 1.2, |
| "grad_norm": 8.023005518353788, |
| "learning_rate": 3.328651685393258e-07, |
| "logps/chosen": -48.07395553588867, |
| "logps/rejected": -60.29581069946289, |
| "loss": 0.5284, |
| "losses/dpo": 0.57308030128479, |
| "losses/sft": 2.2541966438293457, |
| "losses/total": 0.57308030128479, |
| "ref_logps/chosen": -39.92966842651367, |
| "ref_logps/rejected": -46.47784423828125, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.8144292831420898, |
| "rewards/margins": 0.5673672556877136, |
| "rewards/rejected": -1.3817965984344482, |
| "step": 159 |
| }, |
| { |
| "epoch": 1.21, |
| "grad_norm": 7.252828799858416, |
| "learning_rate": 3.314606741573033e-07, |
| "logps/chosen": -45.51723861694336, |
| "logps/rejected": -60.294551849365234, |
| "loss": 0.4636, |
| "losses/dpo": 0.581541895866394, |
| "losses/sft": 1.6298561096191406, |
| "losses/total": 0.581541895866394, |
| "ref_logps/chosen": -38.64426040649414, |
| "ref_logps/rejected": -45.62995147705078, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.6872978210449219, |
| "rewards/margins": 0.7791618704795837, |
| "rewards/rejected": -1.4664596319198608, |
| "step": 160 |
| }, |
| { |
| "epoch": 1.22, |
| "grad_norm": 8.57918326492943, |
| "learning_rate": 3.300561797752809e-07, |
| "logps/chosen": -50.444820404052734, |
| "logps/rejected": -56.87160110473633, |
| "loss": 0.5883, |
| "losses/dpo": 0.5911461710929871, |
| "losses/sft": 2.1341657638549805, |
| "losses/total": 0.5911461710929871, |
| "ref_logps/chosen": -41.919921875, |
| "ref_logps/rejected": -43.998451232910156, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.8524903059005737, |
| "rewards/margins": 0.43482455611228943, |
| "rewards/rejected": -1.2873148918151855, |
| "step": 161 |
| }, |
| { |
| "epoch": 1.22, |
| "grad_norm": 8.38374098065899, |
| "learning_rate": 3.2865168539325844e-07, |
| "logps/chosen": -50.57026672363281, |
| "logps/rejected": -61.819053649902344, |
| "loss": 0.5544, |
| "losses/dpo": 0.5535627603530884, |
| "losses/sft": 1.9382483959197998, |
| "losses/total": 0.5535627603530884, |
| "ref_logps/chosen": -41.4891357421875, |
| "ref_logps/rejected": -47.33740234375, |
| "rewards/accuracies": 0.6953125, |
| "rewards/chosen": -0.9081130027770996, |
| "rewards/margins": 0.540052056312561, |
| "rewards/rejected": -1.4481650590896606, |
| "step": 162 |
| }, |
| { |
| "epoch": 1.23, |
| "grad_norm": 8.053746658561026, |
| "learning_rate": 3.2724719101123594e-07, |
| "logps/chosen": -49.71845245361328, |
| "logps/rejected": -55.962120056152344, |
| "loss": 0.5659, |
| "losses/dpo": 0.5589174628257751, |
| "losses/sft": 2.0584986209869385, |
| "losses/total": 0.5589174628257751, |
| "ref_logps/chosen": -41.30320739746094, |
| "ref_logps/rejected": -42.65535354614258, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.841524600982666, |
| "rewards/margins": 0.4891516864299774, |
| "rewards/rejected": -1.3306763172149658, |
| "step": 163 |
| }, |
| { |
| "epoch": 1.24, |
| "grad_norm": 8.128166160006492, |
| "learning_rate": 3.258426966292135e-07, |
| "logps/chosen": -49.184532165527344, |
| "logps/rejected": -62.28634262084961, |
| "loss": 0.5594, |
| "losses/dpo": 0.6036563515663147, |
| "losses/sft": 1.7208425998687744, |
| "losses/total": 0.6036563515663147, |
| "ref_logps/chosen": -40.15517044067383, |
| "ref_logps/rejected": -47.867679595947266, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.9029368162155151, |
| "rewards/margins": 0.5389291644096375, |
| "rewards/rejected": -1.441866159439087, |
| "step": 164 |
| }, |
| { |
| "epoch": 1.25, |
| "grad_norm": 7.785958027787598, |
| "learning_rate": 3.24438202247191e-07, |
| "logps/chosen": -46.6161003112793, |
| "logps/rejected": -64.03797912597656, |
| "loss": 0.5012, |
| "losses/dpo": 0.36227643489837646, |
| "losses/sft": 1.8801686763763428, |
| "losses/total": 0.36227643489837646, |
| "ref_logps/chosen": -39.068603515625, |
| "ref_logps/rejected": -49.793888092041016, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.7547495365142822, |
| "rewards/margins": 0.6696599125862122, |
| "rewards/rejected": -1.42440927028656, |
| "step": 165 |
| }, |
| { |
| "epoch": 1.25, |
| "grad_norm": 8.005747191405192, |
| "learning_rate": 3.2303370786516856e-07, |
| "logps/chosen": -47.46510696411133, |
| "logps/rejected": -64.76780700683594, |
| "loss": 0.495, |
| "losses/dpo": 0.5762285590171814, |
| "losses/sft": 1.6507606506347656, |
| "losses/total": 0.5762285590171814, |
| "ref_logps/chosen": -39.22062683105469, |
| "ref_logps/rejected": -49.37548065185547, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.824447751045227, |
| "rewards/margins": 0.7147842645645142, |
| "rewards/rejected": -1.5392321348190308, |
| "step": 166 |
| }, |
| { |
| "epoch": 1.26, |
| "grad_norm": 8.587266771042302, |
| "learning_rate": 3.21629213483146e-07, |
| "logps/chosen": -44.891380310058594, |
| "logps/rejected": -63.128692626953125, |
| "loss": 0.476, |
| "losses/dpo": 0.3979690968990326, |
| "losses/sft": 1.9852135181427002, |
| "losses/total": 0.3979690968990326, |
| "ref_logps/chosen": -36.76803970336914, |
| "ref_logps/rejected": -47.980194091796875, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.8123339414596558, |
| "rewards/margins": 0.7025157809257507, |
| "rewards/rejected": -1.5148497819900513, |
| "step": 167 |
| }, |
| { |
| "epoch": 1.27, |
| "grad_norm": 8.326932342308945, |
| "learning_rate": 3.2022471910112357e-07, |
| "logps/chosen": -47.12702941894531, |
| "logps/rejected": -63.2083625793457, |
| "loss": 0.4967, |
| "losses/dpo": 0.46893593668937683, |
| "losses/sft": 2.4472603797912598, |
| "losses/total": 0.46893593668937683, |
| "ref_logps/chosen": -38.752357482910156, |
| "ref_logps/rejected": -48.1095085144043, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -0.8374671339988708, |
| "rewards/margins": 0.6724185943603516, |
| "rewards/rejected": -1.5098857879638672, |
| "step": 168 |
| }, |
| { |
| "epoch": 1.28, |
| "grad_norm": 8.055720143262379, |
| "learning_rate": 3.1882022471910107e-07, |
| "logps/chosen": -47.89605712890625, |
| "logps/rejected": -61.83879852294922, |
| "loss": 0.5001, |
| "losses/dpo": 0.5356773138046265, |
| "losses/sft": 2.059915781021118, |
| "losses/total": 0.5356773138046265, |
| "ref_logps/chosen": -38.51155090332031, |
| "ref_logps/rejected": -45.76554870605469, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9384507536888123, |
| "rewards/margins": 0.6688745021820068, |
| "rewards/rejected": -1.6073251962661743, |
| "step": 169 |
| }, |
| { |
| "epoch": 1.28, |
| "grad_norm": 7.586041013117629, |
| "learning_rate": 3.1741573033707863e-07, |
| "logps/chosen": -48.82367706298828, |
| "logps/rejected": -61.73891830444336, |
| "loss": 0.482, |
| "losses/dpo": 0.4217451214790344, |
| "losses/sft": 1.6581342220306396, |
| "losses/total": 0.4217451214790344, |
| "ref_logps/chosen": -40.12367248535156, |
| "ref_logps/rejected": -45.60664367675781, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.870000422000885, |
| "rewards/margins": 0.7432273626327515, |
| "rewards/rejected": -1.6132278442382812, |
| "step": 170 |
| }, |
| { |
| "epoch": 1.29, |
| "grad_norm": 8.642028616636065, |
| "learning_rate": 3.160112359550562e-07, |
| "logps/chosen": -48.638423919677734, |
| "logps/rejected": -60.67471694946289, |
| "loss": 0.5046, |
| "losses/dpo": 0.5264810919761658, |
| "losses/sft": 1.7720236778259277, |
| "losses/total": 0.5264810919761658, |
| "ref_logps/chosen": -39.852603912353516, |
| "ref_logps/rejected": -44.74626159667969, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.8785818815231323, |
| "rewards/margins": 0.7142631411552429, |
| "rewards/rejected": -1.592844843864441, |
| "step": 171 |
| }, |
| { |
| "epoch": 1.3, |
| "grad_norm": 8.335982070287319, |
| "learning_rate": 3.146067415730337e-07, |
| "logps/chosen": -52.76824951171875, |
| "logps/rejected": -63.022315979003906, |
| "loss": 0.5192, |
| "losses/dpo": 0.4350988268852234, |
| "losses/sft": 2.3970541954040527, |
| "losses/total": 0.4350988268852234, |
| "ref_logps/chosen": -42.989315032958984, |
| "ref_logps/rejected": -47.05906677246094, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.9778933525085449, |
| "rewards/margins": 0.6184311509132385, |
| "rewards/rejected": -1.5963245630264282, |
| "step": 172 |
| }, |
| { |
| "epoch": 1.31, |
| "grad_norm": 8.969238567903124, |
| "learning_rate": 3.1320224719101125e-07, |
| "logps/chosen": -50.0399169921875, |
| "logps/rejected": -61.899139404296875, |
| "loss": 0.5552, |
| "losses/dpo": 0.6638925075531006, |
| "losses/sft": 2.114647150039673, |
| "losses/total": 0.6638925075531006, |
| "ref_logps/chosen": -40.658668518066406, |
| "ref_logps/rejected": -46.74747848510742, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.938124418258667, |
| "rewards/margins": 0.5770419239997864, |
| "rewards/rejected": -1.515166163444519, |
| "step": 173 |
| }, |
| { |
| "epoch": 1.31, |
| "grad_norm": 8.235186141597438, |
| "learning_rate": 3.1179775280898875e-07, |
| "logps/chosen": -52.105690002441406, |
| "logps/rejected": -62.85676574707031, |
| "loss": 0.5092, |
| "losses/dpo": 0.7253843545913696, |
| "losses/sft": 2.424346446990967, |
| "losses/total": 0.7253843545913696, |
| "ref_logps/chosen": -43.18442916870117, |
| "ref_logps/rejected": -47.48811340332031, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.8921262621879578, |
| "rewards/margins": 0.6447390913963318, |
| "rewards/rejected": -1.5368653535842896, |
| "step": 174 |
| }, |
| { |
| "epoch": 1.32, |
| "grad_norm": 7.582218244857748, |
| "learning_rate": 3.103932584269663e-07, |
| "logps/chosen": -50.67258071899414, |
| "logps/rejected": -66.15370178222656, |
| "loss": 0.4585, |
| "losses/dpo": 0.45553505420684814, |
| "losses/sft": 1.935417890548706, |
| "losses/total": 0.45553505420684814, |
| "ref_logps/chosen": -41.14632034301758, |
| "ref_logps/rejected": -48.69713592529297, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.9526264667510986, |
| "rewards/margins": 0.7930303812026978, |
| "rewards/rejected": -1.745656967163086, |
| "step": 175 |
| }, |
| { |
| "epoch": 1.33, |
| "grad_norm": 8.60397689359237, |
| "learning_rate": 3.0898876404494376e-07, |
| "logps/chosen": -48.78178405761719, |
| "logps/rejected": -56.637123107910156, |
| "loss": 0.5424, |
| "losses/dpo": 0.46894580125808716, |
| "losses/sft": 1.4491811990737915, |
| "losses/total": 0.46894580125808716, |
| "ref_logps/chosen": -39.89250183105469, |
| "ref_logps/rejected": -41.9540901184082, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.8889281749725342, |
| "rewards/margins": 0.5793753862380981, |
| "rewards/rejected": -1.4683035612106323, |
| "step": 176 |
| }, |
| { |
| "epoch": 1.34, |
| "grad_norm": 8.889595865380464, |
| "learning_rate": 3.075842696629213e-07, |
| "logps/chosen": -54.22248077392578, |
| "logps/rejected": -62.7822380065918, |
| "loss": 0.5381, |
| "losses/dpo": 0.5002489686012268, |
| "losses/sft": 1.6078195571899414, |
| "losses/total": 0.5002489686012268, |
| "ref_logps/chosen": -43.68180847167969, |
| "ref_logps/rejected": -46.31658172607422, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -1.0540671348571777, |
| "rewards/margins": 0.592498779296875, |
| "rewards/rejected": -1.6465659141540527, |
| "step": 177 |
| }, |
| { |
| "epoch": 1.34, |
| "grad_norm": 8.532472841711035, |
| "learning_rate": 3.0617977528089887e-07, |
| "logps/chosen": -49.95827865600586, |
| "logps/rejected": -61.806705474853516, |
| "loss": 0.5407, |
| "losses/dpo": 0.6499188542366028, |
| "losses/sft": 2.202641725540161, |
| "losses/total": 0.6499188542366028, |
| "ref_logps/chosen": -39.29413986206055, |
| "ref_logps/rejected": -44.9674072265625, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -1.0664141178131104, |
| "rewards/margins": 0.6175155639648438, |
| "rewards/rejected": -1.6839298009872437, |
| "step": 178 |
| }, |
| { |
| "epoch": 1.35, |
| "grad_norm": 8.909287859284307, |
| "learning_rate": 3.047752808988764e-07, |
| "logps/chosen": -51.441497802734375, |
| "logps/rejected": -62.11806106567383, |
| "loss": 0.527, |
| "losses/dpo": 0.2888008952140808, |
| "losses/sft": 1.8648221492767334, |
| "losses/total": 0.2888008952140808, |
| "ref_logps/chosen": -41.599273681640625, |
| "ref_logps/rejected": -46.178306579589844, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.9842216372489929, |
| "rewards/margins": 0.6097543239593506, |
| "rewards/rejected": -1.5939760208129883, |
| "step": 179 |
| }, |
| { |
| "epoch": 1.36, |
| "grad_norm": 8.118331116269069, |
| "learning_rate": 3.0337078651685393e-07, |
| "logps/chosen": -50.35320281982422, |
| "logps/rejected": -68.11038208007812, |
| "loss": 0.4825, |
| "losses/dpo": 0.2915097177028656, |
| "losses/sft": 2.2731690406799316, |
| "losses/total": 0.2915097177028656, |
| "ref_logps/chosen": -40.37377166748047, |
| "ref_logps/rejected": -50.12421798706055, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.997943103313446, |
| "rewards/margins": 0.8006736636161804, |
| "rewards/rejected": -1.7986167669296265, |
| "step": 180 |
| }, |
| { |
| "epoch": 1.37, |
| "grad_norm": 7.611558494319546, |
| "learning_rate": 3.0196629213483144e-07, |
| "logps/chosen": -48.98621368408203, |
| "logps/rejected": -63.958377838134766, |
| "loss": 0.4771, |
| "losses/dpo": 0.8142632246017456, |
| "losses/sft": 2.000248670578003, |
| "losses/total": 0.8142632246017456, |
| "ref_logps/chosen": -39.60884475708008, |
| "ref_logps/rejected": -46.56087875366211, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.9377367496490479, |
| "rewards/margins": 0.8020133376121521, |
| "rewards/rejected": -1.7397500276565552, |
| "step": 181 |
| }, |
| { |
| "epoch": 1.37, |
| "grad_norm": 7.87112340864733, |
| "learning_rate": 3.00561797752809e-07, |
| "logps/chosen": -50.70545959472656, |
| "logps/rejected": -60.79212951660156, |
| "loss": 0.481, |
| "losses/dpo": 0.4779345989227295, |
| "losses/sft": 2.002454996109009, |
| "losses/total": 0.4779345989227295, |
| "ref_logps/chosen": -40.73643112182617, |
| "ref_logps/rejected": -43.39889144897461, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -0.9969026446342468, |
| "rewards/margins": 0.7424213886260986, |
| "rewards/rejected": -1.7393240928649902, |
| "step": 182 |
| }, |
| { |
| "epoch": 1.38, |
| "grad_norm": 8.88113336772084, |
| "learning_rate": 2.991573033707865e-07, |
| "logps/chosen": -49.58301544189453, |
| "logps/rejected": -58.122802734375, |
| "loss": 0.5481, |
| "losses/dpo": 0.5509602427482605, |
| "losses/sft": 2.1736414432525635, |
| "losses/total": 0.5509602427482605, |
| "ref_logps/chosen": -39.8292350769043, |
| "ref_logps/rejected": -42.636268615722656, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -0.9753779768943787, |
| "rewards/margins": 0.5732761025428772, |
| "rewards/rejected": -1.5486540794372559, |
| "step": 183 |
| }, |
| { |
| "epoch": 1.39, |
| "grad_norm": 8.086846655989897, |
| "learning_rate": 2.9775280898876406e-07, |
| "logps/chosen": -49.57462692260742, |
| "logps/rejected": -59.01618194580078, |
| "loss": 0.5191, |
| "losses/dpo": 0.41641122102737427, |
| "losses/sft": 1.5094877481460571, |
| "losses/total": 0.41641122102737427, |
| "ref_logps/chosen": -40.067962646484375, |
| "ref_logps/rejected": -43.40235900878906, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.9506663084030151, |
| "rewards/margins": 0.6107163429260254, |
| "rewards/rejected": -1.561382532119751, |
| "step": 184 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 14.80695955125612, |
| "learning_rate": 2.9634831460674156e-07, |
| "logps/chosen": -49.670318603515625, |
| "logps/rejected": -63.068153381347656, |
| "loss": 0.4798, |
| "losses/dpo": 0.71369868516922, |
| "losses/sft": 2.1753716468811035, |
| "losses/total": 0.71369868516922, |
| "ref_logps/chosen": -40.42032241821289, |
| "ref_logps/rejected": -45.94530487060547, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.9249992966651917, |
| "rewards/margins": 0.7872861623764038, |
| "rewards/rejected": -1.7122855186462402, |
| "step": 185 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 7.764172479016813, |
| "learning_rate": 2.9494382022471906e-07, |
| "logps/chosen": -46.467315673828125, |
| "logps/rejected": -61.956661224365234, |
| "loss": 0.4781, |
| "losses/dpo": 0.6086141467094421, |
| "losses/sft": 1.7402431964874268, |
| "losses/total": 0.6086141467094421, |
| "ref_logps/chosen": -37.99009704589844, |
| "ref_logps/rejected": -45.549415588378906, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.8477218747138977, |
| "rewards/margins": 0.7930029630661011, |
| "rewards/rejected": -1.640724778175354, |
| "step": 186 |
| }, |
| { |
| "epoch": 1.41, |
| "grad_norm": 8.22953072067457, |
| "learning_rate": 2.935393258426966e-07, |
| "logps/chosen": -50.86983871459961, |
| "logps/rejected": -62.61697769165039, |
| "loss": 0.5075, |
| "losses/dpo": 0.4520256519317627, |
| "losses/sft": 2.4947710037231445, |
| "losses/total": 0.4520256519317627, |
| "ref_logps/chosen": -40.587074279785156, |
| "ref_logps/rejected": -45.32030487060547, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.0282765626907349, |
| "rewards/margins": 0.7013900279998779, |
| "rewards/rejected": -1.7296665906906128, |
| "step": 187 |
| }, |
| { |
| "epoch": 1.42, |
| "grad_norm": 7.9279440455740735, |
| "learning_rate": 2.921348314606741e-07, |
| "logps/chosen": -44.96842956542969, |
| "logps/rejected": -62.975059509277344, |
| "loss": 0.5187, |
| "losses/dpo": 0.5396404266357422, |
| "losses/sft": 2.048635482788086, |
| "losses/total": 0.5396404266357422, |
| "ref_logps/chosen": -34.73614501953125, |
| "ref_logps/rejected": -46.000450134277344, |
| "rewards/accuracies": 0.7109375, |
| "rewards/chosen": -1.023228406906128, |
| "rewards/margins": 0.6742324829101562, |
| "rewards/rejected": -1.6974608898162842, |
| "step": 188 |
| }, |
| { |
| "epoch": 1.43, |
| "grad_norm": 7.307548456693543, |
| "learning_rate": 2.907303370786517e-07, |
| "logps/chosen": -45.52065658569336, |
| "logps/rejected": -59.8885498046875, |
| "loss": 0.4755, |
| "losses/dpo": 0.40278539061546326, |
| "losses/sft": 2.2951760292053223, |
| "losses/total": 0.40278539061546326, |
| "ref_logps/chosen": -37.46648406982422, |
| "ref_logps/rejected": -44.16557312011719, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.8054174184799194, |
| "rewards/margins": 0.7668801546096802, |
| "rewards/rejected": -1.5722976922988892, |
| "step": 189 |
| }, |
| { |
| "epoch": 1.43, |
| "grad_norm": 8.912502235874074, |
| "learning_rate": 2.893258426966292e-07, |
| "logps/chosen": -45.76905059814453, |
| "logps/rejected": -57.87416076660156, |
| "loss": 0.5655, |
| "losses/dpo": 0.5564082860946655, |
| "losses/sft": 1.3282924890518188, |
| "losses/total": 0.5564082860946655, |
| "ref_logps/chosen": -36.90949630737305, |
| "ref_logps/rejected": -43.09407424926758, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.8859553933143616, |
| "rewards/margins": 0.5920534133911133, |
| "rewards/rejected": -1.47800874710083, |
| "step": 190 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 7.963453038939498, |
| "learning_rate": 2.8792134831460674e-07, |
| "logps/chosen": -48.343929290771484, |
| "logps/rejected": -68.3438720703125, |
| "loss": 0.4613, |
| "losses/dpo": 0.4805658161640167, |
| "losses/sft": 2.2470126152038574, |
| "losses/total": 0.4805658161640167, |
| "ref_logps/chosen": -40.115596771240234, |
| "ref_logps/rejected": -51.82239532470703, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.8228334784507751, |
| "rewards/margins": 0.8293145895004272, |
| "rewards/rejected": -1.6521480083465576, |
| "step": 191 |
| }, |
| { |
| "epoch": 1.45, |
| "grad_norm": 8.622412292262098, |
| "learning_rate": 2.8651685393258425e-07, |
| "logps/chosen": -51.2635383605957, |
| "logps/rejected": -63.78586959838867, |
| "loss": 0.4851, |
| "losses/dpo": 0.38889288902282715, |
| "losses/sft": 2.7701683044433594, |
| "losses/total": 0.38889288902282715, |
| "ref_logps/chosen": -42.14338684082031, |
| "ref_logps/rejected": -46.94654083251953, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.9120146036148071, |
| "rewards/margins": 0.7719184756278992, |
| "rewards/rejected": -1.683933138847351, |
| "step": 192 |
| }, |
| { |
| "epoch": 1.46, |
| "grad_norm": 7.759600823991896, |
| "learning_rate": 2.851123595505618e-07, |
| "logps/chosen": -46.17103958129883, |
| "logps/rejected": -60.57073974609375, |
| "loss": 0.5056, |
| "losses/dpo": 0.6096426248550415, |
| "losses/sft": 2.101557493209839, |
| "losses/total": 0.6096426248550415, |
| "ref_logps/chosen": -37.342018127441406, |
| "ref_logps/rejected": -44.97611999511719, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.8829020261764526, |
| "rewards/margins": 0.6765601634979248, |
| "rewards/rejected": -1.559462308883667, |
| "step": 193 |
| }, |
| { |
| "epoch": 1.46, |
| "grad_norm": 8.401972111307101, |
| "learning_rate": 2.8370786516853936e-07, |
| "logps/chosen": -49.945350646972656, |
| "logps/rejected": -64.91703033447266, |
| "loss": 0.4635, |
| "losses/dpo": 0.5600602626800537, |
| "losses/sft": 1.5397871732711792, |
| "losses/total": 0.5600602626800537, |
| "ref_logps/chosen": -41.32893371582031, |
| "ref_logps/rejected": -47.677955627441406, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.8616417050361633, |
| "rewards/margins": 0.8622665405273438, |
| "rewards/rejected": -1.7239081859588623, |
| "step": 194 |
| }, |
| { |
| "epoch": 1.47, |
| "grad_norm": 8.847895232096153, |
| "learning_rate": 2.823033707865168e-07, |
| "logps/chosen": -45.08359146118164, |
| "logps/rejected": -65.03630828857422, |
| "loss": 0.5214, |
| "losses/dpo": 0.5835427045822144, |
| "losses/sft": 1.4693069458007812, |
| "losses/total": 0.5835427045822144, |
| "ref_logps/chosen": -35.359466552734375, |
| "ref_logps/rejected": -47.392391204833984, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9724127650260925, |
| "rewards/margins": 0.7919799089431763, |
| "rewards/rejected": -1.764392614364624, |
| "step": 195 |
| }, |
| { |
| "epoch": 1.48, |
| "grad_norm": 8.759081047535012, |
| "learning_rate": 2.8089887640449437e-07, |
| "logps/chosen": -45.32444763183594, |
| "logps/rejected": -63.420326232910156, |
| "loss": 0.5066, |
| "losses/dpo": 0.38791435956954956, |
| "losses/sft": 1.907132625579834, |
| "losses/total": 0.38791435956954956, |
| "ref_logps/chosen": -35.994544982910156, |
| "ref_logps/rejected": -46.32176971435547, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.932990550994873, |
| "rewards/margins": 0.776865541934967, |
| "rewards/rejected": -1.7098561525344849, |
| "step": 196 |
| }, |
| { |
| "epoch": 1.49, |
| "grad_norm": 8.54168900663033, |
| "learning_rate": 2.794943820224719e-07, |
| "logps/chosen": -53.67341613769531, |
| "logps/rejected": -73.58582305908203, |
| "loss": 0.4837, |
| "losses/dpo": 0.5046126842498779, |
| "losses/sft": 2.1525635719299316, |
| "losses/total": 0.5046126842498779, |
| "ref_logps/chosen": -42.92031478881836, |
| "ref_logps/rejected": -54.27576446533203, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0753098726272583, |
| "rewards/margins": 0.8556962013244629, |
| "rewards/rejected": -1.9310060739517212, |
| "step": 197 |
| }, |
| { |
| "epoch": 1.49, |
| "grad_norm": 9.360560561178314, |
| "learning_rate": 2.7808988764044943e-07, |
| "logps/chosen": -49.29429626464844, |
| "logps/rejected": -58.852745056152344, |
| "loss": 0.5219, |
| "losses/dpo": 0.6207550168037415, |
| "losses/sft": 1.4964567422866821, |
| "losses/total": 0.6207550168037415, |
| "ref_logps/chosen": -39.879859924316406, |
| "ref_logps/rejected": -42.66207504272461, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.9414433836936951, |
| "rewards/margins": 0.6776232719421387, |
| "rewards/rejected": -1.6190667152404785, |
| "step": 198 |
| }, |
| { |
| "epoch": 1.5, |
| "grad_norm": 7.766094681277619, |
| "learning_rate": 2.7668539325842694e-07, |
| "logps/chosen": -44.67707443237305, |
| "logps/rejected": -62.46302032470703, |
| "loss": 0.4513, |
| "losses/dpo": 0.3586122393608093, |
| "losses/sft": 1.7712942361831665, |
| "losses/total": 0.3586122393608093, |
| "ref_logps/chosen": -35.756446838378906, |
| "ref_logps/rejected": -45.0746955871582, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.8920624256134033, |
| "rewards/margins": 0.8467705249786377, |
| "rewards/rejected": -1.738832950592041, |
| "step": 199 |
| }, |
| { |
| "epoch": 1.51, |
| "grad_norm": 8.387391054319354, |
| "learning_rate": 2.752808988764045e-07, |
| "logps/chosen": -51.25703430175781, |
| "logps/rejected": -66.00263977050781, |
| "loss": 0.4694, |
| "losses/dpo": 0.4337347447872162, |
| "losses/sft": 2.2087419033050537, |
| "losses/total": 0.4337347447872162, |
| "ref_logps/chosen": -41.61771011352539, |
| "ref_logps/rejected": -48.16603469848633, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.96393221616745, |
| "rewards/margins": 0.8197280764579773, |
| "rewards/rejected": -1.7836604118347168, |
| "step": 200 |
| }, |
| { |
| "epoch": 1.52, |
| "grad_norm": 8.762356246957351, |
| "learning_rate": 2.73876404494382e-07, |
| "logps/chosen": -48.135372161865234, |
| "logps/rejected": -59.56908416748047, |
| "loss": 0.5349, |
| "losses/dpo": 0.5300882458686829, |
| "losses/sft": 1.96072256565094, |
| "losses/total": 0.5300882458686829, |
| "ref_logps/chosen": -39.096519470214844, |
| "ref_logps/rejected": -44.62074279785156, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.9038856625556946, |
| "rewards/margins": 0.5909484028816223, |
| "rewards/rejected": -1.4948341846466064, |
| "step": 201 |
| }, |
| { |
| "epoch": 1.52, |
| "grad_norm": 8.01612481161353, |
| "learning_rate": 2.7247191011235955e-07, |
| "logps/chosen": -48.03499984741211, |
| "logps/rejected": -58.95391845703125, |
| "loss": 0.4596, |
| "losses/dpo": 0.5687890648841858, |
| "losses/sft": 1.5117721557617188, |
| "losses/total": 0.5687890648841858, |
| "ref_logps/chosen": -40.076637268066406, |
| "ref_logps/rejected": -42.60528564453125, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -0.795836865901947, |
| "rewards/margins": 0.839026153087616, |
| "rewards/rejected": -1.634863018989563, |
| "step": 202 |
| }, |
| { |
| "epoch": 1.53, |
| "grad_norm": 8.170779294474082, |
| "learning_rate": 2.710674157303371e-07, |
| "logps/chosen": -53.97732925415039, |
| "logps/rejected": -66.41366577148438, |
| "loss": 0.4697, |
| "losses/dpo": 0.440776526927948, |
| "losses/sft": 2.09440279006958, |
| "losses/total": 0.440776526927948, |
| "ref_logps/chosen": -44.04298782348633, |
| "ref_logps/rejected": -48.34318161010742, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -0.9934341311454773, |
| "rewards/margins": 0.8136138319969177, |
| "rewards/rejected": -1.8070482015609741, |
| "step": 203 |
| }, |
| { |
| "epoch": 1.54, |
| "grad_norm": 8.062617165532144, |
| "learning_rate": 2.6966292134831456e-07, |
| "logps/chosen": -51.31819152832031, |
| "logps/rejected": -66.41058349609375, |
| "loss": 0.4809, |
| "losses/dpo": 0.5263036489486694, |
| "losses/sft": 1.6697288751602173, |
| "losses/total": 0.5263036489486694, |
| "ref_logps/chosen": -41.211219787597656, |
| "ref_logps/rejected": -48.88929748535156, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0106972455978394, |
| "rewards/margins": 0.7414315938949585, |
| "rewards/rejected": -1.7521288394927979, |
| "step": 204 |
| }, |
| { |
| "epoch": 1.55, |
| "grad_norm": 8.232167547461342, |
| "learning_rate": 2.682584269662921e-07, |
| "logps/chosen": -50.255958557128906, |
| "logps/rejected": -64.30512237548828, |
| "loss": 0.4906, |
| "losses/dpo": 0.5959848761558533, |
| "losses/sft": 2.509490966796875, |
| "losses/total": 0.5959848761558533, |
| "ref_logps/chosen": -40.21955490112305, |
| "ref_logps/rejected": -46.348358154296875, |
| "rewards/accuracies": 0.7578125, |
| "rewards/chosen": -1.0036402940750122, |
| "rewards/margins": 0.7920358180999756, |
| "rewards/rejected": -1.7956762313842773, |
| "step": 205 |
| }, |
| { |
| "epoch": 1.55, |
| "grad_norm": 8.658885149622282, |
| "learning_rate": 2.668539325842696e-07, |
| "logps/chosen": -48.23283004760742, |
| "logps/rejected": -63.8531608581543, |
| "loss": 0.5012, |
| "losses/dpo": 0.3958526849746704, |
| "losses/sft": 1.8156052827835083, |
| "losses/total": 0.3958526849746704, |
| "ref_logps/chosen": -39.19361877441406, |
| "ref_logps/rejected": -46.780818939208984, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9039209485054016, |
| "rewards/margins": 0.8033130168914795, |
| "rewards/rejected": -1.7072339057922363, |
| "step": 206 |
| }, |
| { |
| "epoch": 1.56, |
| "grad_norm": 9.062222930248339, |
| "learning_rate": 2.654494382022472e-07, |
| "logps/chosen": -48.18266296386719, |
| "logps/rejected": -60.7551155090332, |
| "loss": 0.5163, |
| "losses/dpo": 0.5344985127449036, |
| "losses/sft": 1.5162503719329834, |
| "losses/total": 0.5344985127449036, |
| "ref_logps/chosen": -38.624839782714844, |
| "ref_logps/rejected": -43.208770751953125, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.9557825326919556, |
| "rewards/margins": 0.798852264881134, |
| "rewards/rejected": -1.7546348571777344, |
| "step": 207 |
| }, |
| { |
| "epoch": 1.57, |
| "grad_norm": 8.337997292070591, |
| "learning_rate": 2.640449438202247e-07, |
| "logps/chosen": -46.034690856933594, |
| "logps/rejected": -58.21810531616211, |
| "loss": 0.5159, |
| "losses/dpo": 0.5175392627716064, |
| "losses/sft": 1.6583523750305176, |
| "losses/total": 0.5175392627716064, |
| "ref_logps/chosen": -37.22336959838867, |
| "ref_logps/rejected": -42.30153274536133, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.8811318278312683, |
| "rewards/margins": 0.7105254530906677, |
| "rewards/rejected": -1.5916571617126465, |
| "step": 208 |
| }, |
| { |
| "epoch": 1.58, |
| "grad_norm": 8.605777081243177, |
| "learning_rate": 2.6264044943820224e-07, |
| "logps/chosen": -49.07252502441406, |
| "logps/rejected": -62.991764068603516, |
| "loss": 0.5234, |
| "losses/dpo": 0.3841923177242279, |
| "losses/sft": 2.0660533905029297, |
| "losses/total": 0.3841923177242279, |
| "ref_logps/chosen": -38.91202163696289, |
| "ref_logps/rejected": -45.201744079589844, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -1.0160505771636963, |
| "rewards/margins": 0.7629517316818237, |
| "rewards/rejected": -1.7790021896362305, |
| "step": 209 |
| }, |
| { |
| "epoch": 1.58, |
| "grad_norm": 8.594902163030543, |
| "learning_rate": 2.612359550561798e-07, |
| "logps/chosen": -48.94342803955078, |
| "logps/rejected": -67.45938110351562, |
| "loss": 0.4628, |
| "losses/dpo": 0.5517236590385437, |
| "losses/sft": 1.831754446029663, |
| "losses/total": 0.5517236590385437, |
| "ref_logps/chosen": -39.35862731933594, |
| "ref_logps/rejected": -48.98806381225586, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.9584797620773315, |
| "rewards/margins": 0.8886520862579346, |
| "rewards/rejected": -1.8471317291259766, |
| "step": 210 |
| }, |
| { |
| "epoch": 1.59, |
| "grad_norm": 9.393987371477554, |
| "learning_rate": 2.598314606741573e-07, |
| "logps/chosen": -51.1754035949707, |
| "logps/rejected": -62.90049362182617, |
| "loss": 0.538, |
| "losses/dpo": 0.4337689280509949, |
| "losses/sft": 1.871553897857666, |
| "losses/total": 0.4337689280509949, |
| "ref_logps/chosen": -40.097930908203125, |
| "ref_logps/rejected": -44.89445877075195, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -1.1077474355697632, |
| "rewards/margins": 0.6928560137748718, |
| "rewards/rejected": -1.8006032705307007, |
| "step": 211 |
| }, |
| { |
| "epoch": 1.6, |
| "grad_norm": 8.268094387246107, |
| "learning_rate": 2.5842696629213486e-07, |
| "logps/chosen": -46.02891159057617, |
| "logps/rejected": -58.77110290527344, |
| "loss": 0.5183, |
| "losses/dpo": 0.3652556836605072, |
| "losses/sft": 2.1132707595825195, |
| "losses/total": 0.3652556836605072, |
| "ref_logps/chosen": -37.71674346923828, |
| "ref_logps/rejected": -43.298362731933594, |
| "rewards/accuracies": 0.734375, |
| "rewards/chosen": -0.8312174677848816, |
| "rewards/margins": 0.7160569429397583, |
| "rewards/rejected": -1.5472744703292847, |
| "step": 212 |
| }, |
| { |
| "epoch": 1.61, |
| "grad_norm": 8.156887652455024, |
| "learning_rate": 2.5702247191011236e-07, |
| "logps/chosen": -53.53913879394531, |
| "logps/rejected": -65.96989440917969, |
| "loss": 0.454, |
| "losses/dpo": 0.6029412746429443, |
| "losses/sft": 2.5100276470184326, |
| "losses/total": 0.6029412746429443, |
| "ref_logps/chosen": -43.845726013183594, |
| "ref_logps/rejected": -47.28349304199219, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.9693412780761719, |
| "rewards/margins": 0.899299681186676, |
| "rewards/rejected": -1.8686408996582031, |
| "step": 213 |
| }, |
| { |
| "epoch": 1.62, |
| "grad_norm": 9.181481809417177, |
| "learning_rate": 2.5561797752808987e-07, |
| "logps/chosen": -51.8013801574707, |
| "logps/rejected": -63.46139907836914, |
| "loss": 0.5059, |
| "losses/dpo": 0.14845682680606842, |
| "losses/sft": 1.4499884843826294, |
| "losses/total": 0.14845682680606842, |
| "ref_logps/chosen": -42.47813415527344, |
| "ref_logps/rejected": -46.913028717041016, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.9323242902755737, |
| "rewards/margins": 0.7225131988525391, |
| "rewards/rejected": -1.6548374891281128, |
| "step": 214 |
| }, |
| { |
| "epoch": 1.62, |
| "grad_norm": 15.612026068166484, |
| "learning_rate": 2.5421348314606737e-07, |
| "logps/chosen": -53.003204345703125, |
| "logps/rejected": -66.69940948486328, |
| "loss": 0.4433, |
| "losses/dpo": 0.37596985697746277, |
| "losses/sft": 1.6896804571151733, |
| "losses/total": 0.37596985697746277, |
| "ref_logps/chosen": -43.648799896240234, |
| "ref_logps/rejected": -48.52040481567383, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -0.9354407787322998, |
| "rewards/margins": 0.8824598789215088, |
| "rewards/rejected": -1.8179006576538086, |
| "step": 215 |
| }, |
| { |
| "epoch": 1.63, |
| "grad_norm": 9.017380971745943, |
| "learning_rate": 2.5280898876404493e-07, |
| "logps/chosen": -53.18143081665039, |
| "logps/rejected": -68.83674621582031, |
| "loss": 0.5005, |
| "losses/dpo": 0.9546025395393372, |
| "losses/sft": 2.2383298873901367, |
| "losses/total": 0.9546025395393372, |
| "ref_logps/chosen": -42.14701843261719, |
| "ref_logps/rejected": -50.13629913330078, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.103441834449768, |
| "rewards/margins": 0.7666029334068298, |
| "rewards/rejected": -1.8700445890426636, |
| "step": 216 |
| }, |
| { |
| "epoch": 1.64, |
| "grad_norm": 7.002144270086572, |
| "learning_rate": 2.5140449438202243e-07, |
| "logps/chosen": -39.812992095947266, |
| "logps/rejected": -58.04911804199219, |
| "loss": 0.4312, |
| "losses/dpo": 0.3237588107585907, |
| "losses/sft": 1.9024207592010498, |
| "losses/total": 0.3237588107585907, |
| "ref_logps/chosen": -31.948955535888672, |
| "ref_logps/rejected": -40.56779479980469, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -0.7864038348197937, |
| "rewards/margins": 0.9617283940315247, |
| "rewards/rejected": -1.7481321096420288, |
| "step": 217 |
| }, |
| { |
| "epoch": 1.65, |
| "grad_norm": 8.018556807115827, |
| "learning_rate": 2.5e-07, |
| "logps/chosen": -47.59326934814453, |
| "logps/rejected": -64.22843170166016, |
| "loss": 0.4616, |
| "losses/dpo": 0.3011893630027771, |
| "losses/sft": 1.4501286745071411, |
| "losses/total": 0.3011893630027771, |
| "ref_logps/chosen": -37.48702621459961, |
| "ref_logps/rejected": -45.30727767944336, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.0106242895126343, |
| "rewards/margins": 0.8814913630485535, |
| "rewards/rejected": -1.892115592956543, |
| "step": 218 |
| }, |
| { |
| "epoch": 1.65, |
| "grad_norm": 9.421297346982232, |
| "learning_rate": 2.485955056179775e-07, |
| "logps/chosen": -52.299922943115234, |
| "logps/rejected": -62.2135009765625, |
| "loss": 0.5647, |
| "losses/dpo": 0.6175757050514221, |
| "losses/sft": 2.058591365814209, |
| "losses/total": 0.6175757050514221, |
| "ref_logps/chosen": -41.575767517089844, |
| "ref_logps/rejected": -45.39363479614258, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -1.0724154710769653, |
| "rewards/margins": 0.6095717549324036, |
| "rewards/rejected": -1.6819872856140137, |
| "step": 219 |
| }, |
| { |
| "epoch": 1.66, |
| "grad_norm": 10.845286708552022, |
| "learning_rate": 2.4719101123595505e-07, |
| "logps/chosen": -49.824981689453125, |
| "logps/rejected": -61.322139739990234, |
| "loss": 0.4823, |
| "losses/dpo": 0.5226192474365234, |
| "losses/sft": 1.8831043243408203, |
| "losses/total": 0.5226192474365234, |
| "ref_logps/chosen": -39.458709716796875, |
| "ref_logps/rejected": -43.33076477050781, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.0366266965866089, |
| "rewards/margins": 0.7625109553337097, |
| "rewards/rejected": -1.7991377115249634, |
| "step": 220 |
| }, |
| { |
| "epoch": 1.67, |
| "grad_norm": 7.736788176006532, |
| "learning_rate": 2.4578651685393255e-07, |
| "logps/chosen": -48.30406951904297, |
| "logps/rejected": -62.53293228149414, |
| "loss": 0.4473, |
| "losses/dpo": 0.24700236320495605, |
| "losses/sft": 1.5504862070083618, |
| "losses/total": 0.24700236320495605, |
| "ref_logps/chosen": -38.438228607177734, |
| "ref_logps/rejected": -43.876705169677734, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -0.9865838289260864, |
| "rewards/margins": 0.8790390491485596, |
| "rewards/rejected": -1.8656229972839355, |
| "step": 221 |
| }, |
| { |
| "epoch": 1.68, |
| "grad_norm": 7.9918716943613894, |
| "learning_rate": 2.443820224719101e-07, |
| "logps/chosen": -46.76765441894531, |
| "logps/rejected": -63.072418212890625, |
| "loss": 0.4597, |
| "losses/dpo": 0.5004298686981201, |
| "losses/sft": 2.039869785308838, |
| "losses/total": 0.5004298686981201, |
| "ref_logps/chosen": -37.515079498291016, |
| "ref_logps/rejected": -44.76315689086914, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.9252572655677795, |
| "rewards/margins": 0.9056685566902161, |
| "rewards/rejected": -1.8309259414672852, |
| "step": 222 |
| }, |
| { |
| "epoch": 1.68, |
| "grad_norm": 8.812574527536418, |
| "learning_rate": 2.429775280898876e-07, |
| "logps/chosen": -48.68953323364258, |
| "logps/rejected": -70.00579071044922, |
| "loss": 0.4842, |
| "losses/dpo": 0.5064282417297363, |
| "losses/sft": 1.860622763633728, |
| "losses/total": 0.5064282417297363, |
| "ref_logps/chosen": -37.93241500854492, |
| "ref_logps/rejected": -49.59477996826172, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -1.0757118463516235, |
| "rewards/margins": 0.9653894305229187, |
| "rewards/rejected": -2.0411009788513184, |
| "step": 223 |
| }, |
| { |
| "epoch": 1.69, |
| "grad_norm": 8.262706110631338, |
| "learning_rate": 2.4157303370786517e-07, |
| "logps/chosen": -51.853031158447266, |
| "logps/rejected": -66.55961608886719, |
| "loss": 0.4643, |
| "losses/dpo": 0.32682162523269653, |
| "losses/sft": 2.5111522674560547, |
| "losses/total": 0.32682162523269653, |
| "ref_logps/chosen": -42.025306701660156, |
| "ref_logps/rejected": -47.904380798339844, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -0.9827719330787659, |
| "rewards/margins": 0.8827516436576843, |
| "rewards/rejected": -1.8655235767364502, |
| "step": 224 |
| }, |
| { |
| "epoch": 1.7, |
| "grad_norm": 9.91340992789526, |
| "learning_rate": 2.401685393258427e-07, |
| "logps/chosen": -51.620609283447266, |
| "logps/rejected": -61.01588821411133, |
| "loss": 0.5382, |
| "losses/dpo": 0.47428151965141296, |
| "losses/sft": 1.9888023138046265, |
| "losses/total": 0.47428151965141296, |
| "ref_logps/chosen": -40.93183135986328, |
| "ref_logps/rejected": -43.67429733276367, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -1.0688778162002563, |
| "rewards/margins": 0.6652814745903015, |
| "rewards/rejected": -1.7341593503952026, |
| "step": 225 |
| }, |
| { |
| "epoch": 1.71, |
| "grad_norm": 8.353660910725022, |
| "learning_rate": 2.3876404494382023e-07, |
| "logps/chosen": -49.402103424072266, |
| "logps/rejected": -61.825992584228516, |
| "loss": 0.4796, |
| "losses/dpo": 0.4575900137424469, |
| "losses/sft": 2.0813965797424316, |
| "losses/total": 0.4575900137424469, |
| "ref_logps/chosen": -39.205528259277344, |
| "ref_logps/rejected": -43.332237243652344, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.019657850265503, |
| "rewards/margins": 0.8297175765037537, |
| "rewards/rejected": -1.8493753671646118, |
| "step": 226 |
| }, |
| { |
| "epoch": 1.71, |
| "grad_norm": 8.489694346068541, |
| "learning_rate": 2.3735955056179774e-07, |
| "logps/chosen": -49.869972229003906, |
| "logps/rejected": -61.926387786865234, |
| "loss": 0.5007, |
| "losses/dpo": 0.5196319222450256, |
| "losses/sft": 2.3197567462921143, |
| "losses/total": 0.5196319222450256, |
| "ref_logps/chosen": -40.345924377441406, |
| "ref_logps/rejected": -43.80282211303711, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -0.9524051547050476, |
| "rewards/margins": 0.8599514365196228, |
| "rewards/rejected": -1.8123565912246704, |
| "step": 227 |
| }, |
| { |
| "epoch": 1.72, |
| "grad_norm": 8.455711622023179, |
| "learning_rate": 2.3595505617977527e-07, |
| "logps/chosen": -48.83377456665039, |
| "logps/rejected": -62.39826202392578, |
| "loss": 0.4853, |
| "losses/dpo": 0.33744513988494873, |
| "losses/sft": 2.174797296524048, |
| "losses/total": 0.33744513988494873, |
| "ref_logps/chosen": -38.7586555480957, |
| "ref_logps/rejected": -44.35865783691406, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0075117349624634, |
| "rewards/margins": 0.7964487075805664, |
| "rewards/rejected": -1.8039604425430298, |
| "step": 228 |
| }, |
| { |
| "epoch": 1.73, |
| "grad_norm": 8.501384419530382, |
| "learning_rate": 2.345505617977528e-07, |
| "logps/chosen": -46.079429626464844, |
| "logps/rejected": -62.38417053222656, |
| "loss": 0.4593, |
| "losses/dpo": 0.45904022455215454, |
| "losses/sft": 2.14939546585083, |
| "losses/total": 0.45904022455215454, |
| "ref_logps/chosen": -37.05137634277344, |
| "ref_logps/rejected": -44.551025390625, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.9028058052062988, |
| "rewards/margins": 0.8805083632469177, |
| "rewards/rejected": -1.7833141088485718, |
| "step": 229 |
| }, |
| { |
| "epoch": 1.74, |
| "grad_norm": 8.42568627673927, |
| "learning_rate": 2.331460674157303e-07, |
| "logps/chosen": -53.63593292236328, |
| "logps/rejected": -67.56592559814453, |
| "loss": 0.4141, |
| "losses/dpo": 0.35481470823287964, |
| "losses/sft": 1.9238269329071045, |
| "losses/total": 0.35481470823287964, |
| "ref_logps/chosen": -43.60770797729492, |
| "ref_logps/rejected": -47.71710205078125, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.0028222799301147, |
| "rewards/margins": 0.9820606112480164, |
| "rewards/rejected": -1.9848829507827759, |
| "step": 230 |
| }, |
| { |
| "epoch": 1.74, |
| "grad_norm": 8.985097587921354, |
| "learning_rate": 2.3174157303370786e-07, |
| "logps/chosen": -49.890769958496094, |
| "logps/rejected": -64.36235046386719, |
| "loss": 0.5005, |
| "losses/dpo": 0.4699353277683258, |
| "losses/sft": 1.5774719715118408, |
| "losses/total": 0.4699353277683258, |
| "ref_logps/chosen": -39.884613037109375, |
| "ref_logps/rejected": -46.45123291015625, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0006158351898193, |
| "rewards/margins": 0.7904958724975586, |
| "rewards/rejected": -1.7911114692687988, |
| "step": 231 |
| }, |
| { |
| "epoch": 1.75, |
| "grad_norm": 8.37260934759885, |
| "learning_rate": 2.303370786516854e-07, |
| "logps/chosen": -47.88700485229492, |
| "logps/rejected": -65.87992095947266, |
| "loss": 0.4563, |
| "losses/dpo": 0.4289396107196808, |
| "losses/sft": 1.9490468502044678, |
| "losses/total": 0.4289396107196808, |
| "ref_logps/chosen": -38.816673278808594, |
| "ref_logps/rejected": -48.137718200683594, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.9070336222648621, |
| "rewards/margins": 0.8671874403953552, |
| "rewards/rejected": -1.7742209434509277, |
| "step": 232 |
| }, |
| { |
| "epoch": 1.76, |
| "grad_norm": 8.439180796005859, |
| "learning_rate": 2.2893258426966292e-07, |
| "logps/chosen": -47.75567626953125, |
| "logps/rejected": -59.802364349365234, |
| "loss": 0.5339, |
| "losses/dpo": 0.3888584077358246, |
| "losses/sft": 1.453101396560669, |
| "losses/total": 0.3888584077358246, |
| "ref_logps/chosen": -38.37287521362305, |
| "ref_logps/rejected": -43.31857681274414, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.9382801055908203, |
| "rewards/margins": 0.7100984454154968, |
| "rewards/rejected": -1.6483784914016724, |
| "step": 233 |
| }, |
| { |
| "epoch": 1.77, |
| "grad_norm": 8.820079708795298, |
| "learning_rate": 2.2752808988764045e-07, |
| "logps/chosen": -53.505958557128906, |
| "logps/rejected": -66.17211151123047, |
| "loss": 0.4792, |
| "losses/dpo": 0.5977008938789368, |
| "losses/sft": 1.5070393085479736, |
| "losses/total": 0.5977008938789368, |
| "ref_logps/chosen": -42.90943145751953, |
| "ref_logps/rejected": -47.009498596191406, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.0596526861190796, |
| "rewards/margins": 0.8566086292266846, |
| "rewards/rejected": -1.9162614345550537, |
| "step": 234 |
| }, |
| { |
| "epoch": 1.77, |
| "grad_norm": 8.954294680805216, |
| "learning_rate": 2.2612359550561795e-07, |
| "logps/chosen": -51.98886489868164, |
| "logps/rejected": -65.2154769897461, |
| "loss": 0.4863, |
| "losses/dpo": 0.4061537981033325, |
| "losses/sft": 1.7505851984024048, |
| "losses/total": 0.4061537981033325, |
| "ref_logps/chosen": -41.45209503173828, |
| "ref_logps/rejected": -46.05805969238281, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.0536775588989258, |
| "rewards/margins": 0.8620648980140686, |
| "rewards/rejected": -1.9157423973083496, |
| "step": 235 |
| }, |
| { |
| "epoch": 1.78, |
| "grad_norm": 8.276096992832155, |
| "learning_rate": 2.2471910112359549e-07, |
| "logps/chosen": -46.52098846435547, |
| "logps/rejected": -60.28398895263672, |
| "loss": 0.4849, |
| "losses/dpo": 0.4493502676486969, |
| "losses/sft": 1.8399590253829956, |
| "losses/total": 0.4493502676486969, |
| "ref_logps/chosen": -37.41412353515625, |
| "ref_logps/rejected": -42.95138931274414, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.9106867909431458, |
| "rewards/margins": 0.8225734233856201, |
| "rewards/rejected": -1.733260154724121, |
| "step": 236 |
| }, |
| { |
| "epoch": 1.79, |
| "grad_norm": 9.148667637325849, |
| "learning_rate": 2.2331460674157302e-07, |
| "logps/chosen": -49.735443115234375, |
| "logps/rejected": -68.21788787841797, |
| "loss": 0.49, |
| "losses/dpo": 0.2873924672603607, |
| "losses/sft": 1.5266362428665161, |
| "losses/total": 0.2873924672603607, |
| "ref_logps/chosen": -39.63804244995117, |
| "ref_logps/rejected": -49.29356384277344, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0097399950027466, |
| "rewards/margins": 0.8826919198036194, |
| "rewards/rejected": -1.8924317359924316, |
| "step": 237 |
| }, |
| { |
| "epoch": 1.8, |
| "grad_norm": 8.143852894216051, |
| "learning_rate": 2.2191011235955055e-07, |
| "logps/chosen": -48.24225616455078, |
| "logps/rejected": -63.4071044921875, |
| "loss": 0.4505, |
| "losses/dpo": 0.4800674617290497, |
| "losses/sft": 1.840663194656372, |
| "losses/total": 0.4800674617290497, |
| "ref_logps/chosen": -38.12638854980469, |
| "ref_logps/rejected": -44.40414047241211, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.0115869045257568, |
| "rewards/margins": 0.8887090682983398, |
| "rewards/rejected": -1.9002958536148071, |
| "step": 238 |
| }, |
| { |
| "epoch": 1.8, |
| "grad_norm": 8.978669612809192, |
| "learning_rate": 2.205056179775281e-07, |
| "logps/chosen": -49.006935119628906, |
| "logps/rejected": -64.8754653930664, |
| "loss": 0.4701, |
| "losses/dpo": 0.3446974754333496, |
| "losses/sft": 1.800228476524353, |
| "losses/total": 0.3446974754333496, |
| "ref_logps/chosen": -39.40437698364258, |
| "ref_logps/rejected": -46.17237091064453, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9602562189102173, |
| "rewards/margins": 0.9100530743598938, |
| "rewards/rejected": -1.8703094720840454, |
| "step": 239 |
| }, |
| { |
| "epoch": 1.81, |
| "grad_norm": 9.39799976416451, |
| "learning_rate": 2.191011235955056e-07, |
| "logps/chosen": -49.0966682434082, |
| "logps/rejected": -62.466331481933594, |
| "loss": 0.4979, |
| "losses/dpo": 0.7114862203598022, |
| "losses/sft": 1.6450343132019043, |
| "losses/total": 0.7114862203598022, |
| "ref_logps/chosen": -38.396942138671875, |
| "ref_logps/rejected": -43.12867736816406, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -1.0699726343154907, |
| "rewards/margins": 0.8637927770614624, |
| "rewards/rejected": -1.9337654113769531, |
| "step": 240 |
| }, |
| { |
| "epoch": 1.82, |
| "grad_norm": 7.9075518471198585, |
| "learning_rate": 2.1769662921348314e-07, |
| "logps/chosen": -49.44485092163086, |
| "logps/rejected": -64.67866516113281, |
| "loss": 0.4424, |
| "losses/dpo": 0.3071708679199219, |
| "losses/sft": 2.3602986335754395, |
| "losses/total": 0.3071708679199219, |
| "ref_logps/chosen": -39.44845199584961, |
| "ref_logps/rejected": -45.48222351074219, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -0.999640703201294, |
| "rewards/margins": 0.9200041890144348, |
| "rewards/rejected": -1.919644832611084, |
| "step": 241 |
| }, |
| { |
| "epoch": 1.83, |
| "grad_norm": 8.785359064590306, |
| "learning_rate": 2.1629213483146067e-07, |
| "logps/chosen": -53.305233001708984, |
| "logps/rejected": -65.16683959960938, |
| "loss": 0.4948, |
| "losses/dpo": 0.5694843530654907, |
| "losses/sft": 1.6856664419174194, |
| "losses/total": 0.5694843530654907, |
| "ref_logps/chosen": -42.595603942871094, |
| "ref_logps/rejected": -46.201744079589844, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.0709630250930786, |
| "rewards/margins": 0.8255467414855957, |
| "rewards/rejected": -1.8965098857879639, |
| "step": 242 |
| }, |
| { |
| "epoch": 1.83, |
| "grad_norm": 9.446502314051532, |
| "learning_rate": 2.148876404494382e-07, |
| "logps/chosen": -48.44091033935547, |
| "logps/rejected": -60.52621078491211, |
| "loss": 0.526, |
| "losses/dpo": 0.45424705743789673, |
| "losses/sft": 2.151371479034424, |
| "losses/total": 0.45424705743789673, |
| "ref_logps/chosen": -38.4775390625, |
| "ref_logps/rejected": -43.26105499267578, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.9963367581367493, |
| "rewards/margins": 0.730178952217102, |
| "rewards/rejected": -1.726515531539917, |
| "step": 243 |
| }, |
| { |
| "epoch": 1.84, |
| "grad_norm": 8.81696350872476, |
| "learning_rate": 2.134831460674157e-07, |
| "logps/chosen": -50.6672477722168, |
| "logps/rejected": -66.28202819824219, |
| "loss": 0.4731, |
| "losses/dpo": 0.3640851080417633, |
| "losses/sft": 1.5808416604995728, |
| "losses/total": 0.3640851080417633, |
| "ref_logps/chosen": -40.499908447265625, |
| "ref_logps/rejected": -47.59101486206055, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0167338848114014, |
| "rewards/margins": 0.852367103099823, |
| "rewards/rejected": -1.8691009283065796, |
| "step": 244 |
| }, |
| { |
| "epoch": 1.85, |
| "grad_norm": 9.28428850947705, |
| "learning_rate": 2.1207865168539323e-07, |
| "logps/chosen": -54.07783889770508, |
| "logps/rejected": -65.76669311523438, |
| "loss": 0.5163, |
| "losses/dpo": 0.8544118404388428, |
| "losses/sft": 1.9635812044143677, |
| "losses/total": 0.8544118404388428, |
| "ref_logps/chosen": -42.8679084777832, |
| "ref_logps/rejected": -46.79513931274414, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -1.120992660522461, |
| "rewards/margins": 0.7761632204055786, |
| "rewards/rejected": -1.897156000137329, |
| "step": 245 |
| }, |
| { |
| "epoch": 1.86, |
| "grad_norm": 9.752420565118218, |
| "learning_rate": 2.1067415730337076e-07, |
| "logps/chosen": -49.50055694580078, |
| "logps/rejected": -65.74305725097656, |
| "loss": 0.4852, |
| "losses/dpo": 0.23906151950359344, |
| "losses/sft": 1.417936086654663, |
| "losses/total": 0.23906151950359344, |
| "ref_logps/chosen": -39.56950378417969, |
| "ref_logps/rejected": -47.11359405517578, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9931050539016724, |
| "rewards/margins": 0.8698407411575317, |
| "rewards/rejected": -1.8629456758499146, |
| "step": 246 |
| }, |
| { |
| "epoch": 1.86, |
| "grad_norm": 9.475549639179285, |
| "learning_rate": 2.0926966292134832e-07, |
| "logps/chosen": -49.32406997680664, |
| "logps/rejected": -63.67836380004883, |
| "loss": 0.5131, |
| "losses/dpo": 0.4199092984199524, |
| "losses/sft": 1.6027624607086182, |
| "losses/total": 0.4199092984199524, |
| "ref_logps/chosen": -39.44010925292969, |
| "ref_logps/rejected": -46.58472442626953, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.988396942615509, |
| "rewards/margins": 0.7209669947624207, |
| "rewards/rejected": -1.7093638181686401, |
| "step": 247 |
| }, |
| { |
| "epoch": 1.87, |
| "grad_norm": 8.691211910246372, |
| "learning_rate": 2.0786516853932585e-07, |
| "logps/chosen": -50.76154327392578, |
| "logps/rejected": -63.2697868347168, |
| "loss": 0.4931, |
| "losses/dpo": 0.48489272594451904, |
| "losses/sft": 2.4353795051574707, |
| "losses/total": 0.48489272594451904, |
| "ref_logps/chosen": -40.483741760253906, |
| "ref_logps/rejected": -45.29812240600586, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0277801752090454, |
| "rewards/margins": 0.7693858742713928, |
| "rewards/rejected": -1.7971662282943726, |
| "step": 248 |
| }, |
| { |
| "epoch": 1.88, |
| "grad_norm": 9.872927469509172, |
| "learning_rate": 2.0646067415730336e-07, |
| "logps/chosen": -49.138465881347656, |
| "logps/rejected": -66.16885375976562, |
| "loss": 0.5055, |
| "losses/dpo": 0.5177363753318787, |
| "losses/sft": 1.834108829498291, |
| "losses/total": 0.5177363753318787, |
| "ref_logps/chosen": -38.63552474975586, |
| "ref_logps/rejected": -48.135929107666016, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.050294280052185, |
| "rewards/margins": 0.7529983520507812, |
| "rewards/rejected": -1.8032926321029663, |
| "step": 249 |
| }, |
| { |
| "epoch": 1.89, |
| "grad_norm": 9.265714326509956, |
| "learning_rate": 2.0505617977528089e-07, |
| "logps/chosen": -48.308753967285156, |
| "logps/rejected": -66.52729034423828, |
| "loss": 0.486, |
| "losses/dpo": 0.6348212957382202, |
| "losses/sft": 1.772031307220459, |
| "losses/total": 0.6348212957382202, |
| "ref_logps/chosen": -38.7848014831543, |
| "ref_logps/rejected": -48.895755767822266, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.9523951411247253, |
| "rewards/margins": 0.8107584714889526, |
| "rewards/rejected": -1.7631536722183228, |
| "step": 250 |
| }, |
| { |
| "epoch": 1.89, |
| "grad_norm": 9.119904977454695, |
| "learning_rate": 2.0365168539325842e-07, |
| "logps/chosen": -50.29534912109375, |
| "logps/rejected": -64.1716537475586, |
| "loss": 0.4977, |
| "losses/dpo": 0.522094190120697, |
| "losses/sft": 1.8622956275939941, |
| "losses/total": 0.522094190120697, |
| "ref_logps/chosen": -39.614479064941406, |
| "ref_logps/rejected": -45.53257751464844, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -1.0680873394012451, |
| "rewards/margins": 0.7958202958106995, |
| "rewards/rejected": -1.8639075756072998, |
| "step": 251 |
| }, |
| { |
| "epoch": 1.9, |
| "grad_norm": 8.300519651321538, |
| "learning_rate": 2.0224719101123595e-07, |
| "logps/chosen": -48.15717315673828, |
| "logps/rejected": -66.43309783935547, |
| "loss": 0.4749, |
| "losses/dpo": 0.7441291809082031, |
| "losses/sft": 2.440709352493286, |
| "losses/total": 0.7441291809082031, |
| "ref_logps/chosen": -38.77737808227539, |
| "ref_logps/rejected": -47.56039047241211, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -0.9379786252975464, |
| "rewards/margins": 0.9492916464805603, |
| "rewards/rejected": -1.887270212173462, |
| "step": 252 |
| }, |
| { |
| "epoch": 1.91, |
| "grad_norm": 9.317842509294723, |
| "learning_rate": 2.0084269662921348e-07, |
| "logps/chosen": -44.09011459350586, |
| "logps/rejected": -59.16832733154297, |
| "loss": 0.5178, |
| "losses/dpo": 0.8791393041610718, |
| "losses/sft": 2.401695489883423, |
| "losses/total": 0.8791393041610718, |
| "ref_logps/chosen": -35.33125305175781, |
| "ref_logps/rejected": -43.098087310791016, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -0.8758863210678101, |
| "rewards/margins": 0.7311373353004456, |
| "rewards/rejected": -1.60702383518219, |
| "step": 253 |
| }, |
| { |
| "epoch": 1.92, |
| "grad_norm": 8.201438226460152, |
| "learning_rate": 1.9943820224719098e-07, |
| "logps/chosen": -51.51540756225586, |
| "logps/rejected": -67.69857788085938, |
| "loss": 0.4104, |
| "losses/dpo": 0.48046156764030457, |
| "losses/sft": 1.7079527378082275, |
| "losses/total": 0.48046156764030457, |
| "ref_logps/chosen": -42.20224380493164, |
| "ref_logps/rejected": -47.85743713378906, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -0.9313161373138428, |
| "rewards/margins": 1.0527985095977783, |
| "rewards/rejected": -1.9841147661209106, |
| "step": 254 |
| }, |
| { |
| "epoch": 1.92, |
| "grad_norm": 9.182124453243851, |
| "learning_rate": 1.9803370786516854e-07, |
| "logps/chosen": -52.86988067626953, |
| "logps/rejected": -64.82228088378906, |
| "loss": 0.4958, |
| "losses/dpo": 0.4478102922439575, |
| "losses/sft": 1.8136005401611328, |
| "losses/total": 0.4478102922439575, |
| "ref_logps/chosen": -43.32461929321289, |
| "ref_logps/rejected": -48.31917190551758, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -0.9545266032218933, |
| "rewards/margins": 0.6957840323448181, |
| "rewards/rejected": -1.650310754776001, |
| "step": 255 |
| }, |
| { |
| "epoch": 1.93, |
| "grad_norm": 8.664903887618609, |
| "learning_rate": 1.9662921348314607e-07, |
| "logps/chosen": -47.40139389038086, |
| "logps/rejected": -61.24211502075195, |
| "loss": 0.5069, |
| "losses/dpo": 0.45637544989585876, |
| "losses/sft": 2.080510139465332, |
| "losses/total": 0.45637544989585876, |
| "ref_logps/chosen": -37.96201705932617, |
| "ref_logps/rejected": -44.53689956665039, |
| "rewards/accuracies": 0.7265625, |
| "rewards/chosen": -0.9439379572868347, |
| "rewards/margins": 0.7265833616256714, |
| "rewards/rejected": -1.6705212593078613, |
| "step": 256 |
| }, |
| { |
| "epoch": 1.94, |
| "grad_norm": 8.357609816704013, |
| "learning_rate": 1.952247191011236e-07, |
| "logps/chosen": -51.97878646850586, |
| "logps/rejected": -66.72062683105469, |
| "loss": 0.4312, |
| "losses/dpo": 0.3072975277900696, |
| "losses/sft": 1.8392665386199951, |
| "losses/total": 0.3072975277900696, |
| "ref_logps/chosen": -42.230186462402344, |
| "ref_logps/rejected": -47.218257904052734, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.9748601913452148, |
| "rewards/margins": 0.9753769040107727, |
| "rewards/rejected": -1.9502369165420532, |
| "step": 257 |
| }, |
| { |
| "epoch": 1.95, |
| "grad_norm": 8.434371561430744, |
| "learning_rate": 1.938202247191011e-07, |
| "logps/chosen": -50.701534271240234, |
| "logps/rejected": -65.6552734375, |
| "loss": 0.4402, |
| "losses/dpo": 0.4483451545238495, |
| "losses/sft": 1.6883811950683594, |
| "losses/total": 0.4483451545238495, |
| "ref_logps/chosen": -40.96206283569336, |
| "ref_logps/rejected": -46.52254104614258, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -0.9739474654197693, |
| "rewards/margins": 0.9393259286880493, |
| "rewards/rejected": -1.9132733345031738, |
| "step": 258 |
| }, |
| { |
| "epoch": 1.95, |
| "grad_norm": 8.326106947840566, |
| "learning_rate": 1.9241573033707863e-07, |
| "logps/chosen": -49.05642318725586, |
| "logps/rejected": -62.65263366699219, |
| "loss": 0.4559, |
| "losses/dpo": 0.43069154024124146, |
| "losses/sft": 2.6451079845428467, |
| "losses/total": 0.43069154024124146, |
| "ref_logps/chosen": -39.344356536865234, |
| "ref_logps/rejected": -43.80147933959961, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9712071418762207, |
| "rewards/margins": 0.9139088988304138, |
| "rewards/rejected": -1.8851161003112793, |
| "step": 259 |
| }, |
| { |
| "epoch": 1.96, |
| "grad_norm": 10.317747735678177, |
| "learning_rate": 1.9101123595505617e-07, |
| "logps/chosen": -47.29045867919922, |
| "logps/rejected": -60.76853942871094, |
| "loss": 0.5233, |
| "losses/dpo": 0.6233609914779663, |
| "losses/sft": 2.166818618774414, |
| "losses/total": 0.6233609914779663, |
| "ref_logps/chosen": -37.4037971496582, |
| "ref_logps/rejected": -43.672698974609375, |
| "rewards/accuracies": 0.7578125, |
| "rewards/chosen": -0.9886665344238281, |
| "rewards/margins": 0.7209180593490601, |
| "rewards/rejected": -1.7095845937728882, |
| "step": 260 |
| }, |
| { |
| "epoch": 1.97, |
| "grad_norm": 8.570482678193486, |
| "learning_rate": 1.896067415730337e-07, |
| "logps/chosen": -47.826866149902344, |
| "logps/rejected": -64.69200897216797, |
| "loss": 0.4646, |
| "losses/dpo": 0.39222848415374756, |
| "losses/sft": 1.7622792720794678, |
| "losses/total": 0.39222848415374756, |
| "ref_logps/chosen": -37.465206146240234, |
| "ref_logps/rejected": -45.77225112915039, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.0361659526824951, |
| "rewards/margins": 0.8558104634284973, |
| "rewards/rejected": -1.8919763565063477, |
| "step": 261 |
| }, |
| { |
| "epoch": 1.98, |
| "grad_norm": 8.082044540577316, |
| "learning_rate": 1.8820224719101123e-07, |
| "logps/chosen": -44.3437385559082, |
| "logps/rejected": -65.46928405761719, |
| "loss": 0.4247, |
| "losses/dpo": 0.25466495752334595, |
| "losses/sft": 1.6573872566223145, |
| "losses/total": 0.25466495752334595, |
| "ref_logps/chosen": -35.26055908203125, |
| "ref_logps/rejected": -46.34375, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.908318042755127, |
| "rewards/margins": 1.0042363405227661, |
| "rewards/rejected": -1.9125542640686035, |
| "step": 262 |
| }, |
| { |
| "epoch": 1.98, |
| "grad_norm": 7.886213605680278, |
| "learning_rate": 1.8679775280898876e-07, |
| "logps/chosen": -44.65882873535156, |
| "logps/rejected": -65.91732025146484, |
| "loss": 0.4233, |
| "losses/dpo": 0.37609466910362244, |
| "losses/sft": 1.7309682369232178, |
| "losses/total": 0.37609466910362244, |
| "ref_logps/chosen": -34.98197555541992, |
| "ref_logps/rejected": -45.339569091796875, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.9676854610443115, |
| "rewards/margins": 1.0900897979736328, |
| "rewards/rejected": -2.0577754974365234, |
| "step": 263 |
| }, |
| { |
| "epoch": 1.99, |
| "grad_norm": 9.420349702400047, |
| "learning_rate": 1.853932584269663e-07, |
| "logps/chosen": -55.511512756347656, |
| "logps/rejected": -67.9210205078125, |
| "loss": 0.4846, |
| "losses/dpo": 0.35744136571884155, |
| "losses/sft": 1.9689966440200806, |
| "losses/total": 0.35744136571884155, |
| "ref_logps/chosen": -44.93099594116211, |
| "ref_logps/rejected": -48.529052734375, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.058051347732544, |
| "rewards/margins": 0.8811461925506592, |
| "rewards/rejected": -1.9391975402832031, |
| "step": 264 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 8.90872581509833, |
| "learning_rate": 1.8398876404494382e-07, |
| "logps/chosen": -50.006412506103516, |
| "logps/rejected": -65.82926940917969, |
| "loss": 0.4658, |
| "losses/dpo": 0.42064571380615234, |
| "losses/sft": 1.965097427368164, |
| "losses/total": 0.42064571380615234, |
| "ref_logps/chosen": -39.1809196472168, |
| "ref_logps/rejected": -45.722938537597656, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.0825499296188354, |
| "rewards/margins": 0.9280825853347778, |
| "rewards/rejected": -2.0106325149536133, |
| "step": 265 |
| }, |
| { |
| "epoch": 2.01, |
| "grad_norm": 7.965650717330079, |
| "learning_rate": 1.8258426966292135e-07, |
| "logps/chosen": -50.00637435913086, |
| "logps/rejected": -65.08876037597656, |
| "loss": 0.4116, |
| "losses/dpo": 0.36081230640411377, |
| "losses/sft": 2.0271382331848145, |
| "losses/total": 0.36081230640411377, |
| "ref_logps/chosen": -39.65681076049805, |
| "ref_logps/rejected": -44.160343170166016, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -1.0349565744400024, |
| "rewards/margins": 1.0578850507736206, |
| "rewards/rejected": -2.092841625213623, |
| "step": 266 |
| }, |
| { |
| "epoch": 2.02, |
| "grad_norm": 8.213010773638015, |
| "learning_rate": 1.8117977528089888e-07, |
| "logps/chosen": -49.70448303222656, |
| "logps/rejected": -64.45352172851562, |
| "loss": 0.4328, |
| "losses/dpo": 0.4751141667366028, |
| "losses/sft": 2.163590431213379, |
| "losses/total": 0.4751141667366028, |
| "ref_logps/chosen": -39.66984558105469, |
| "ref_logps/rejected": -45.02206802368164, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.0034637451171875, |
| "rewards/margins": 0.9396811723709106, |
| "rewards/rejected": -1.943144679069519, |
| "step": 267 |
| }, |
| { |
| "epoch": 2.02, |
| "grad_norm": 8.627785952714778, |
| "learning_rate": 1.7977528089887638e-07, |
| "logps/chosen": -47.43844985961914, |
| "logps/rejected": -61.57966995239258, |
| "loss": 0.4366, |
| "losses/dpo": 0.4777096211910248, |
| "losses/sft": 2.011448860168457, |
| "losses/total": 0.4777096211910248, |
| "ref_logps/chosen": -37.887752532958984, |
| "ref_logps/rejected": -42.577537536621094, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -0.9550699591636658, |
| "rewards/margins": 0.945143461227417, |
| "rewards/rejected": -1.9002132415771484, |
| "step": 268 |
| }, |
| { |
| "epoch": 2.03, |
| "grad_norm": 9.54969996951565, |
| "learning_rate": 1.7837078651685391e-07, |
| "logps/chosen": -50.50410842895508, |
| "logps/rejected": -64.97512817382812, |
| "loss": 0.4367, |
| "losses/dpo": 0.4186415672302246, |
| "losses/sft": 2.4884321689605713, |
| "losses/total": 0.4186415672302246, |
| "ref_logps/chosen": -39.366878509521484, |
| "ref_logps/rejected": -44.09857940673828, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.1137233972549438, |
| "rewards/margins": 0.973931610584259, |
| "rewards/rejected": -2.0876548290252686, |
| "step": 269 |
| }, |
| { |
| "epoch": 2.04, |
| "grad_norm": 7.541865642493473, |
| "learning_rate": 1.7696629213483144e-07, |
| "logps/chosen": -53.07981491088867, |
| "logps/rejected": -69.67108917236328, |
| "loss": 0.3686, |
| "losses/dpo": 0.23445191979408264, |
| "losses/sft": 1.9627153873443604, |
| "losses/total": 0.23445191979408264, |
| "ref_logps/chosen": -41.86531448364258, |
| "ref_logps/rejected": -46.77862548828125, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -1.121450424194336, |
| "rewards/margins": 1.1677953004837036, |
| "rewards/rejected": -2.289245843887329, |
| "step": 270 |
| }, |
| { |
| "epoch": 2.05, |
| "grad_norm": 7.78557132067994, |
| "learning_rate": 1.75561797752809e-07, |
| "logps/chosen": -44.59587478637695, |
| "logps/rejected": -61.11756896972656, |
| "loss": 0.4304, |
| "losses/dpo": 0.40728461742401123, |
| "losses/sft": 2.1670854091644287, |
| "losses/total": 0.40728461742401123, |
| "ref_logps/chosen": -34.50210189819336, |
| "ref_logps/rejected": -41.35382843017578, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.009376883506775, |
| "rewards/margins": 0.9669971466064453, |
| "rewards/rejected": -1.9763740301132202, |
| "step": 271 |
| }, |
| { |
| "epoch": 2.05, |
| "grad_norm": 8.691528110147415, |
| "learning_rate": 1.741573033707865e-07, |
| "logps/chosen": -47.1044807434082, |
| "logps/rejected": -62.402366638183594, |
| "loss": 0.4439, |
| "losses/dpo": 0.6048084497451782, |
| "losses/sft": 2.738722801208496, |
| "losses/total": 0.6048084497451782, |
| "ref_logps/chosen": -36.921566009521484, |
| "ref_logps/rejected": -41.66175842285156, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.0182914733886719, |
| "rewards/margins": 1.0557701587677002, |
| "rewards/rejected": -2.074061632156372, |
| "step": 272 |
| }, |
| { |
| "epoch": 2.06, |
| "grad_norm": 7.3620175100007135, |
| "learning_rate": 1.7275280898876404e-07, |
| "logps/chosen": -51.797447204589844, |
| "logps/rejected": -69.3549575805664, |
| "loss": 0.3635, |
| "losses/dpo": 0.4010230600833893, |
| "losses/sft": 1.782325029373169, |
| "losses/total": 0.4010230600833893, |
| "ref_logps/chosen": -41.61540222167969, |
| "ref_logps/rejected": -47.2431526184082, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -1.0182045698165894, |
| "rewards/margins": 1.192975640296936, |
| "rewards/rejected": -2.2111802101135254, |
| "step": 273 |
| }, |
| { |
| "epoch": 2.07, |
| "grad_norm": 8.804306158193548, |
| "learning_rate": 1.7134831460674157e-07, |
| "logps/chosen": -51.407501220703125, |
| "logps/rejected": -64.26744079589844, |
| "loss": 0.4578, |
| "losses/dpo": 0.6319560408592224, |
| "losses/sft": 2.214840888977051, |
| "losses/total": 0.6319560408592224, |
| "ref_logps/chosen": -40.07988739013672, |
| "ref_logps/rejected": -43.7442741394043, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.1327617168426514, |
| "rewards/margins": 0.919555127620697, |
| "rewards/rejected": -2.052316665649414, |
| "step": 274 |
| }, |
| { |
| "epoch": 2.08, |
| "grad_norm": 9.887135650412674, |
| "learning_rate": 1.699438202247191e-07, |
| "logps/chosen": -48.88153839111328, |
| "logps/rejected": -64.42852783203125, |
| "loss": 0.4198, |
| "losses/dpo": 0.36265987157821655, |
| "losses/sft": 2.2624428272247314, |
| "losses/total": 0.36265987157821655, |
| "ref_logps/chosen": -38.071693420410156, |
| "ref_logps/rejected": -42.96941375732422, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.080984115600586, |
| "rewards/margins": 1.064927339553833, |
| "rewards/rejected": -2.145911455154419, |
| "step": 275 |
| }, |
| { |
| "epoch": 2.08, |
| "grad_norm": 7.789984027153804, |
| "learning_rate": 1.6853932584269663e-07, |
| "logps/chosen": -53.47975158691406, |
| "logps/rejected": -69.208251953125, |
| "loss": 0.4077, |
| "losses/dpo": 0.3306717872619629, |
| "losses/sft": 1.8213623762130737, |
| "losses/total": 0.3306717872619629, |
| "ref_logps/chosen": -42.26997375488281, |
| "ref_logps/rejected": -47.255218505859375, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.1209776401519775, |
| "rewards/margins": 1.0743255615234375, |
| "rewards/rejected": -2.195303440093994, |
| "step": 276 |
| }, |
| { |
| "epoch": 2.09, |
| "grad_norm": 8.577961125042698, |
| "learning_rate": 1.6713483146067413e-07, |
| "logps/chosen": -48.9437255859375, |
| "logps/rejected": -66.896484375, |
| "loss": 0.4237, |
| "losses/dpo": 0.375847727060318, |
| "losses/sft": 1.7302836179733276, |
| "losses/total": 0.375847727060318, |
| "ref_logps/chosen": -38.10498046875, |
| "ref_logps/rejected": -45.20066452026367, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.0838744640350342, |
| "rewards/margins": 1.0857088565826416, |
| "rewards/rejected": -2.1695830821990967, |
| "step": 277 |
| }, |
| { |
| "epoch": 2.1, |
| "grad_norm": 7.88842347856655, |
| "learning_rate": 1.6573033707865166e-07, |
| "logps/chosen": -50.38557815551758, |
| "logps/rejected": -71.69691467285156, |
| "loss": 0.3884, |
| "losses/dpo": 0.20783157646656036, |
| "losses/sft": 1.65842604637146, |
| "losses/total": 0.20783157646656036, |
| "ref_logps/chosen": -38.84345245361328, |
| "ref_logps/rejected": -47.66883850097656, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -1.1542127132415771, |
| "rewards/margins": 1.2485952377319336, |
| "rewards/rejected": -2.40280818939209, |
| "step": 278 |
| }, |
| { |
| "epoch": 2.11, |
| "grad_norm": 8.27071868454665, |
| "learning_rate": 1.6432584269662922e-07, |
| "logps/chosen": -50.050907135009766, |
| "logps/rejected": -71.91120147705078, |
| "loss": 0.3873, |
| "losses/dpo": 0.36935174465179443, |
| "losses/sft": 2.4310526847839355, |
| "losses/total": 0.36935174465179443, |
| "ref_logps/chosen": -38.60224914550781, |
| "ref_logps/rejected": -48.42378234863281, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.1448655128479004, |
| "rewards/margins": 1.2038761377334595, |
| "rewards/rejected": -2.3487415313720703, |
| "step": 279 |
| }, |
| { |
| "epoch": 2.11, |
| "grad_norm": 8.202604558490952, |
| "learning_rate": 1.6292134831460675e-07, |
| "logps/chosen": -49.78974533081055, |
| "logps/rejected": -63.648345947265625, |
| "loss": 0.4287, |
| "losses/dpo": 0.4768953025341034, |
| "losses/sft": 1.486172080039978, |
| "losses/total": 0.4768953025341034, |
| "ref_logps/chosen": -38.987091064453125, |
| "ref_logps/rejected": -42.706016540527344, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.0802651643753052, |
| "rewards/margins": 1.0139687061309814, |
| "rewards/rejected": -2.094233751296997, |
| "step": 280 |
| }, |
| { |
| "epoch": 2.12, |
| "grad_norm": 7.6837084768811295, |
| "learning_rate": 1.6151685393258428e-07, |
| "logps/chosen": -46.654842376708984, |
| "logps/rejected": -65.57063293457031, |
| "loss": 0.4002, |
| "losses/dpo": 0.25231242179870605, |
| "losses/sft": 1.6833255290985107, |
| "losses/total": 0.25231242179870605, |
| "ref_logps/chosen": -35.983055114746094, |
| "ref_logps/rejected": -43.31926345825195, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.0671789646148682, |
| "rewards/margins": 1.1579577922821045, |
| "rewards/rejected": -2.2251367568969727, |
| "step": 281 |
| }, |
| { |
| "epoch": 2.13, |
| "grad_norm": 7.367492054152066, |
| "learning_rate": 1.6011235955056178e-07, |
| "logps/chosen": -47.970069885253906, |
| "logps/rejected": -67.12972259521484, |
| "loss": 0.3817, |
| "losses/dpo": 0.2698941230773926, |
| "losses/sft": 2.0059823989868164, |
| "losses/total": 0.2698941230773926, |
| "ref_logps/chosen": -37.96726608276367, |
| "ref_logps/rejected": -44.837013244628906, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.0002803802490234, |
| "rewards/margins": 1.2289901971817017, |
| "rewards/rejected": -2.2292706966400146, |
| "step": 282 |
| }, |
| { |
| "epoch": 2.14, |
| "grad_norm": 8.198398756823499, |
| "learning_rate": 1.5870786516853931e-07, |
| "logps/chosen": -52.00371551513672, |
| "logps/rejected": -71.66854095458984, |
| "loss": 0.373, |
| "losses/dpo": 0.37371307611465454, |
| "losses/sft": 2.166947841644287, |
| "losses/total": 0.37371307611465454, |
| "ref_logps/chosen": -40.8586540222168, |
| "ref_logps/rejected": -47.34501266479492, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.1145060062408447, |
| "rewards/margins": 1.3178460597991943, |
| "rewards/rejected": -2.432352304458618, |
| "step": 283 |
| }, |
| { |
| "epoch": 2.14, |
| "grad_norm": 7.211631798978324, |
| "learning_rate": 1.5730337078651685e-07, |
| "logps/chosen": -45.30519104003906, |
| "logps/rejected": -67.6802749633789, |
| "loss": 0.3529, |
| "losses/dpo": 0.6495727896690369, |
| "losses/sft": 2.1896119117736816, |
| "losses/total": 0.6495727896690369, |
| "ref_logps/chosen": -35.870975494384766, |
| "ref_logps/rejected": -45.01594924926758, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.9434216022491455, |
| "rewards/margins": 1.3230111598968506, |
| "rewards/rejected": -2.266432523727417, |
| "step": 284 |
| }, |
| { |
| "epoch": 2.15, |
| "grad_norm": 8.73649186021766, |
| "learning_rate": 1.5589887640449438e-07, |
| "logps/chosen": -51.24886703491211, |
| "logps/rejected": -73.5231704711914, |
| "loss": 0.3706, |
| "losses/dpo": 0.2774621248245239, |
| "losses/sft": 2.14704966545105, |
| "losses/total": 0.2774621248245239, |
| "ref_logps/chosen": -39.21043395996094, |
| "ref_logps/rejected": -48.85007858276367, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.2038426399230957, |
| "rewards/margins": 1.2634668350219727, |
| "rewards/rejected": -2.4673094749450684, |
| "step": 285 |
| }, |
| { |
| "epoch": 2.16, |
| "grad_norm": 9.965422332405613, |
| "learning_rate": 1.5449438202247188e-07, |
| "logps/chosen": -56.63559341430664, |
| "logps/rejected": -69.31437683105469, |
| "loss": 0.4581, |
| "losses/dpo": 1.019913673400879, |
| "losses/sft": 2.6583502292633057, |
| "losses/total": 1.019913673400879, |
| "ref_logps/chosen": -42.96681594848633, |
| "ref_logps/rejected": -44.904361724853516, |
| "rewards/accuracies": 0.765625, |
| "rewards/chosen": -1.3668776750564575, |
| "rewards/margins": 1.0741242170333862, |
| "rewards/rejected": -2.4410018920898438, |
| "step": 286 |
| }, |
| { |
| "epoch": 2.17, |
| "grad_norm": 9.04029000539786, |
| "learning_rate": 1.5308988764044944e-07, |
| "logps/chosen": -50.918128967285156, |
| "logps/rejected": -67.66094970703125, |
| "loss": 0.4491, |
| "losses/dpo": 0.536713182926178, |
| "losses/sft": 2.0050907135009766, |
| "losses/total": 0.536713182926178, |
| "ref_logps/chosen": -37.77289581298828, |
| "ref_logps/rejected": -43.736061096191406, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.3145227432250977, |
| "rewards/margins": 1.0779664516448975, |
| "rewards/rejected": -2.392489194869995, |
| "step": 287 |
| }, |
| { |
| "epoch": 2.17, |
| "grad_norm": 7.9909005031619476, |
| "learning_rate": 1.5168539325842697e-07, |
| "logps/chosen": -48.0228271484375, |
| "logps/rejected": -72.348876953125, |
| "loss": 0.3569, |
| "losses/dpo": 0.22953173518180847, |
| "losses/sft": 1.752846360206604, |
| "losses/total": 0.22953173518180847, |
| "ref_logps/chosen": -36.91614532470703, |
| "ref_logps/rejected": -47.88288497924805, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -1.1106677055358887, |
| "rewards/margins": 1.3359307050704956, |
| "rewards/rejected": -2.4465982913970947, |
| "step": 288 |
| }, |
| { |
| "epoch": 2.18, |
| "grad_norm": 9.189297221788385, |
| "learning_rate": 1.502808988764045e-07, |
| "logps/chosen": -55.16273880004883, |
| "logps/rejected": -74.13533782958984, |
| "loss": 0.4081, |
| "losses/dpo": 0.39626190066337585, |
| "losses/sft": 2.1484994888305664, |
| "losses/total": 0.39626190066337585, |
| "ref_logps/chosen": -42.12677001953125, |
| "ref_logps/rejected": -48.699642181396484, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.3035968542099, |
| "rewards/margins": 1.2399725914001465, |
| "rewards/rejected": -2.543569326400757, |
| "step": 289 |
| }, |
| { |
| "epoch": 2.19, |
| "grad_norm": 8.223258235820202, |
| "learning_rate": 1.4887640449438203e-07, |
| "logps/chosen": -46.8680534362793, |
| "logps/rejected": -64.84513854980469, |
| "loss": 0.3998, |
| "losses/dpo": 0.38319000601768494, |
| "losses/sft": 2.0748698711395264, |
| "losses/total": 0.38319000601768494, |
| "ref_logps/chosen": -35.66654968261719, |
| "ref_logps/rejected": -42.36066818237305, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.1201505661010742, |
| "rewards/margins": 1.1282968521118164, |
| "rewards/rejected": -2.2484474182128906, |
| "step": 290 |
| }, |
| { |
| "epoch": 2.2, |
| "grad_norm": 9.322598904199673, |
| "learning_rate": 1.4747191011235953e-07, |
| "logps/chosen": -51.94579315185547, |
| "logps/rejected": -76.23491668701172, |
| "loss": 0.4365, |
| "losses/dpo": 0.6639813184738159, |
| "losses/sft": 3.0463194847106934, |
| "losses/total": 0.6639813184738159, |
| "ref_logps/chosen": -37.92943572998047, |
| "ref_logps/rejected": -49.77875518798828, |
| "rewards/accuracies": 0.7578125, |
| "rewards/chosen": -1.4016355276107788, |
| "rewards/margins": 1.243980884552002, |
| "rewards/rejected": -2.6456165313720703, |
| "step": 291 |
| }, |
| { |
| "epoch": 2.2, |
| "grad_norm": 9.28502050286711, |
| "learning_rate": 1.4606741573033706e-07, |
| "logps/chosen": -51.56089782714844, |
| "logps/rejected": -67.3809814453125, |
| "loss": 0.4295, |
| "losses/dpo": 0.2485855668783188, |
| "losses/sft": 2.5399341583251953, |
| "losses/total": 0.2485855668783188, |
| "ref_logps/chosen": -38.323081970214844, |
| "ref_logps/rejected": -43.60871505737305, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.3237823247909546, |
| "rewards/margins": 1.0534443855285645, |
| "rewards/rejected": -2.3772268295288086, |
| "step": 292 |
| }, |
| { |
| "epoch": 2.21, |
| "grad_norm": 10.352671653799021, |
| "learning_rate": 1.446629213483146e-07, |
| "logps/chosen": -60.71104431152344, |
| "logps/rejected": -76.54723358154297, |
| "loss": 0.4205, |
| "losses/dpo": 0.5032411813735962, |
| "losses/sft": 2.2452447414398193, |
| "losses/total": 0.5032411813735962, |
| "ref_logps/chosen": -45.97325897216797, |
| "ref_logps/rejected": -49.70647430419922, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.4737780094146729, |
| "rewards/margins": 1.2102973461151123, |
| "rewards/rejected": -2.684075117111206, |
| "step": 293 |
| }, |
| { |
| "epoch": 2.22, |
| "grad_norm": 9.489700991538179, |
| "learning_rate": 1.4325842696629212e-07, |
| "logps/chosen": -51.33311462402344, |
| "logps/rejected": -72.56240844726562, |
| "loss": 0.4041, |
| "losses/dpo": 0.39366570115089417, |
| "losses/sft": 1.5643844604492188, |
| "losses/total": 0.39366570115089417, |
| "ref_logps/chosen": -38.85292053222656, |
| "ref_logps/rejected": -47.41047668457031, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -1.2480189800262451, |
| "rewards/margins": 1.2671747207641602, |
| "rewards/rejected": -2.5151939392089844, |
| "step": 294 |
| }, |
| { |
| "epoch": 2.23, |
| "grad_norm": 8.202282597323313, |
| "learning_rate": 1.4185393258426968e-07, |
| "logps/chosen": -51.06114959716797, |
| "logps/rejected": -74.44361114501953, |
| "loss": 0.3473, |
| "losses/dpo": 0.40783169865608215, |
| "losses/sft": 2.3142831325531006, |
| "losses/total": 0.40783169865608215, |
| "ref_logps/chosen": -39.6861457824707, |
| "ref_logps/rejected": -49.402000427246094, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.1375010013580322, |
| "rewards/margins": 1.3666609525680542, |
| "rewards/rejected": -2.504162073135376, |
| "step": 295 |
| }, |
| { |
| "epoch": 2.23, |
| "grad_norm": 8.616438775614668, |
| "learning_rate": 1.4044943820224718e-07, |
| "logps/chosen": -50.71443557739258, |
| "logps/rejected": -67.99952697753906, |
| "loss": 0.4191, |
| "losses/dpo": 0.3447108566761017, |
| "losses/sft": 1.6047589778900146, |
| "losses/total": 0.3447108566761017, |
| "ref_logps/chosen": -38.159549713134766, |
| "ref_logps/rejected": -44.2279052734375, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.2554888725280762, |
| "rewards/margins": 1.12167227268219, |
| "rewards/rejected": -2.3771612644195557, |
| "step": 296 |
| }, |
| { |
| "epoch": 2.24, |
| "grad_norm": 8.42295456953331, |
| "learning_rate": 1.3904494382022472e-07, |
| "logps/chosen": -50.11585235595703, |
| "logps/rejected": -69.99391174316406, |
| "loss": 0.4154, |
| "losses/dpo": 0.6180249452590942, |
| "losses/sft": 1.9767247438430786, |
| "losses/total": 0.6180249452590942, |
| "ref_logps/chosen": -37.855167388916016, |
| "ref_logps/rejected": -45.18976974487305, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.2260689735412598, |
| "rewards/margins": 1.2543449401855469, |
| "rewards/rejected": -2.4804139137268066, |
| "step": 297 |
| }, |
| { |
| "epoch": 2.25, |
| "grad_norm": 7.916024207524859, |
| "learning_rate": 1.3764044943820225e-07, |
| "logps/chosen": -48.01890563964844, |
| "logps/rejected": -71.55242919921875, |
| "loss": 0.3286, |
| "losses/dpo": 0.39366215467453003, |
| "losses/sft": 1.6183239221572876, |
| "losses/total": 0.39366215467453003, |
| "ref_logps/chosen": -36.65507888793945, |
| "ref_logps/rejected": -45.11772537231445, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.1363829374313354, |
| "rewards/margins": 1.5070867538452148, |
| "rewards/rejected": -2.64346981048584, |
| "step": 298 |
| }, |
| { |
| "epoch": 2.26, |
| "grad_norm": 8.381587007100304, |
| "learning_rate": 1.3623595505617978e-07, |
| "logps/chosen": -48.97580337524414, |
| "logps/rejected": -71.05843353271484, |
| "loss": 0.369, |
| "losses/dpo": 0.4669285714626312, |
| "losses/sft": 2.4736876487731934, |
| "losses/total": 0.4669285714626312, |
| "ref_logps/chosen": -36.59314727783203, |
| "ref_logps/rejected": -46.38197708129883, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -1.2382655143737793, |
| "rewards/margins": 1.229379415512085, |
| "rewards/rejected": -2.4676451683044434, |
| "step": 299 |
| }, |
| { |
| "epoch": 2.26, |
| "grad_norm": 7.816178695034966, |
| "learning_rate": 1.3483146067415728e-07, |
| "logps/chosen": -48.929466247558594, |
| "logps/rejected": -70.30461883544922, |
| "loss": 0.3683, |
| "losses/dpo": 0.30282700061798096, |
| "losses/sft": 2.068610906600952, |
| "losses/total": 0.30282700061798096, |
| "ref_logps/chosen": -36.281883239746094, |
| "ref_logps/rejected": -44.1247444152832, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.2647581100463867, |
| "rewards/margins": 1.3532286882400513, |
| "rewards/rejected": -2.6179869174957275, |
| "step": 300 |
| }, |
| { |
| "epoch": 2.27, |
| "grad_norm": 9.42088429106351, |
| "learning_rate": 1.334269662921348e-07, |
| "logps/chosen": -53.99761199951172, |
| "logps/rejected": -68.16130828857422, |
| "loss": 0.4154, |
| "losses/dpo": 0.7334519028663635, |
| "losses/sft": 2.3190560340881348, |
| "losses/total": 0.7334519028663635, |
| "ref_logps/chosen": -41.94767379760742, |
| "ref_logps/rejected": -43.898292541503906, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.2049940824508667, |
| "rewards/margins": 1.2213077545166016, |
| "rewards/rejected": -2.4263014793395996, |
| "step": 301 |
| }, |
| { |
| "epoch": 2.28, |
| "grad_norm": 8.474867745213691, |
| "learning_rate": 1.3202247191011234e-07, |
| "logps/chosen": -56.917606353759766, |
| "logps/rejected": -78.60893249511719, |
| "loss": 0.3643, |
| "losses/dpo": 0.30533695220947266, |
| "losses/sft": 2.2758193016052246, |
| "losses/total": 0.30533695220947266, |
| "ref_logps/chosen": -43.154685974121094, |
| "ref_logps/rejected": -50.671146392822266, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.3762919902801514, |
| "rewards/margins": 1.4174861907958984, |
| "rewards/rejected": -2.793778419494629, |
| "step": 302 |
| }, |
| { |
| "epoch": 2.29, |
| "grad_norm": 9.098388851491974, |
| "learning_rate": 1.306179775280899e-07, |
| "logps/chosen": -52.541847229003906, |
| "logps/rejected": -75.27273559570312, |
| "loss": 0.4022, |
| "losses/dpo": 0.7690958380699158, |
| "losses/sft": 2.406214714050293, |
| "losses/total": 0.7690958380699158, |
| "ref_logps/chosen": -39.33592224121094, |
| "ref_logps/rejected": -50.239105224609375, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.3205927610397339, |
| "rewards/margins": 1.1827703714370728, |
| "rewards/rejected": -2.5033628940582275, |
| "step": 303 |
| }, |
| { |
| "epoch": 2.29, |
| "grad_norm": 7.915112908663982, |
| "learning_rate": 1.2921348314606743e-07, |
| "logps/chosen": -52.54154968261719, |
| "logps/rejected": -79.33431243896484, |
| "loss": 0.3, |
| "losses/dpo": 0.22580446302890778, |
| "losses/sft": 2.1299729347229004, |
| "losses/total": 0.22580446302890778, |
| "ref_logps/chosen": -39.62370300292969, |
| "ref_logps/rejected": -51.30101013183594, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.2917848825454712, |
| "rewards/margins": 1.511545181274414, |
| "rewards/rejected": -2.8033299446105957, |
| "step": 304 |
| }, |
| { |
| "epoch": 2.3, |
| "grad_norm": 8.60697942251823, |
| "learning_rate": 1.2780898876404493e-07, |
| "logps/chosen": -54.95063781738281, |
| "logps/rejected": -76.17906188964844, |
| "loss": 0.3787, |
| "losses/dpo": 0.48840370774269104, |
| "losses/sft": 2.068924903869629, |
| "losses/total": 0.48840370774269104, |
| "ref_logps/chosen": -40.245113372802734, |
| "ref_logps/rejected": -48.299339294433594, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.4705531597137451, |
| "rewards/margins": 1.317419409751892, |
| "rewards/rejected": -2.7879724502563477, |
| "step": 305 |
| }, |
| { |
| "epoch": 2.31, |
| "grad_norm": 8.916108799954989, |
| "learning_rate": 1.2640449438202246e-07, |
| "logps/chosen": -54.08748245239258, |
| "logps/rejected": -72.3311767578125, |
| "loss": 0.384, |
| "losses/dpo": 0.27609461545944214, |
| "losses/sft": 2.02748703956604, |
| "losses/total": 0.27609461545944214, |
| "ref_logps/chosen": -40.30693817138672, |
| "ref_logps/rejected": -46.038856506347656, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.3780547380447388, |
| "rewards/margins": 1.2511768341064453, |
| "rewards/rejected": -2.6292316913604736, |
| "step": 306 |
| }, |
| { |
| "epoch": 2.32, |
| "grad_norm": 9.08799197478477, |
| "learning_rate": 1.25e-07, |
| "logps/chosen": -54.119285583496094, |
| "logps/rejected": -65.3755111694336, |
| "loss": 0.4074, |
| "losses/dpo": 0.314796507358551, |
| "losses/sft": 2.275911808013916, |
| "losses/total": 0.314796507358551, |
| "ref_logps/chosen": -40.99217224121094, |
| "ref_logps/rejected": -41.43158721923828, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.312711477279663, |
| "rewards/margins": 1.081681251525879, |
| "rewards/rejected": -2.394392967224121, |
| "step": 307 |
| }, |
| { |
| "epoch": 2.32, |
| "grad_norm": 8.103580469459054, |
| "learning_rate": 1.2359550561797752e-07, |
| "logps/chosen": -51.23326873779297, |
| "logps/rejected": -75.66114807128906, |
| "loss": 0.3713, |
| "losses/dpo": 0.387349396944046, |
| "losses/sft": 2.3062949180603027, |
| "losses/total": 0.387349396944046, |
| "ref_logps/chosen": -36.80121994018555, |
| "ref_logps/rejected": -47.23471450805664, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.4432051181793213, |
| "rewards/margins": 1.399438738822937, |
| "rewards/rejected": -2.842643976211548, |
| "step": 308 |
| }, |
| { |
| "epoch": 2.33, |
| "grad_norm": 8.263469946945373, |
| "learning_rate": 1.2219101123595506e-07, |
| "logps/chosen": -51.04032897949219, |
| "logps/rejected": -73.06275177001953, |
| "loss": 0.3571, |
| "losses/dpo": 0.4852275252342224, |
| "losses/sft": 2.0593721866607666, |
| "losses/total": 0.4852275252342224, |
| "ref_logps/chosen": -37.46986770629883, |
| "ref_logps/rejected": -45.448211669921875, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.3570460081100464, |
| "rewards/margins": 1.4044082164764404, |
| "rewards/rejected": -2.7614541053771973, |
| "step": 309 |
| }, |
| { |
| "epoch": 2.34, |
| "grad_norm": 9.638610818670363, |
| "learning_rate": 1.2078651685393259e-07, |
| "logps/chosen": -61.78599166870117, |
| "logps/rejected": -79.4295883178711, |
| "loss": 0.3814, |
| "losses/dpo": 0.3531866669654846, |
| "losses/sft": 2.5623600482940674, |
| "losses/total": 0.3531866669654846, |
| "ref_logps/chosen": -47.63096618652344, |
| "ref_logps/rejected": -52.177154541015625, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.4155021905899048, |
| "rewards/margins": 1.3097403049468994, |
| "rewards/rejected": -2.7252423763275146, |
| "step": 310 |
| }, |
| { |
| "epoch": 2.35, |
| "grad_norm": 8.100555853256143, |
| "learning_rate": 1.1938202247191012e-07, |
| "logps/chosen": -51.98362350463867, |
| "logps/rejected": -76.51303100585938, |
| "loss": 0.3451, |
| "losses/dpo": 0.25824400782585144, |
| "losses/sft": 1.8786492347717285, |
| "losses/total": 0.25824400782585144, |
| "ref_logps/chosen": -38.72646713256836, |
| "ref_logps/rejected": -48.84027862548828, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.3257157802581787, |
| "rewards/margins": 1.4415602684020996, |
| "rewards/rejected": -2.767275810241699, |
| "step": 311 |
| }, |
| { |
| "epoch": 2.35, |
| "grad_norm": 8.895756271806501, |
| "learning_rate": 1.1797752808988763e-07, |
| "logps/chosen": -54.543087005615234, |
| "logps/rejected": -75.70824432373047, |
| "loss": 0.3937, |
| "losses/dpo": 0.44249895215034485, |
| "losses/sft": 2.0751869678497314, |
| "losses/total": 0.44249895215034485, |
| "ref_logps/chosen": -40.311073303222656, |
| "ref_logps/rejected": -47.72747802734375, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.423201322555542, |
| "rewards/margins": 1.3748749494552612, |
| "rewards/rejected": -2.7980761528015137, |
| "step": 312 |
| }, |
| { |
| "epoch": 2.36, |
| "grad_norm": 9.181986369222448, |
| "learning_rate": 1.1657303370786515e-07, |
| "logps/chosen": -53.058631896972656, |
| "logps/rejected": -76.43999481201172, |
| "loss": 0.3426, |
| "losses/dpo": 0.37313902378082275, |
| "losses/sft": 1.9281624555587769, |
| "losses/total": 0.37313902378082275, |
| "ref_logps/chosen": -38.72821044921875, |
| "ref_logps/rejected": -47.35258483886719, |
| "rewards/accuracies": 0.9140625, |
| "rewards/chosen": -1.433042049407959, |
| "rewards/margins": 1.4756982326507568, |
| "rewards/rejected": -2.908740282058716, |
| "step": 313 |
| }, |
| { |
| "epoch": 2.37, |
| "grad_norm": 10.667806756150233, |
| "learning_rate": 1.151685393258427e-07, |
| "logps/chosen": -55.91019058227539, |
| "logps/rejected": -73.06080627441406, |
| "loss": 0.4539, |
| "losses/dpo": 0.27354708313941956, |
| "losses/sft": 2.2720413208007812, |
| "losses/total": 0.27354708313941956, |
| "ref_logps/chosen": -40.70777893066406, |
| "ref_logps/rejected": -46.47440719604492, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.520241141319275, |
| "rewards/margins": 1.138399600982666, |
| "rewards/rejected": -2.6586403846740723, |
| "step": 314 |
| }, |
| { |
| "epoch": 2.38, |
| "grad_norm": 9.237157441443594, |
| "learning_rate": 1.1376404494382023e-07, |
| "logps/chosen": -52.99339294433594, |
| "logps/rejected": -77.64442443847656, |
| "loss": 0.413, |
| "losses/dpo": 0.5332150459289551, |
| "losses/sft": 2.2471044063568115, |
| "losses/total": 0.5332150459289551, |
| "ref_logps/chosen": -38.779422760009766, |
| "ref_logps/rejected": -49.12788772583008, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -1.4213968515396118, |
| "rewards/margins": 1.4302568435668945, |
| "rewards/rejected": -2.851653814315796, |
| "step": 315 |
| }, |
| { |
| "epoch": 2.38, |
| "grad_norm": 10.26818606773468, |
| "learning_rate": 1.1235955056179774e-07, |
| "logps/chosen": -54.45465087890625, |
| "logps/rejected": -77.27416229248047, |
| "loss": 0.3916, |
| "losses/dpo": 0.3256514072418213, |
| "losses/sft": 2.224207878112793, |
| "losses/total": 0.3256514072418213, |
| "ref_logps/chosen": -39.11296463012695, |
| "ref_logps/rejected": -47.37888717651367, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.5341691970825195, |
| "rewards/margins": 1.4553582668304443, |
| "rewards/rejected": -2.9895272254943848, |
| "step": 316 |
| }, |
| { |
| "epoch": 2.39, |
| "grad_norm": 8.908115110424331, |
| "learning_rate": 1.1095505617977527e-07, |
| "logps/chosen": -52.21783447265625, |
| "logps/rejected": -74.33990478515625, |
| "loss": 0.3491, |
| "losses/dpo": 0.4351283013820648, |
| "losses/sft": 2.3193869590759277, |
| "losses/total": 0.4351283013820648, |
| "ref_logps/chosen": -38.05769348144531, |
| "ref_logps/rejected": -46.181705474853516, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.4160147905349731, |
| "rewards/margins": 1.3998043537139893, |
| "rewards/rejected": -2.815819263458252, |
| "step": 317 |
| }, |
| { |
| "epoch": 2.4, |
| "grad_norm": 9.20983406154459, |
| "learning_rate": 1.095505617977528e-07, |
| "logps/chosen": -51.61767578125, |
| "logps/rejected": -72.1242446899414, |
| "loss": 0.4144, |
| "losses/dpo": 0.21413980424404144, |
| "losses/sft": 1.885907530784607, |
| "losses/total": 0.21413980424404144, |
| "ref_logps/chosen": -37.064613342285156, |
| "ref_logps/rejected": -44.01332473754883, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -1.4553061723709106, |
| "rewards/margins": 1.355785608291626, |
| "rewards/rejected": -2.811091899871826, |
| "step": 318 |
| }, |
| { |
| "epoch": 2.41, |
| "grad_norm": 9.683467779206334, |
| "learning_rate": 1.0814606741573033e-07, |
| "logps/chosen": -52.75745391845703, |
| "logps/rejected": -68.34822082519531, |
| "loss": 0.4051, |
| "losses/dpo": 0.5242694020271301, |
| "losses/sft": 1.8490060567855835, |
| "losses/total": 0.5242694020271301, |
| "ref_logps/chosen": -38.73335266113281, |
| "ref_logps/rejected": -42.334041595458984, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.4024099111557007, |
| "rewards/margins": 1.1990087032318115, |
| "rewards/rejected": -2.6014187335968018, |
| "step": 319 |
| }, |
| { |
| "epoch": 2.42, |
| "grad_norm": 10.094879725137059, |
| "learning_rate": 1.0674157303370785e-07, |
| "logps/chosen": -55.7940788269043, |
| "logps/rejected": -71.7145767211914, |
| "loss": 0.4083, |
| "losses/dpo": 0.33178359270095825, |
| "losses/sft": 2.2600364685058594, |
| "losses/total": 0.33178359270095825, |
| "ref_logps/chosen": -41.903411865234375, |
| "ref_logps/rejected": -45.71622848510742, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.389066457748413, |
| "rewards/margins": 1.2107690572738647, |
| "rewards/rejected": -2.5998356342315674, |
| "step": 320 |
| }, |
| { |
| "epoch": 2.42, |
| "grad_norm": 8.737981073393495, |
| "learning_rate": 1.0533707865168538e-07, |
| "logps/chosen": -52.43000793457031, |
| "logps/rejected": -70.14034271240234, |
| "loss": 0.4104, |
| "losses/dpo": 0.4298698902130127, |
| "losses/sft": 1.832787036895752, |
| "losses/total": 0.4298698902130127, |
| "ref_logps/chosen": -38.05841064453125, |
| "ref_logps/rejected": -43.76458740234375, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.4371598958969116, |
| "rewards/margins": 1.2004159688949585, |
| "rewards/rejected": -2.63757586479187, |
| "step": 321 |
| }, |
| { |
| "epoch": 2.43, |
| "grad_norm": 8.489155654631809, |
| "learning_rate": 1.0393258426966293e-07, |
| "logps/chosen": -54.25529479980469, |
| "logps/rejected": -76.93711853027344, |
| "loss": 0.3281, |
| "losses/dpo": 0.2001137137413025, |
| "losses/sft": 1.605088710784912, |
| "losses/total": 0.2001137137413025, |
| "ref_logps/chosen": -41.82042694091797, |
| "ref_logps/rejected": -49.489280700683594, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.2434866428375244, |
| "rewards/margins": 1.5012969970703125, |
| "rewards/rejected": -2.744783401489258, |
| "step": 322 |
| }, |
| { |
| "epoch": 2.44, |
| "grad_norm": 8.976932361180113, |
| "learning_rate": 1.0252808988764044e-07, |
| "logps/chosen": -51.483985900878906, |
| "logps/rejected": -75.30467224121094, |
| "loss": 0.3253, |
| "losses/dpo": 0.28753212094306946, |
| "losses/sft": 2.173304557800293, |
| "losses/total": 0.28753212094306946, |
| "ref_logps/chosen": -38.227813720703125, |
| "ref_logps/rejected": -47.102657318115234, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.3256170749664307, |
| "rewards/margins": 1.4945844411849976, |
| "rewards/rejected": -2.8202013969421387, |
| "step": 323 |
| }, |
| { |
| "epoch": 2.45, |
| "grad_norm": 7.593474701084862, |
| "learning_rate": 1.0112359550561797e-07, |
| "logps/chosen": -47.74129867553711, |
| "logps/rejected": -69.7999267578125, |
| "loss": 0.3669, |
| "losses/dpo": 0.43520650267601013, |
| "losses/sft": 1.8445793390274048, |
| "losses/total": 0.43520650267601013, |
| "ref_logps/chosen": -35.14937210083008, |
| "ref_logps/rejected": -43.280242919921875, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.259192943572998, |
| "rewards/margins": 1.3927757740020752, |
| "rewards/rejected": -2.6519687175750732, |
| "step": 324 |
| }, |
| { |
| "epoch": 2.45, |
| "grad_norm": 9.113140126947922, |
| "learning_rate": 9.971910112359549e-08, |
| "logps/chosen": -51.48912811279297, |
| "logps/rejected": -75.93363189697266, |
| "loss": 0.3625, |
| "losses/dpo": 0.271272748708725, |
| "losses/sft": 2.1029720306396484, |
| "losses/total": 0.271272748708725, |
| "ref_logps/chosen": -38.47105026245117, |
| "ref_logps/rejected": -48.987518310546875, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.3018079996109009, |
| "rewards/margins": 1.3928041458129883, |
| "rewards/rejected": -2.6946120262145996, |
| "step": 325 |
| }, |
| { |
| "epoch": 2.46, |
| "grad_norm": 9.203607647069793, |
| "learning_rate": 9.831460674157303e-08, |
| "logps/chosen": -56.46796417236328, |
| "logps/rejected": -72.37566375732422, |
| "loss": 0.3758, |
| "losses/dpo": 0.27879202365875244, |
| "losses/sft": 1.8894522190093994, |
| "losses/total": 0.27879202365875244, |
| "ref_logps/chosen": -43.5599365234375, |
| "ref_logps/rejected": -46.51646423339844, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.2908027172088623, |
| "rewards/margins": 1.295116901397705, |
| "rewards/rejected": -2.5859196186065674, |
| "step": 326 |
| }, |
| { |
| "epoch": 2.47, |
| "grad_norm": 8.9943305703751, |
| "learning_rate": 9.691011235955055e-08, |
| "logps/chosen": -55.962684631347656, |
| "logps/rejected": -77.36865997314453, |
| "loss": 0.383, |
| "losses/dpo": 0.2792072296142578, |
| "losses/sft": 1.8898770809173584, |
| "losses/total": 0.2792072296142578, |
| "ref_logps/chosen": -41.8868522644043, |
| "ref_logps/rejected": -48.986183166503906, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.4075829982757568, |
| "rewards/margins": 1.430665373802185, |
| "rewards/rejected": -2.8382484912872314, |
| "step": 327 |
| }, |
| { |
| "epoch": 2.48, |
| "grad_norm": 9.502289617698672, |
| "learning_rate": 9.550561797752808e-08, |
| "logps/chosen": -50.20685577392578, |
| "logps/rejected": -66.51139831542969, |
| "loss": 0.441, |
| "losses/dpo": 0.2940795123577118, |
| "losses/sft": 2.298060894012451, |
| "losses/total": 0.2940795123577118, |
| "ref_logps/chosen": -37.659244537353516, |
| "ref_logps/rejected": -42.670841217041016, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -1.254760980606079, |
| "rewards/margins": 1.129294514656067, |
| "rewards/rejected": -2.3840553760528564, |
| "step": 328 |
| }, |
| { |
| "epoch": 2.48, |
| "grad_norm": 9.657263439264016, |
| "learning_rate": 9.410112359550561e-08, |
| "logps/chosen": -54.32592010498047, |
| "logps/rejected": -69.8943862915039, |
| "loss": 0.4163, |
| "losses/dpo": 0.6257603764533997, |
| "losses/sft": 2.595241069793701, |
| "losses/total": 0.6257603764533997, |
| "ref_logps/chosen": -40.53725814819336, |
| "ref_logps/rejected": -43.76425552368164, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -1.37886643409729, |
| "rewards/margins": 1.2341458797454834, |
| "rewards/rejected": -2.6130123138427734, |
| "step": 329 |
| }, |
| { |
| "epoch": 2.49, |
| "grad_norm": 9.274122078417514, |
| "learning_rate": 9.269662921348314e-08, |
| "logps/chosen": -54.48114013671875, |
| "logps/rejected": -74.62823486328125, |
| "loss": 0.3801, |
| "losses/dpo": 0.46192625164985657, |
| "losses/sft": 2.048821449279785, |
| "losses/total": 0.46192625164985657, |
| "ref_logps/chosen": -40.628318786621094, |
| "ref_logps/rejected": -47.394065856933594, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.385282278060913, |
| "rewards/margins": 1.3381340503692627, |
| "rewards/rejected": -2.7234160900115967, |
| "step": 330 |
| }, |
| { |
| "epoch": 2.5, |
| "grad_norm": 11.530277636718996, |
| "learning_rate": 9.129213483146067e-08, |
| "logps/chosen": -51.81992721557617, |
| "logps/rejected": -78.34001922607422, |
| "loss": 0.3378, |
| "losses/dpo": 0.36353716254234314, |
| "losses/sft": 2.3431830406188965, |
| "losses/total": 0.36353716254234314, |
| "ref_logps/chosen": -38.79859161376953, |
| "ref_logps/rejected": -50.49066925048828, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.302133560180664, |
| "rewards/margins": 1.4828013181686401, |
| "rewards/rejected": -2.7849345207214355, |
| "step": 331 |
| }, |
| { |
| "epoch": 2.51, |
| "grad_norm": 9.663796087127917, |
| "learning_rate": 8.988764044943819e-08, |
| "logps/chosen": -56.09492492675781, |
| "logps/rejected": -77.04325866699219, |
| "loss": 0.3814, |
| "losses/dpo": 0.25816428661346436, |
| "losses/sft": 2.8091163635253906, |
| "losses/total": 0.25816428661346436, |
| "ref_logps/chosen": -40.704925537109375, |
| "ref_logps/rejected": -47.98851013183594, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.5390002727508545, |
| "rewards/margins": 1.3664746284484863, |
| "rewards/rejected": -2.90547513961792, |
| "step": 332 |
| }, |
| { |
| "epoch": 2.51, |
| "grad_norm": 9.817894775320998, |
| "learning_rate": 8.848314606741572e-08, |
| "logps/chosen": -53.74916076660156, |
| "logps/rejected": -71.23921966552734, |
| "loss": 0.4129, |
| "losses/dpo": 0.5282669067382812, |
| "losses/sft": 2.027956962585449, |
| "losses/total": 0.5282669067382812, |
| "ref_logps/chosen": -40.569034576416016, |
| "ref_logps/rejected": -45.03144073486328, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.3180131912231445, |
| "rewards/margins": 1.302764654159546, |
| "rewards/rejected": -2.6207778453826904, |
| "step": 333 |
| }, |
| { |
| "epoch": 2.52, |
| "grad_norm": 9.915518749588111, |
| "learning_rate": 8.707865168539325e-08, |
| "logps/chosen": -53.48023986816406, |
| "logps/rejected": -72.4287109375, |
| "loss": 0.4231, |
| "losses/dpo": 0.4929217994213104, |
| "losses/sft": 2.577164888381958, |
| "losses/total": 0.4929217994213104, |
| "ref_logps/chosen": -39.79558563232422, |
| "ref_logps/rejected": -45.82670593261719, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -1.3684654235839844, |
| "rewards/margins": 1.291735291481018, |
| "rewards/rejected": -2.660200595855713, |
| "step": 334 |
| }, |
| { |
| "epoch": 2.53, |
| "grad_norm": 9.335165389726255, |
| "learning_rate": 8.567415730337078e-08, |
| "logps/chosen": -52.41722106933594, |
| "logps/rejected": -71.85494995117188, |
| "loss": 0.3694, |
| "losses/dpo": 0.317619651556015, |
| "losses/sft": 2.0792832374572754, |
| "losses/total": 0.317619651556015, |
| "ref_logps/chosen": -39.57048416137695, |
| "ref_logps/rejected": -46.20240783691406, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.2846734523773193, |
| "rewards/margins": 1.280580997467041, |
| "rewards/rejected": -2.5652544498443604, |
| "step": 335 |
| }, |
| { |
| "epoch": 2.54, |
| "grad_norm": 9.185058383312379, |
| "learning_rate": 8.426966292134831e-08, |
| "logps/chosen": -56.19029235839844, |
| "logps/rejected": -80.49638366699219, |
| "loss": 0.3411, |
| "losses/dpo": 0.2321043312549591, |
| "losses/sft": 1.5318742990493774, |
| "losses/total": 0.2321043312549591, |
| "ref_logps/chosen": -41.53580856323242, |
| "ref_logps/rejected": -50.73744201660156, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.4654479026794434, |
| "rewards/margins": 1.5104467868804932, |
| "rewards/rejected": -2.9758946895599365, |
| "step": 336 |
| }, |
| { |
| "epoch": 2.54, |
| "grad_norm": 8.541745993770446, |
| "learning_rate": 8.286516853932583e-08, |
| "logps/chosen": -49.73480987548828, |
| "logps/rejected": -73.88976287841797, |
| "loss": 0.3512, |
| "losses/dpo": 0.2616669237613678, |
| "losses/sft": 1.7109529972076416, |
| "losses/total": 0.2616669237613678, |
| "ref_logps/chosen": -37.30692672729492, |
| "ref_logps/rejected": -46.78838348388672, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.2427881956100464, |
| "rewards/margins": 1.4673501253128052, |
| "rewards/rejected": -2.7101383209228516, |
| "step": 337 |
| }, |
| { |
| "epoch": 2.55, |
| "grad_norm": 10.42078485613911, |
| "learning_rate": 8.146067415730337e-08, |
| "logps/chosen": -52.26924514770508, |
| "logps/rejected": -67.19551086425781, |
| "loss": 0.4575, |
| "losses/dpo": 0.4895854890346527, |
| "losses/sft": 2.276334762573242, |
| "losses/total": 0.4895854890346527, |
| "ref_logps/chosen": -39.17838668823242, |
| "ref_logps/rejected": -43.06511688232422, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.3090859651565552, |
| "rewards/margins": 1.1039537191390991, |
| "rewards/rejected": -2.4130399227142334, |
| "step": 338 |
| }, |
| { |
| "epoch": 2.56, |
| "grad_norm": 10.673884752576804, |
| "learning_rate": 8.005617977528089e-08, |
| "logps/chosen": -54.1285514831543, |
| "logps/rejected": -66.32559967041016, |
| "loss": 0.4616, |
| "losses/dpo": 0.3069703280925751, |
| "losses/sft": 1.7043497562408447, |
| "losses/total": 0.3069703280925751, |
| "ref_logps/chosen": -39.07845687866211, |
| "ref_logps/rejected": -40.795433044433594, |
| "rewards/accuracies": 0.7421875, |
| "rewards/chosen": -1.5050091743469238, |
| "rewards/margins": 1.0480072498321533, |
| "rewards/rejected": -2.5530166625976562, |
| "step": 339 |
| }, |
| { |
| "epoch": 2.57, |
| "grad_norm": 7.734430324176608, |
| "learning_rate": 7.865168539325842e-08, |
| "logps/chosen": -51.84765625, |
| "logps/rejected": -73.76638793945312, |
| "loss": 0.343, |
| "losses/dpo": 0.21601220965385437, |
| "losses/sft": 1.731180191040039, |
| "losses/total": 0.21601220965385437, |
| "ref_logps/chosen": -38.94496154785156, |
| "ref_logps/rejected": -47.23394012451172, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -1.2902700901031494, |
| "rewards/margins": 1.3629752397537231, |
| "rewards/rejected": -2.653244972229004, |
| "step": 340 |
| }, |
| { |
| "epoch": 2.57, |
| "grad_norm": 9.718523923824968, |
| "learning_rate": 7.724719101123594e-08, |
| "logps/chosen": -54.3284797668457, |
| "logps/rejected": -74.42861938476562, |
| "loss": 0.4342, |
| "losses/dpo": 0.5424623489379883, |
| "losses/sft": 2.519442081451416, |
| "losses/total": 0.5424623489379883, |
| "ref_logps/chosen": -40.32830047607422, |
| "ref_logps/rejected": -47.90203094482422, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.4000180959701538, |
| "rewards/margins": 1.2526406049728394, |
| "rewards/rejected": -2.652658700942993, |
| "step": 341 |
| }, |
| { |
| "epoch": 2.58, |
| "grad_norm": 8.766478003038216, |
| "learning_rate": 7.584269662921348e-08, |
| "logps/chosen": -56.134185791015625, |
| "logps/rejected": -74.1839370727539, |
| "loss": 0.3627, |
| "losses/dpo": 0.2985873520374298, |
| "losses/sft": 2.3777570724487305, |
| "losses/total": 0.2985873520374298, |
| "ref_logps/chosen": -40.96846008300781, |
| "ref_logps/rejected": -45.8743896484375, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.5165728330612183, |
| "rewards/margins": 1.3143821954727173, |
| "rewards/rejected": -2.8309547901153564, |
| "step": 342 |
| }, |
| { |
| "epoch": 2.59, |
| "grad_norm": 10.011243908821191, |
| "learning_rate": 7.443820224719101e-08, |
| "logps/chosen": -51.62477493286133, |
| "logps/rejected": -70.33786010742188, |
| "loss": 0.4342, |
| "losses/dpo": 0.337339848279953, |
| "losses/sft": 2.341553211212158, |
| "losses/total": 0.337339848279953, |
| "ref_logps/chosen": -37.967830657958984, |
| "ref_logps/rejected": -45.28611755371094, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.365694522857666, |
| "rewards/margins": 1.1394801139831543, |
| "rewards/rejected": -2.5051746368408203, |
| "step": 343 |
| }, |
| { |
| "epoch": 2.6, |
| "grad_norm": 9.59771843166304, |
| "learning_rate": 7.303370786516853e-08, |
| "logps/chosen": -51.54905319213867, |
| "logps/rejected": -71.37974548339844, |
| "loss": 0.4085, |
| "losses/dpo": 0.39244934916496277, |
| "losses/sft": 1.864844799041748, |
| "losses/total": 0.39244934916496277, |
| "ref_logps/chosen": -38.66598129272461, |
| "ref_logps/rejected": -45.7724609375, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.2883074283599854, |
| "rewards/margins": 1.272420883178711, |
| "rewards/rejected": -2.5607285499572754, |
| "step": 344 |
| }, |
| { |
| "epoch": 2.6, |
| "grad_norm": 8.700784807594031, |
| "learning_rate": 7.162921348314606e-08, |
| "logps/chosen": -56.775753021240234, |
| "logps/rejected": -77.216796875, |
| "loss": 0.3416, |
| "losses/dpo": 0.22181375324726105, |
| "losses/sft": 2.4348971843719482, |
| "losses/total": 0.22181375324726105, |
| "ref_logps/chosen": -42.833614349365234, |
| "ref_logps/rejected": -49.19655227661133, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -1.3942136764526367, |
| "rewards/margins": 1.4078103303909302, |
| "rewards/rejected": -2.8020238876342773, |
| "step": 345 |
| }, |
| { |
| "epoch": 2.61, |
| "grad_norm": 9.30159664736646, |
| "learning_rate": 7.022471910112359e-08, |
| "logps/chosen": -48.35330581665039, |
| "logps/rejected": -68.90961456298828, |
| "loss": 0.4087, |
| "losses/dpo": 0.43862560391426086, |
| "losses/sft": 1.861382007598877, |
| "losses/total": 0.43862560391426086, |
| "ref_logps/chosen": -35.64689254760742, |
| "ref_logps/rejected": -43.799800872802734, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.270641803741455, |
| "rewards/margins": 1.240339756011963, |
| "rewards/rejected": -2.510981559753418, |
| "step": 346 |
| }, |
| { |
| "epoch": 2.62, |
| "grad_norm": 9.466845106547478, |
| "learning_rate": 6.882022471910112e-08, |
| "logps/chosen": -52.57171630859375, |
| "logps/rejected": -66.74671173095703, |
| "loss": 0.3995, |
| "losses/dpo": 0.5215581655502319, |
| "losses/sft": 2.0002975463867188, |
| "losses/total": 0.5215581655502319, |
| "ref_logps/chosen": -39.363014221191406, |
| "ref_logps/rejected": -41.616329193115234, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.3208706378936768, |
| "rewards/margins": 1.1921679973602295, |
| "rewards/rejected": -2.5130386352539062, |
| "step": 347 |
| }, |
| { |
| "epoch": 2.63, |
| "grad_norm": 8.897974125441753, |
| "learning_rate": 6.741573033707864e-08, |
| "logps/chosen": -54.96879577636719, |
| "logps/rejected": -71.28080749511719, |
| "loss": 0.4172, |
| "losses/dpo": 0.6290773749351501, |
| "losses/sft": 2.65497088432312, |
| "losses/total": 0.6290773749351501, |
| "ref_logps/chosen": -41.37034225463867, |
| "ref_logps/rejected": -45.2210578918457, |
| "rewards/accuracies": 0.796875, |
| "rewards/chosen": -1.3598453998565674, |
| "rewards/margins": 1.2461297512054443, |
| "rewards/rejected": -2.605975389480591, |
| "step": 348 |
| }, |
| { |
| "epoch": 2.63, |
| "grad_norm": 8.450179202589874, |
| "learning_rate": 6.601123595505617e-08, |
| "logps/chosen": -56.42803192138672, |
| "logps/rejected": -78.08199310302734, |
| "loss": 0.3279, |
| "losses/dpo": 0.2672095000743866, |
| "losses/sft": 1.7004587650299072, |
| "losses/total": 0.2672095000743866, |
| "ref_logps/chosen": -43.05862808227539, |
| "ref_logps/rejected": -49.85722732543945, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -1.3369402885437012, |
| "rewards/margins": 1.4855366945266724, |
| "rewards/rejected": -2.822477102279663, |
| "step": 349 |
| }, |
| { |
| "epoch": 2.64, |
| "grad_norm": 9.946490252779716, |
| "learning_rate": 6.460674157303371e-08, |
| "logps/chosen": -52.30256652832031, |
| "logps/rejected": -67.33949279785156, |
| "loss": 0.4179, |
| "losses/dpo": 0.23541654646396637, |
| "losses/sft": 1.6304823160171509, |
| "losses/total": 0.23541654646396637, |
| "ref_logps/chosen": -39.795745849609375, |
| "ref_logps/rejected": -43.05610275268555, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.2506815195083618, |
| "rewards/margins": 1.177657961845398, |
| "rewards/rejected": -2.4283392429351807, |
| "step": 350 |
| }, |
| { |
| "epoch": 2.65, |
| "grad_norm": 8.420756732795637, |
| "learning_rate": 6.320224719101123e-08, |
| "logps/chosen": -50.62635040283203, |
| "logps/rejected": -71.55708312988281, |
| "loss": 0.3839, |
| "losses/dpo": 0.3438160717487335, |
| "losses/sft": 1.8534799814224243, |
| "losses/total": 0.3438160717487335, |
| "ref_logps/chosen": -39.325897216796875, |
| "ref_logps/rejected": -46.86465835571289, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.1300455331802368, |
| "rewards/margins": 1.3391977548599243, |
| "rewards/rejected": -2.469243049621582, |
| "step": 351 |
| }, |
| { |
| "epoch": 2.66, |
| "grad_norm": 9.697968850403186, |
| "learning_rate": 6.179775280898876e-08, |
| "logps/chosen": -54.621578216552734, |
| "logps/rejected": -70.27922058105469, |
| "loss": 0.4183, |
| "losses/dpo": 0.35120806097984314, |
| "losses/sft": 2.030266284942627, |
| "losses/total": 0.35120806097984314, |
| "ref_logps/chosen": -41.76460266113281, |
| "ref_logps/rejected": -45.33925247192383, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.2856972217559814, |
| "rewards/margins": 1.2083001136779785, |
| "rewards/rejected": -2.49399733543396, |
| "step": 352 |
| }, |
| { |
| "epoch": 2.66, |
| "grad_norm": 9.120026907057964, |
| "learning_rate": 6.039325842696629e-08, |
| "logps/chosen": -52.02517318725586, |
| "logps/rejected": -74.53661346435547, |
| "loss": 0.409, |
| "losses/dpo": 0.3873208463191986, |
| "losses/sft": 1.7444610595703125, |
| "losses/total": 0.3873208463191986, |
| "ref_logps/chosen": -38.21184539794922, |
| "ref_logps/rejected": -49.15116882324219, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.3813323974609375, |
| "rewards/margins": 1.157212495803833, |
| "rewards/rejected": -2.5385448932647705, |
| "step": 353 |
| }, |
| { |
| "epoch": 2.67, |
| "grad_norm": 9.69567118291811, |
| "learning_rate": 5.898876404494382e-08, |
| "logps/chosen": -52.73221969604492, |
| "logps/rejected": -70.13288879394531, |
| "loss": 0.4226, |
| "losses/dpo": 0.3050675392150879, |
| "losses/sft": 1.7437413930892944, |
| "losses/total": 0.3050675392150879, |
| "ref_logps/chosen": -40.23515701293945, |
| "ref_logps/rejected": -45.879547119140625, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -1.2497066259384155, |
| "rewards/margins": 1.1756272315979004, |
| "rewards/rejected": -2.4253337383270264, |
| "step": 354 |
| }, |
| { |
| "epoch": 2.68, |
| "grad_norm": 8.584053557094956, |
| "learning_rate": 5.758426966292135e-08, |
| "logps/chosen": -57.64381408691406, |
| "logps/rejected": -72.0084457397461, |
| "loss": 0.3835, |
| "losses/dpo": 0.40820345282554626, |
| "losses/sft": 2.4096083641052246, |
| "losses/total": 0.40820345282554626, |
| "ref_logps/chosen": -43.7611198425293, |
| "ref_logps/rejected": -46.36372375488281, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.3882694244384766, |
| "rewards/margins": 1.176202654838562, |
| "rewards/rejected": -2.564471960067749, |
| "step": 355 |
| }, |
| { |
| "epoch": 2.69, |
| "grad_norm": 8.221246825721817, |
| "learning_rate": 5.617977528089887e-08, |
| "logps/chosen": -46.82723617553711, |
| "logps/rejected": -68.99041748046875, |
| "loss": 0.3539, |
| "losses/dpo": 0.29153013229370117, |
| "losses/sft": 1.427022099494934, |
| "losses/total": 0.29153013229370117, |
| "ref_logps/chosen": -35.502262115478516, |
| "ref_logps/rejected": -43.46721649169922, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.132498025894165, |
| "rewards/margins": 1.419821858406067, |
| "rewards/rejected": -2.5523197650909424, |
| "step": 356 |
| }, |
| { |
| "epoch": 2.69, |
| "grad_norm": 9.482797590566683, |
| "learning_rate": 5.47752808988764e-08, |
| "logps/chosen": -51.7399787902832, |
| "logps/rejected": -71.89605712890625, |
| "loss": 0.3979, |
| "losses/dpo": 0.21694956719875336, |
| "losses/sft": 1.9680830240249634, |
| "losses/total": 0.21694956719875336, |
| "ref_logps/chosen": -38.23194122314453, |
| "ref_logps/rejected": -45.57975769042969, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.350803256034851, |
| "rewards/margins": 1.2808265686035156, |
| "rewards/rejected": -2.631629705429077, |
| "step": 357 |
| }, |
| { |
| "epoch": 2.7, |
| "grad_norm": 8.17039044420366, |
| "learning_rate": 5.3370786516853926e-08, |
| "logps/chosen": -52.407249450683594, |
| "logps/rejected": -71.58457946777344, |
| "loss": 0.3517, |
| "losses/dpo": 0.45670050382614136, |
| "losses/sft": 2.3598852157592773, |
| "losses/total": 0.45670050382614136, |
| "ref_logps/chosen": -40.96891784667969, |
| "ref_logps/rejected": -45.695167541503906, |
| "rewards/accuracies": 0.8828125, |
| "rewards/chosen": -1.143832802772522, |
| "rewards/margins": 1.4451087713241577, |
| "rewards/rejected": -2.5889415740966797, |
| "step": 358 |
| }, |
| { |
| "epoch": 2.71, |
| "grad_norm": 7.803291186480779, |
| "learning_rate": 5.196629213483146e-08, |
| "logps/chosen": -47.12774658203125, |
| "logps/rejected": -70.28164672851562, |
| "loss": 0.331, |
| "losses/dpo": 0.3020516037940979, |
| "losses/sft": 1.725950837135315, |
| "losses/total": 0.3020516037940979, |
| "ref_logps/chosen": -36.658851623535156, |
| "ref_logps/rejected": -44.756195068359375, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.0468891859054565, |
| "rewards/margins": 1.5056557655334473, |
| "rewards/rejected": -2.5525450706481934, |
| "step": 359 |
| }, |
| { |
| "epoch": 2.72, |
| "grad_norm": 8.941952443820963, |
| "learning_rate": 5.056179775280899e-08, |
| "logps/chosen": -51.66205596923828, |
| "logps/rejected": -69.529296875, |
| "loss": 0.3878, |
| "losses/dpo": 0.4756266474723816, |
| "losses/sft": 1.6768076419830322, |
| "losses/total": 0.4756266474723816, |
| "ref_logps/chosen": -39.167083740234375, |
| "ref_logps/rejected": -45.206932067871094, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.2494975328445435, |
| "rewards/margins": 1.1827386617660522, |
| "rewards/rejected": -2.4322359561920166, |
| "step": 360 |
| }, |
| { |
| "epoch": 2.72, |
| "grad_norm": 9.404438765920613, |
| "learning_rate": 4.915730337078652e-08, |
| "logps/chosen": -54.10792922973633, |
| "logps/rejected": -70.66584014892578, |
| "loss": 0.385, |
| "losses/dpo": 0.25079599022865295, |
| "losses/sft": 2.628451108932495, |
| "losses/total": 0.25079599022865295, |
| "ref_logps/chosen": -41.44969177246094, |
| "ref_logps/rejected": -44.49009704589844, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.2658233642578125, |
| "rewards/margins": 1.3517518043518066, |
| "rewards/rejected": -2.6175754070281982, |
| "step": 361 |
| }, |
| { |
| "epoch": 2.73, |
| "grad_norm": 8.339761715598929, |
| "learning_rate": 4.775280898876404e-08, |
| "logps/chosen": -52.00060272216797, |
| "logps/rejected": -70.13935852050781, |
| "loss": 0.3657, |
| "losses/dpo": 0.410220742225647, |
| "losses/sft": 2.064330577850342, |
| "losses/total": 0.410220742225647, |
| "ref_logps/chosen": -38.8874626159668, |
| "ref_logps/rejected": -44.29195785522461, |
| "rewards/accuracies": 0.890625, |
| "rewards/chosen": -1.311313509941101, |
| "rewards/margins": 1.27342689037323, |
| "rewards/rejected": -2.584740400314331, |
| "step": 362 |
| }, |
| { |
| "epoch": 2.74, |
| "grad_norm": 8.695799719443274, |
| "learning_rate": 4.634831460674157e-08, |
| "logps/chosen": -54.19892120361328, |
| "logps/rejected": -70.33839416503906, |
| "loss": 0.3834, |
| "losses/dpo": 0.4246940612792969, |
| "losses/sft": 1.6766891479492188, |
| "losses/total": 0.4246940612792969, |
| "ref_logps/chosen": -40.98695755004883, |
| "ref_logps/rejected": -44.57014846801758, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -1.3211965560913086, |
| "rewards/margins": 1.255626916885376, |
| "rewards/rejected": -2.5768234729766846, |
| "step": 363 |
| }, |
| { |
| "epoch": 2.75, |
| "grad_norm": 7.953708445192091, |
| "learning_rate": 4.4943820224719096e-08, |
| "logps/chosen": -51.214752197265625, |
| "logps/rejected": -75.3336181640625, |
| "loss": 0.3295, |
| "losses/dpo": 0.17288488149642944, |
| "losses/sft": 2.220893383026123, |
| "losses/total": 0.17288488149642944, |
| "ref_logps/chosen": -38.46108627319336, |
| "ref_logps/rejected": -48.44053268432617, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -1.2753666639328003, |
| "rewards/margins": 1.4139418601989746, |
| "rewards/rejected": -2.6893081665039062, |
| "step": 364 |
| }, |
| { |
| "epoch": 2.75, |
| "grad_norm": 8.821196988330435, |
| "learning_rate": 4.3539325842696626e-08, |
| "logps/chosen": -56.51776123046875, |
| "logps/rejected": -75.40132904052734, |
| "loss": 0.35, |
| "losses/dpo": 0.169864684343338, |
| "losses/sft": 2.520303964614868, |
| "losses/total": 0.169864684343338, |
| "ref_logps/chosen": -43.79001235961914, |
| "ref_logps/rejected": -48.247989654541016, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.2727751731872559, |
| "rewards/margins": 1.4425586462020874, |
| "rewards/rejected": -2.715333938598633, |
| "step": 365 |
| }, |
| { |
| "epoch": 2.76, |
| "grad_norm": 9.689618011799487, |
| "learning_rate": 4.213483146067416e-08, |
| "logps/chosen": -57.19207000732422, |
| "logps/rejected": -72.71266174316406, |
| "loss": 0.407, |
| "losses/dpo": 0.27488580346107483, |
| "losses/sft": 1.8573498725891113, |
| "losses/total": 0.27488580346107483, |
| "ref_logps/chosen": -43.24185562133789, |
| "ref_logps/rejected": -46.41039276123047, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.3950214385986328, |
| "rewards/margins": 1.2352051734924316, |
| "rewards/rejected": -2.6302266120910645, |
| "step": 366 |
| }, |
| { |
| "epoch": 2.77, |
| "grad_norm": 9.338062839876327, |
| "learning_rate": 4.073033707865169e-08, |
| "logps/chosen": -50.966712951660156, |
| "logps/rejected": -68.7747802734375, |
| "loss": 0.4169, |
| "losses/dpo": 0.3666359782218933, |
| "losses/sft": 2.0023789405822754, |
| "losses/total": 0.3666359782218933, |
| "ref_logps/chosen": -38.069068908691406, |
| "ref_logps/rejected": -44.10002899169922, |
| "rewards/accuracies": 0.8046875, |
| "rewards/chosen": -1.2897647619247437, |
| "rewards/margins": 1.1777102947235107, |
| "rewards/rejected": -2.467475175857544, |
| "step": 367 |
| }, |
| { |
| "epoch": 2.78, |
| "grad_norm": 7.949833391744097, |
| "learning_rate": 3.932584269662921e-08, |
| "logps/chosen": -47.72471237182617, |
| "logps/rejected": -70.84202575683594, |
| "loss": 0.3803, |
| "losses/dpo": 0.3086835443973541, |
| "losses/sft": 1.9907643795013428, |
| "losses/total": 0.3086835443973541, |
| "ref_logps/chosen": -35.20145034790039, |
| "ref_logps/rejected": -44.49205780029297, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.2523258924484253, |
| "rewards/margins": 1.3826706409454346, |
| "rewards/rejected": -2.6349964141845703, |
| "step": 368 |
| }, |
| { |
| "epoch": 2.78, |
| "grad_norm": 9.235066572868517, |
| "learning_rate": 3.792134831460674e-08, |
| "logps/chosen": -52.30189895629883, |
| "logps/rejected": -70.8028335571289, |
| "loss": 0.3852, |
| "losses/dpo": 0.2891031503677368, |
| "losses/sft": 1.8405730724334717, |
| "losses/total": 0.2891031503677368, |
| "ref_logps/chosen": -39.477725982666016, |
| "ref_logps/rejected": -45.31201934814453, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.2824174165725708, |
| "rewards/margins": 1.266663908958435, |
| "rewards/rejected": -2.549081563949585, |
| "step": 369 |
| }, |
| { |
| "epoch": 2.79, |
| "grad_norm": 9.190353581688715, |
| "learning_rate": 3.6516853932584266e-08, |
| "logps/chosen": -49.18308639526367, |
| "logps/rejected": -67.69566345214844, |
| "loss": 0.4, |
| "losses/dpo": 0.3436277508735657, |
| "losses/sft": 2.0218303203582764, |
| "losses/total": 0.3436277508735657, |
| "ref_logps/chosen": -36.68975830078125, |
| "ref_logps/rejected": -43.71223449707031, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.2493327856063843, |
| "rewards/margins": 1.1490094661712646, |
| "rewards/rejected": -2.3983423709869385, |
| "step": 370 |
| }, |
| { |
| "epoch": 2.8, |
| "grad_norm": 8.148479571122722, |
| "learning_rate": 3.5112359550561796e-08, |
| "logps/chosen": -51.570220947265625, |
| "logps/rejected": -71.7831039428711, |
| "loss": 0.3307, |
| "losses/dpo": 0.5651198625564575, |
| "losses/sft": 2.007855176925659, |
| "losses/total": 0.5651198625564575, |
| "ref_logps/chosen": -40.369407653808594, |
| "ref_logps/rejected": -46.21443176269531, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.1200807094573975, |
| "rewards/margins": 1.4367868900299072, |
| "rewards/rejected": -2.5568673610687256, |
| "step": 371 |
| }, |
| { |
| "epoch": 2.81, |
| "grad_norm": 9.663237502055296, |
| "learning_rate": 3.370786516853932e-08, |
| "logps/chosen": -55.40485382080078, |
| "logps/rejected": -73.81289672851562, |
| "loss": 0.4029, |
| "losses/dpo": 0.7689430713653564, |
| "losses/sft": 1.9435756206512451, |
| "losses/total": 0.7689430713653564, |
| "ref_logps/chosen": -42.52079772949219, |
| "ref_logps/rejected": -47.747249603271484, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.2884055376052856, |
| "rewards/margins": 1.3181602954864502, |
| "rewards/rejected": -2.6065659523010254, |
| "step": 372 |
| }, |
| { |
| "epoch": 2.82, |
| "grad_norm": 8.41997319865235, |
| "learning_rate": 3.230337078651686e-08, |
| "logps/chosen": -55.254249572753906, |
| "logps/rejected": -81.1642074584961, |
| "loss": 0.3146, |
| "losses/dpo": 0.3366415500640869, |
| "losses/sft": 1.8378387689590454, |
| "losses/total": 0.3366415500640869, |
| "ref_logps/chosen": -41.7273063659668, |
| "ref_logps/rejected": -51.86448669433594, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -1.3526947498321533, |
| "rewards/margins": 1.577277421951294, |
| "rewards/rejected": -2.929971933364868, |
| "step": 373 |
| }, |
| { |
| "epoch": 2.82, |
| "grad_norm": 9.08048419718233, |
| "learning_rate": 3.089887640449438e-08, |
| "logps/chosen": -52.86750793457031, |
| "logps/rejected": -77.41828918457031, |
| "loss": 0.3294, |
| "losses/dpo": 0.23508216440677643, |
| "losses/sft": 1.5872150659561157, |
| "losses/total": 0.23508216440677643, |
| "ref_logps/chosen": -39.141883850097656, |
| "ref_logps/rejected": -49.06714630126953, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.3725626468658447, |
| "rewards/margins": 1.4625511169433594, |
| "rewards/rejected": -2.835113763809204, |
| "step": 374 |
| }, |
| { |
| "epoch": 2.83, |
| "grad_norm": 9.149313086592246, |
| "learning_rate": 2.949438202247191e-08, |
| "logps/chosen": -50.02390670776367, |
| "logps/rejected": -75.56754302978516, |
| "loss": 0.362, |
| "losses/dpo": 0.6652272939682007, |
| "losses/sft": 2.926239252090454, |
| "losses/total": 0.6652272939682007, |
| "ref_logps/chosen": -37.26597595214844, |
| "ref_logps/rejected": -48.15172576904297, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.2757928371429443, |
| "rewards/margins": 1.4657888412475586, |
| "rewards/rejected": -2.741581916809082, |
| "step": 375 |
| }, |
| { |
| "epoch": 2.84, |
| "grad_norm": 9.114150173411574, |
| "learning_rate": 2.8089887640449436e-08, |
| "logps/chosen": -55.91659164428711, |
| "logps/rejected": -74.04378509521484, |
| "loss": 0.3668, |
| "losses/dpo": 0.2893008589744568, |
| "losses/sft": 2.1376471519470215, |
| "losses/total": 0.2893008589744568, |
| "ref_logps/chosen": -42.005611419677734, |
| "ref_logps/rejected": -46.799495697021484, |
| "rewards/accuracies": 0.859375, |
| "rewards/chosen": -1.3910987377166748, |
| "rewards/margins": 1.333329677581787, |
| "rewards/rejected": -2.724428415298462, |
| "step": 376 |
| }, |
| { |
| "epoch": 2.85, |
| "grad_norm": 10.02587161317734, |
| "learning_rate": 2.6685393258426963e-08, |
| "logps/chosen": -53.21587371826172, |
| "logps/rejected": -71.60870361328125, |
| "loss": 0.4256, |
| "losses/dpo": 0.7751315236091614, |
| "losses/sft": 2.11029314994812, |
| "losses/total": 0.7751315236091614, |
| "ref_logps/chosen": -40.663108825683594, |
| "ref_logps/rejected": -46.480648040771484, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.2552767992019653, |
| "rewards/margins": 1.25752854347229, |
| "rewards/rejected": -2.512805700302124, |
| "step": 377 |
| }, |
| { |
| "epoch": 2.85, |
| "grad_norm": 8.810937586248796, |
| "learning_rate": 2.5280898876404493e-08, |
| "logps/chosen": -52.51762008666992, |
| "logps/rejected": -77.57078552246094, |
| "loss": 0.3381, |
| "losses/dpo": 0.28002500534057617, |
| "losses/sft": 1.5633399486541748, |
| "losses/total": 0.28002500534057617, |
| "ref_logps/chosen": -39.36627960205078, |
| "ref_logps/rejected": -50.626708984375, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.315134048461914, |
| "rewards/margins": 1.3792742490768433, |
| "rewards/rejected": -2.6944081783294678, |
| "step": 378 |
| }, |
| { |
| "epoch": 2.86, |
| "grad_norm": 9.293485858540686, |
| "learning_rate": 2.387640449438202e-08, |
| "logps/chosen": -51.378074645996094, |
| "logps/rejected": -64.74043273925781, |
| "loss": 0.4262, |
| "losses/dpo": 0.7660055756568909, |
| "losses/sft": 2.2007508277893066, |
| "losses/total": 0.7660055756568909, |
| "ref_logps/chosen": -39.29324722290039, |
| "ref_logps/rejected": -41.33624267578125, |
| "rewards/accuracies": 0.7890625, |
| "rewards/chosen": -1.2084828615188599, |
| "rewards/margins": 1.131935954093933, |
| "rewards/rejected": -2.340418815612793, |
| "step": 379 |
| }, |
| { |
| "epoch": 2.87, |
| "grad_norm": 8.825439696536032, |
| "learning_rate": 2.2471910112359548e-08, |
| "logps/chosen": -55.97784423828125, |
| "logps/rejected": -78.24275207519531, |
| "loss": 0.345, |
| "losses/dpo": 0.33630573749542236, |
| "losses/sft": 2.7268307209014893, |
| "losses/total": 0.33630573749542236, |
| "ref_logps/chosen": -41.14834976196289, |
| "ref_logps/rejected": -49.048004150390625, |
| "rewards/accuracies": 0.8671875, |
| "rewards/chosen": -1.4829493761062622, |
| "rewards/margins": 1.4365259408950806, |
| "rewards/rejected": -2.9194750785827637, |
| "step": 380 |
| }, |
| { |
| "epoch": 2.88, |
| "grad_norm": 8.683259186904719, |
| "learning_rate": 2.106741573033708e-08, |
| "logps/chosen": -52.257469177246094, |
| "logps/rejected": -67.984130859375, |
| "loss": 0.4087, |
| "losses/dpo": 0.3859608471393585, |
| "losses/sft": 1.8246614933013916, |
| "losses/total": 0.3859608471393585, |
| "ref_logps/chosen": -38.84019470214844, |
| "ref_logps/rejected": -43.60353088378906, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.341727375984192, |
| "rewards/margins": 1.0963327884674072, |
| "rewards/rejected": -2.4380600452423096, |
| "step": 381 |
| }, |
| { |
| "epoch": 2.88, |
| "grad_norm": 8.270857768884154, |
| "learning_rate": 1.9662921348314606e-08, |
| "logps/chosen": -54.033363342285156, |
| "logps/rejected": -77.31697082519531, |
| "loss": 0.3287, |
| "losses/dpo": 0.19281096756458282, |
| "losses/sft": 1.8291985988616943, |
| "losses/total": 0.19281096756458282, |
| "ref_logps/chosen": -39.56074523925781, |
| "ref_logps/rejected": -47.55027770996094, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.4472615718841553, |
| "rewards/margins": 1.5294086933135986, |
| "rewards/rejected": -2.9766697883605957, |
| "step": 382 |
| }, |
| { |
| "epoch": 2.89, |
| "grad_norm": 9.68267538629506, |
| "learning_rate": 1.8258426966292133e-08, |
| "logps/chosen": -53.385498046875, |
| "logps/rejected": -68.63336181640625, |
| "loss": 0.391, |
| "losses/dpo": 0.5480431318283081, |
| "losses/sft": 2.3586955070495605, |
| "losses/total": 0.5480431318283081, |
| "ref_logps/chosen": -40.53166198730469, |
| "ref_logps/rejected": -44.218746185302734, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.2853829860687256, |
| "rewards/margins": 1.156078815460205, |
| "rewards/rejected": -2.4414615631103516, |
| "step": 383 |
| }, |
| { |
| "epoch": 2.9, |
| "grad_norm": 8.588925065399472, |
| "learning_rate": 1.685393258426966e-08, |
| "logps/chosen": -53.107017517089844, |
| "logps/rejected": -73.8092041015625, |
| "loss": 0.3472, |
| "losses/dpo": 0.5091351866722107, |
| "losses/sft": 2.4279067516326904, |
| "losses/total": 0.5091351866722107, |
| "ref_logps/chosen": -39.874427795410156, |
| "ref_logps/rejected": -47.138092041015625, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.3232589960098267, |
| "rewards/margins": 1.34385085105896, |
| "rewards/rejected": -2.667109966278076, |
| "step": 384 |
| }, |
| { |
| "epoch": 2.91, |
| "grad_norm": 9.061222152581863, |
| "learning_rate": 1.544943820224719e-08, |
| "logps/chosen": -55.36473846435547, |
| "logps/rejected": -71.19318389892578, |
| "loss": 0.3925, |
| "losses/dpo": 0.7466526627540588, |
| "losses/sft": 2.359135627746582, |
| "losses/total": 0.7466526627540588, |
| "ref_logps/chosen": -41.025726318359375, |
| "ref_logps/rejected": -44.06968688964844, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.4339020252227783, |
| "rewards/margins": 1.2784475088119507, |
| "rewards/rejected": -2.7123494148254395, |
| "step": 385 |
| }, |
| { |
| "epoch": 2.91, |
| "grad_norm": 9.909969914239525, |
| "learning_rate": 1.4044943820224718e-08, |
| "logps/chosen": -51.63406753540039, |
| "logps/rejected": -77.14066314697266, |
| "loss": 0.3813, |
| "losses/dpo": 0.6047529578208923, |
| "losses/sft": 1.7509853839874268, |
| "losses/total": 0.6047529578208923, |
| "ref_logps/chosen": -37.65089416503906, |
| "ref_logps/rejected": -48.93791198730469, |
| "rewards/accuracies": 0.7734375, |
| "rewards/chosen": -1.398316740989685, |
| "rewards/margins": 1.4219584465026855, |
| "rewards/rejected": -2.82027530670166, |
| "step": 386 |
| }, |
| { |
| "epoch": 2.92, |
| "grad_norm": 8.501319268805696, |
| "learning_rate": 1.2640449438202247e-08, |
| "logps/chosen": -53.17372512817383, |
| "logps/rejected": -68.52401733398438, |
| "loss": 0.3433, |
| "losses/dpo": 0.2757856249809265, |
| "losses/sft": 2.1045045852661133, |
| "losses/total": 0.2757856249809265, |
| "ref_logps/chosen": -40.52703857421875, |
| "ref_logps/rejected": -42.728126525878906, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.264668345451355, |
| "rewards/margins": 1.3149209022521973, |
| "rewards/rejected": -2.579589366912842, |
| "step": 387 |
| }, |
| { |
| "epoch": 2.93, |
| "grad_norm": 8.718869385679747, |
| "learning_rate": 1.1235955056179774e-08, |
| "logps/chosen": -54.90761184692383, |
| "logps/rejected": -70.40953826904297, |
| "loss": 0.3789, |
| "losses/dpo": 0.5766834020614624, |
| "losses/sft": 2.222163200378418, |
| "losses/total": 0.5766834020614624, |
| "ref_logps/chosen": -41.6505012512207, |
| "ref_logps/rejected": -44.63560485839844, |
| "rewards/accuracies": 0.8515625, |
| "rewards/chosen": -1.3257105350494385, |
| "rewards/margins": 1.2516822814941406, |
| "rewards/rejected": -2.577392816543579, |
| "step": 388 |
| }, |
| { |
| "epoch": 2.94, |
| "grad_norm": 9.426380018310086, |
| "learning_rate": 9.831460674157303e-09, |
| "logps/chosen": -54.11854934692383, |
| "logps/rejected": -71.68621826171875, |
| "loss": 0.386, |
| "losses/dpo": 0.20794588327407837, |
| "losses/sft": 2.1166539192199707, |
| "losses/total": 0.20794588327407837, |
| "ref_logps/chosen": -40.966426849365234, |
| "ref_logps/rejected": -45.38508605957031, |
| "rewards/accuracies": 0.828125, |
| "rewards/chosen": -1.3152116537094116, |
| "rewards/margins": 1.3149020671844482, |
| "rewards/rejected": -2.6301136016845703, |
| "step": 389 |
| }, |
| { |
| "epoch": 2.94, |
| "grad_norm": 10.46768940793886, |
| "learning_rate": 8.42696629213483e-09, |
| "logps/chosen": -54.164424896240234, |
| "logps/rejected": -71.98051452636719, |
| "loss": 0.4893, |
| "losses/dpo": 0.39178702235221863, |
| "losses/sft": 2.2996134757995605, |
| "losses/total": 0.39178702235221863, |
| "ref_logps/chosen": -40.31678009033203, |
| "ref_logps/rejected": -46.53054428100586, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.384764313697815, |
| "rewards/margins": 1.1602333784103394, |
| "rewards/rejected": -2.5449976921081543, |
| "step": 390 |
| }, |
| { |
| "epoch": 2.95, |
| "grad_norm": 8.941416739373434, |
| "learning_rate": 7.022471910112359e-09, |
| "logps/chosen": -53.463233947753906, |
| "logps/rejected": -71.62788391113281, |
| "loss": 0.3318, |
| "losses/dpo": 0.2409718632698059, |
| "losses/sft": 1.6294231414794922, |
| "losses/total": 0.2409718632698059, |
| "ref_logps/chosen": -41.0477294921875, |
| "ref_logps/rejected": -44.89318084716797, |
| "rewards/accuracies": 0.8984375, |
| "rewards/chosen": -1.2415508031845093, |
| "rewards/margins": 1.4319190979003906, |
| "rewards/rejected": -2.6734697818756104, |
| "step": 391 |
| }, |
| { |
| "epoch": 2.96, |
| "grad_norm": 8.98210941116973, |
| "learning_rate": 5.617977528089887e-09, |
| "logps/chosen": -54.616600036621094, |
| "logps/rejected": -73.2689208984375, |
| "loss": 0.3832, |
| "losses/dpo": 0.43616408109664917, |
| "losses/sft": 2.2494640350341797, |
| "losses/total": 0.43616408109664917, |
| "ref_logps/chosen": -40.2120361328125, |
| "ref_logps/rejected": -46.42675018310547, |
| "rewards/accuracies": 0.8203125, |
| "rewards/chosen": -1.4404562711715698, |
| "rewards/margins": 1.2437611818313599, |
| "rewards/rejected": -2.684217691421509, |
| "step": 392 |
| }, |
| { |
| "epoch": 2.97, |
| "grad_norm": 9.151862468783703, |
| "learning_rate": 4.213483146067415e-09, |
| "logps/chosen": -51.817893981933594, |
| "logps/rejected": -69.46862030029297, |
| "loss": 0.3906, |
| "losses/dpo": 0.2829042077064514, |
| "losses/sft": 2.443455696105957, |
| "losses/total": 0.2829042077064514, |
| "ref_logps/chosen": -38.848846435546875, |
| "ref_logps/rejected": -43.35674285888672, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.2969045639038086, |
| "rewards/margins": 1.3142831325531006, |
| "rewards/rejected": -2.61118745803833, |
| "step": 393 |
| }, |
| { |
| "epoch": 2.97, |
| "grad_norm": 10.678776530633808, |
| "learning_rate": 2.8089887640449435e-09, |
| "logps/chosen": -54.69252014160156, |
| "logps/rejected": -73.899658203125, |
| "loss": 0.4591, |
| "losses/dpo": 0.4431733191013336, |
| "losses/sft": 2.1250791549682617, |
| "losses/total": 0.4431733191013336, |
| "ref_logps/chosen": -39.718658447265625, |
| "ref_logps/rejected": -48.00882339477539, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.4973857402801514, |
| "rewards/margins": 1.0916969776153564, |
| "rewards/rejected": -2.589082717895508, |
| "step": 394 |
| }, |
| { |
| "epoch": 2.98, |
| "grad_norm": 8.192368817594446, |
| "learning_rate": 1.4044943820224717e-09, |
| "logps/chosen": -50.57025909423828, |
| "logps/rejected": -69.49359130859375, |
| "loss": 0.3625, |
| "losses/dpo": 0.550754964351654, |
| "losses/sft": 2.1057140827178955, |
| "losses/total": 0.550754964351654, |
| "ref_logps/chosen": -38.39902877807617, |
| "ref_logps/rejected": -43.90056228637695, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.2171236276626587, |
| "rewards/margins": 1.342179298400879, |
| "rewards/rejected": -2.559302806854248, |
| "step": 395 |
| }, |
| { |
| "epoch": 2.99, |
| "grad_norm": 8.91219728509694, |
| "learning_rate": 0.0, |
| "logps/chosen": -56.36674499511719, |
| "logps/rejected": -75.95218658447266, |
| "loss": 0.3695, |
| "losses/dpo": 0.45165300369262695, |
| "losses/sft": 1.7463542222976685, |
| "losses/total": 0.45165300369262695, |
| "ref_logps/chosen": -42.22056198120117, |
| "ref_logps/rejected": -48.152099609375, |
| "rewards/accuracies": 0.8359375, |
| "rewards/chosen": -1.4146177768707275, |
| "rewards/margins": 1.3653908967971802, |
| "rewards/rejected": -2.7800087928771973, |
| "step": 396 |
| }, |
| { |
| "epoch": 2.99, |
| "step": 396, |
| "total_flos": 0.0, |
| "train_loss": 0.5140665640132596, |
| "train_runtime": 34070.7646, |
| "train_samples_per_second": 1.493, |
| "train_steps_per_second": 0.012 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 396, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 70, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
