| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.0, |
| "eval_steps": 500, |
| "global_step": 182, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.005494505494505495, |
| "grad_norm": 2.0219836235046387, |
| "learning_rate": 0.0, |
| "logits/chosen": -0.29912787675857544, |
| "logits/rejected": -0.26033276319503784, |
| "logps/chosen": -1493.069580078125, |
| "logps/rejected": -1464.670654296875, |
| "loss": 0.6931473016738892, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.01098901098901099, |
| "grad_norm": 1.2694443464279175, |
| "learning_rate": 1.3157894736842107e-07, |
| "logits/chosen": -0.5480956435203552, |
| "logits/rejected": -0.6893957853317261, |
| "logps/chosen": -1354.66455078125, |
| "logps/rejected": -1234.7755126953125, |
| "loss": 0.6931473016738892, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.016483516483516484, |
| "grad_norm": 1.7238407135009766, |
| "learning_rate": 2.6315789473684213e-07, |
| "logits/chosen": -0.27911028265953064, |
| "logits/rejected": -0.146735280752182, |
| "logps/chosen": -1381.031982421875, |
| "logps/rejected": -1174.5887451171875, |
| "loss": 0.6932641267776489, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": 0.002829666016623378, |
| "rewards/margins": 0.001636037603020668, |
| "rewards/rejected": 0.001193628297187388, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.02197802197802198, |
| "grad_norm": 1.3951466083526611, |
| "learning_rate": 3.9473684210526315e-07, |
| "logits/chosen": -0.168782040476799, |
| "logits/rejected": -0.25047171115875244, |
| "logps/chosen": -1239.2984619140625, |
| "logps/rejected": -1150.9674072265625, |
| "loss": 0.6944670081138611, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": 0.0013842176413163543, |
| "rewards/margins": -0.0002266278606839478, |
| "rewards/rejected": 0.0016108450945466757, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.027472527472527472, |
| "grad_norm": 1.1389007568359375, |
| "learning_rate": 5.263157894736843e-07, |
| "logits/chosen": -0.6456503868103027, |
| "logits/rejected": -0.6371781229972839, |
| "logps/chosen": -1409.6326904296875, |
| "logps/rejected": -1272.1806640625, |
| "loss": 0.6931549906730652, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.00566960358992219, |
| "rewards/margins": 0.0016335974214598536, |
| "rewards/rejected": 0.004036006983369589, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.03296703296703297, |
| "grad_norm": 0.974063515663147, |
| "learning_rate": 6.578947368421053e-07, |
| "logits/chosen": -0.30573540925979614, |
| "logits/rejected": -0.2751460373401642, |
| "logps/chosen": -1334.066650390625, |
| "logps/rejected": -1179.54248046875, |
| "loss": 0.6933455467224121, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.015271691605448723, |
| "rewards/margins": 0.002894277684390545, |
| "rewards/rejected": 0.012377413921058178, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.038461538461538464, |
| "grad_norm": 1.2145659923553467, |
| "learning_rate": 7.894736842105263e-07, |
| "logits/chosen": 0.048936627805233, |
| "logits/rejected": -0.05937391147017479, |
| "logps/chosen": -1329.2144775390625, |
| "logps/rejected": -1263.015869140625, |
| "loss": 0.6919921040534973, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.012406377121806145, |
| "rewards/margins": 0.0037618777714669704, |
| "rewards/rejected": 0.008644499816000462, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.04395604395604396, |
| "grad_norm": 1.1337213516235352, |
| "learning_rate": 9.210526315789474e-07, |
| "logits/chosen": -0.6898252964019775, |
| "logits/rejected": -0.5947282314300537, |
| "logps/chosen": -1531.9923095703125, |
| "logps/rejected": -1458.8387451171875, |
| "loss": 0.692238450050354, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.010655415244400501, |
| "rewards/margins": 0.006124755833297968, |
| "rewards/rejected": 0.004530658945441246, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.04945054945054945, |
| "grad_norm": 1.4570075273513794, |
| "learning_rate": 1.0526315789473685e-06, |
| "logits/chosen": -0.6079831123352051, |
| "logits/rejected": -0.6475998759269714, |
| "logps/chosen": -1284.917236328125, |
| "logps/rejected": -1239.587158203125, |
| "loss": 0.6913315653800964, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.008226280100643635, |
| "rewards/margins": 0.00454014353454113, |
| "rewards/rejected": 0.0036861367989331484, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.054945054945054944, |
| "grad_norm": 1.1168543100357056, |
| "learning_rate": 1.1842105263157894e-06, |
| "logits/chosen": -0.4832429587841034, |
| "logits/rejected": -0.6784942150115967, |
| "logps/chosen": -1411.264404296875, |
| "logps/rejected": -1368.5498046875, |
| "loss": 0.6921623349189758, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.017078913748264313, |
| "rewards/margins": 0.003607435617595911, |
| "rewards/rejected": 0.013471478596329689, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.06043956043956044, |
| "grad_norm": 1.6089200973510742, |
| "learning_rate": 1.3157894736842106e-06, |
| "logits/chosen": -0.3044881820678711, |
| "logits/rejected": -0.4217117428779602, |
| "logps/chosen": -1790.7430419921875, |
| "logps/rejected": -1541.77587890625, |
| "loss": 0.6924711465835571, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.017292898148298264, |
| "rewards/margins": 0.0005286832456476986, |
| "rewards/rejected": 0.016764217987656593, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.06593406593406594, |
| "grad_norm": 0.6998999714851379, |
| "learning_rate": 1.4473684210526317e-06, |
| "logits/chosen": -0.484161376953125, |
| "logits/rejected": -0.4741247892379761, |
| "logps/chosen": -1405.0679931640625, |
| "logps/rejected": -1307.798095703125, |
| "loss": 0.6923574209213257, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.03834030032157898, |
| "rewards/margins": 0.016229379922151566, |
| "rewards/rejected": 0.022110918536782265, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.07142857142857142, |
| "grad_norm": 0.7959760427474976, |
| "learning_rate": 1.5789473684210526e-06, |
| "logits/chosen": -0.35663121938705444, |
| "logits/rejected": -0.44265735149383545, |
| "logps/chosen": -1244.1324462890625, |
| "logps/rejected": -1132.490478515625, |
| "loss": 0.6914532780647278, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0250022504478693, |
| "rewards/margins": 0.010114078409969807, |
| "rewards/rejected": 0.014888172969222069, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.07692307692307693, |
| "grad_norm": 0.8386003971099854, |
| "learning_rate": 1.710526315789474e-06, |
| "logits/chosen": -0.19147101044654846, |
| "logits/rejected": -0.1299954354763031, |
| "logps/chosen": -1534.652099609375, |
| "logps/rejected": -1399.5792236328125, |
| "loss": 0.6917384266853333, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.04474790394306183, |
| "rewards/margins": 0.02114131860435009, |
| "rewards/rejected": 0.023606587201356888, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.08241758241758242, |
| "grad_norm": 0.7331187725067139, |
| "learning_rate": 1.8421052631578948e-06, |
| "logits/chosen": -0.444730281829834, |
| "logits/rejected": -0.5467236638069153, |
| "logps/chosen": -1175.93017578125, |
| "logps/rejected": -1152.0045166015625, |
| "loss": 0.6896647214889526, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.01556220930069685, |
| "rewards/margins": 0.008490944281220436, |
| "rewards/rejected": 0.007071265950798988, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.08791208791208792, |
| "grad_norm": 0.7852977514266968, |
| "learning_rate": 1.973684210526316e-06, |
| "logits/chosen": -0.3976231515407562, |
| "logits/rejected": -0.6108076572418213, |
| "logps/chosen": -1739.89404296875, |
| "logps/rejected": -1463.2957763671875, |
| "loss": 0.6904494166374207, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.04959982633590698, |
| "rewards/margins": 0.015834834426641464, |
| "rewards/rejected": 0.03376498818397522, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.09340659340659341, |
| "grad_norm": 0.6691881418228149, |
| "learning_rate": 2.105263157894737e-06, |
| "logits/chosen": -0.35132962465286255, |
| "logits/rejected": -0.4285813570022583, |
| "logps/chosen": -1459.74755859375, |
| "logps/rejected": -1310.574951171875, |
| "loss": 0.6907796263694763, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.03758728504180908, |
| "rewards/margins": 0.011371154338121414, |
| "rewards/rejected": 0.026216136291623116, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.0989010989010989, |
| "grad_norm": 0.8749127984046936, |
| "learning_rate": 2.236842105263158e-06, |
| "logits/chosen": -0.4039854407310486, |
| "logits/rejected": -0.5412226319313049, |
| "logps/chosen": -1535.715087890625, |
| "logps/rejected": -1592.76123046875, |
| "loss": 0.6916483044624329, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.029426174238324165, |
| "rewards/margins": 0.004588168114423752, |
| "rewards/rejected": 0.024838006123900414, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.1043956043956044, |
| "grad_norm": 0.6890782713890076, |
| "learning_rate": 2.368421052631579e-06, |
| "logits/chosen": -0.5154158473014832, |
| "logits/rejected": -0.4239461421966553, |
| "logps/chosen": -1366.4837646484375, |
| "logps/rejected": -1341.562744140625, |
| "loss": 0.6925311088562012, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.02481229603290558, |
| "rewards/margins": 0.014975612983107567, |
| "rewards/rejected": 0.009836683049798012, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.10989010989010989, |
| "grad_norm": 0.7121235728263855, |
| "learning_rate": 2.5e-06, |
| "logits/chosen": -0.24524424970149994, |
| "logits/rejected": -0.1629214882850647, |
| "logps/chosen": -1579.9754638671875, |
| "logps/rejected": -1489.90625, |
| "loss": 0.6927945613861084, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.008585358038544655, |
| "rewards/margins": 0.009461937472224236, |
| "rewards/rejected": -0.0008765789680182934, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.11538461538461539, |
| "grad_norm": 0.7799815535545349, |
| "learning_rate": 2.484662576687117e-06, |
| "logits/chosen": -0.7088998556137085, |
| "logits/rejected": -0.6623613834381104, |
| "logps/chosen": -1458.49853515625, |
| "logps/rejected": -1424.711181640625, |
| "loss": 0.6919652819633484, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.00381654710508883, |
| "rewards/margins": 0.008333091624081135, |
| "rewards/rejected": -0.0045165447518229485, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.12087912087912088, |
| "grad_norm": 0.9595739245414734, |
| "learning_rate": 2.4693251533742334e-06, |
| "logits/chosen": -0.18136945366859436, |
| "logits/rejected": -0.2704053819179535, |
| "logps/chosen": -1288.148193359375, |
| "logps/rejected": -1242.779296875, |
| "loss": 0.6891352534294128, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.004841099493205547, |
| "rewards/margins": 0.010067347437143326, |
| "rewards/rejected": -0.014908447861671448, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.12637362637362637, |
| "grad_norm": 0.8795596957206726, |
| "learning_rate": 2.45398773006135e-06, |
| "logits/chosen": -0.5372273325920105, |
| "logits/rejected": -0.4621075391769409, |
| "logps/chosen": -1187.1552734375, |
| "logps/rejected": -1210.7431640625, |
| "loss": 0.6919819116592407, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.02427225187420845, |
| "rewards/margins": 0.0051599410362541676, |
| "rewards/rejected": -0.02943219244480133, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.13186813186813187, |
| "grad_norm": 0.4124164581298828, |
| "learning_rate": 2.4386503067484667e-06, |
| "logits/chosen": -0.6080984473228455, |
| "logits/rejected": -0.6881457567214966, |
| "logps/chosen": -1473.2374267578125, |
| "logps/rejected": -1214.7386474609375, |
| "loss": 0.6928576231002808, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.050905995070934296, |
| "rewards/margins": -0.004748010542243719, |
| "rewards/rejected": -0.04615798965096474, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.13736263736263737, |
| "grad_norm": 0.2888234555721283, |
| "learning_rate": 2.423312883435583e-06, |
| "logits/chosen": -0.31380707025527954, |
| "logits/rejected": -0.2592722773551941, |
| "logps/chosen": -1513.1060791015625, |
| "logps/rejected": -1577.8150634765625, |
| "loss": 0.6926202178001404, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.06563874334096909, |
| "rewards/margins": 0.008732328191399574, |
| "rewards/rejected": -0.07437106966972351, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.14285714285714285, |
| "grad_norm": 0.5071586966514587, |
| "learning_rate": 2.4079754601226995e-06, |
| "logits/chosen": -0.6376692056655884, |
| "logits/rejected": -0.6288321614265442, |
| "logps/chosen": -1425.6273193359375, |
| "logps/rejected": -1174.107421875, |
| "loss": 0.6917777061462402, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.039514925330877304, |
| "rewards/margins": -0.0018771695904433727, |
| "rewards/rejected": -0.03763775900006294, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.14835164835164835, |
| "grad_norm": 0.7409914135932922, |
| "learning_rate": 2.392638036809816e-06, |
| "logits/chosen": -0.37571296095848083, |
| "logits/rejected": -0.3492288589477539, |
| "logps/chosen": -1434.89697265625, |
| "logps/rejected": -1126.47314453125, |
| "loss": 0.6918377876281738, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.019896959885954857, |
| "rewards/margins": 0.004404466599225998, |
| "rewards/rejected": -0.024301424622535706, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.15384615384615385, |
| "grad_norm": 1.4375437498092651, |
| "learning_rate": 2.3773006134969327e-06, |
| "logits/chosen": -0.4284808337688446, |
| "logits/rejected": -0.4831678867340088, |
| "logps/chosen": -1346.5638427734375, |
| "logps/rejected": -1136.7266845703125, |
| "loss": 0.6918051838874817, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.022301796823740005, |
| "rewards/margins": -0.003937081433832645, |
| "rewards/rejected": -0.018364714458584785, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.15934065934065933, |
| "grad_norm": 0.48184096813201904, |
| "learning_rate": 2.3619631901840493e-06, |
| "logits/chosen": -0.6718910336494446, |
| "logits/rejected": -0.7271767854690552, |
| "logps/chosen": -1507.7259521484375, |
| "logps/rejected": -1534.1622314453125, |
| "loss": 0.6906634569168091, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.05005672574043274, |
| "rewards/margins": 0.0061715878546237946, |
| "rewards/rejected": 0.043885137885808945, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.16483516483516483, |
| "grad_norm": 0.6680644750595093, |
| "learning_rate": 2.346625766871166e-06, |
| "logits/chosen": -0.5794727206230164, |
| "logits/rejected": -0.3858964741230011, |
| "logps/chosen": -1158.666748046875, |
| "logps/rejected": -1081.162109375, |
| "loss": 0.6908907890319824, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.012693958356976509, |
| "rewards/margins": 0.008182154037058353, |
| "rewards/rejected": 0.004511804785579443, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.17032967032967034, |
| "grad_norm": 0.9391990900039673, |
| "learning_rate": 2.331288343558282e-06, |
| "logits/chosen": -0.5199452638626099, |
| "logits/rejected": -0.6326333284378052, |
| "logps/chosen": -1681.7022705078125, |
| "logps/rejected": -1511.8980712890625, |
| "loss": 0.6941480040550232, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.027041777968406677, |
| "rewards/margins": -0.001771755749359727, |
| "rewards/rejected": 0.02881353348493576, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.17582417582417584, |
| "grad_norm": 0.46425074338912964, |
| "learning_rate": 2.3159509202453988e-06, |
| "logits/chosen": -0.11656185239553452, |
| "logits/rejected": -0.18580833077430725, |
| "logps/chosen": -1278.0001220703125, |
| "logps/rejected": -1106.93408203125, |
| "loss": 0.692402720451355, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.022857239469885826, |
| "rewards/margins": 0.0052628712728619576, |
| "rewards/rejected": -0.02812010981142521, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.1813186813186813, |
| "grad_norm": 0.6273348331451416, |
| "learning_rate": 2.3006134969325154e-06, |
| "logits/chosen": -0.5983235836029053, |
| "logits/rejected": -0.4410645663738251, |
| "logps/chosen": -1521.9276123046875, |
| "logps/rejected": -1621.8740234375, |
| "loss": 0.6908812522888184, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.011174187064170837, |
| "rewards/margins": 0.010386137291789055, |
| "rewards/rejected": -0.02156032808125019, |
| "step": 33 |
| }, |
| { |
| "epoch": 0.18681318681318682, |
| "grad_norm": 0.9936891794204712, |
| "learning_rate": 2.2852760736196324e-06, |
| "logits/chosen": -0.6642717719078064, |
| "logits/rejected": -0.6868173480033875, |
| "logps/chosen": -1817.156494140625, |
| "logps/rejected": -1669.7921142578125, |
| "loss": 0.6925203800201416, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.02615175023674965, |
| "rewards/margins": -0.004008379764854908, |
| "rewards/rejected": -0.022143369540572166, |
| "step": 34 |
| }, |
| { |
| "epoch": 0.19230769230769232, |
| "grad_norm": 0.297181099653244, |
| "learning_rate": 2.2699386503067486e-06, |
| "logits/chosen": -0.8816071152687073, |
| "logits/rejected": -0.7729086875915527, |
| "logps/chosen": -1493.5185546875, |
| "logps/rejected": -1499.0738525390625, |
| "loss": 0.6920033693313599, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.04162220656871796, |
| "rewards/margins": 0.004699449986219406, |
| "rewards/rejected": -0.04632166028022766, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.1978021978021978, |
| "grad_norm": 0.6562932729721069, |
| "learning_rate": 2.2546012269938652e-06, |
| "logits/chosen": -0.4074722230434418, |
| "logits/rejected": -0.5253076553344727, |
| "logps/chosen": -1445.3853759765625, |
| "logps/rejected": -1373.7484130859375, |
| "loss": 0.6925094127655029, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.038453079760074615, |
| "rewards/margins": -0.002792082028463483, |
| "rewards/rejected": -0.0356609970331192, |
| "step": 36 |
| }, |
| { |
| "epoch": 0.2032967032967033, |
| "grad_norm": 0.42993712425231934, |
| "learning_rate": 2.239263803680982e-06, |
| "logits/chosen": -0.10876737534999847, |
| "logits/rejected": -0.1684437394142151, |
| "logps/chosen": -1450.543701171875, |
| "logps/rejected": -1280.059326171875, |
| "loss": 0.6924898624420166, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.0590565986931324, |
| "rewards/margins": -0.012079971842467785, |
| "rewards/rejected": -0.04697662591934204, |
| "step": 37 |
| }, |
| { |
| "epoch": 0.2087912087912088, |
| "grad_norm": 0.6291781067848206, |
| "learning_rate": 2.2239263803680985e-06, |
| "logits/chosen": -0.4873208999633789, |
| "logits/rejected": -0.5962563753128052, |
| "logps/chosen": -1363.3477783203125, |
| "logps/rejected": -1316.7086181640625, |
| "loss": 0.6922313570976257, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.03899176046252251, |
| "rewards/margins": 0.0033729556016623974, |
| "rewards/rejected": -0.04236472398042679, |
| "step": 38 |
| }, |
| { |
| "epoch": 0.21428571428571427, |
| "grad_norm": 0.6414478421211243, |
| "learning_rate": 2.208588957055215e-06, |
| "logits/chosen": -0.3744695782661438, |
| "logits/rejected": -0.45585522055625916, |
| "logps/chosen": -1323.5474853515625, |
| "logps/rejected": -1350.0068359375, |
| "loss": 0.6929305791854858, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.0353596955537796, |
| "rewards/margins": 0.007274856325238943, |
| "rewards/rejected": -0.04263455420732498, |
| "step": 39 |
| }, |
| { |
| "epoch": 0.21978021978021978, |
| "grad_norm": 1.121484398841858, |
| "learning_rate": 2.1932515337423317e-06, |
| "logits/chosen": -0.3999641537666321, |
| "logits/rejected": -0.5990539789199829, |
| "logps/chosen": -1318.0543212890625, |
| "logps/rejected": -1136.1405029296875, |
| "loss": 0.6921414136886597, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.0097041679546237, |
| "rewards/margins": 0.002448990475386381, |
| "rewards/rejected": -0.012153157964348793, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.22527472527472528, |
| "grad_norm": 1.376361608505249, |
| "learning_rate": 2.177914110429448e-06, |
| "logits/chosen": -0.538985550403595, |
| "logits/rejected": -0.3736581802368164, |
| "logps/chosen": -1605.4700927734375, |
| "logps/rejected": -1410.381103515625, |
| "loss": 0.6900919079780579, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.015452098101377487, |
| "rewards/margins": 0.001963262911885977, |
| "rewards/rejected": -0.01741536147892475, |
| "step": 41 |
| }, |
| { |
| "epoch": 0.23076923076923078, |
| "grad_norm": 0.9494011402130127, |
| "learning_rate": 2.1625766871165645e-06, |
| "logits/chosen": -0.3661588430404663, |
| "logits/rejected": -0.30248114466667175, |
| "logps/chosen": -1482.0816650390625, |
| "logps/rejected": -1587.1827392578125, |
| "loss": 0.6923238635063171, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.011823957785964012, |
| "rewards/margins": 0.004251518286764622, |
| "rewards/rejected": -0.01607547700405121, |
| "step": 42 |
| }, |
| { |
| "epoch": 0.23626373626373626, |
| "grad_norm": 0.9896334409713745, |
| "learning_rate": 2.147239263803681e-06, |
| "logits/chosen": -0.5797193646430969, |
| "logits/rejected": -0.6257858276367188, |
| "logps/chosen": -1530.91552734375, |
| "logps/rejected": -1547.6806640625, |
| "loss": 0.6907969117164612, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.021539613604545593, |
| "rewards/margins": 0.01467025838792324, |
| "rewards/rejected": -0.036209866404533386, |
| "step": 43 |
| }, |
| { |
| "epoch": 0.24175824175824176, |
| "grad_norm": 0.9827613830566406, |
| "learning_rate": 2.1319018404907978e-06, |
| "logits/chosen": -0.7638057470321655, |
| "logits/rejected": -0.8149251937866211, |
| "logps/chosen": -1518.649169921875, |
| "logps/rejected": -1399.217529296875, |
| "loss": 0.6919956207275391, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.00867112074047327, |
| "rewards/margins": 0.0037965585943311453, |
| "rewards/rejected": -0.01246767956763506, |
| "step": 44 |
| }, |
| { |
| "epoch": 0.24725274725274726, |
| "grad_norm": 1.2675107717514038, |
| "learning_rate": 2.1165644171779144e-06, |
| "logits/chosen": -0.2081916481256485, |
| "logits/rejected": -0.17389225959777832, |
| "logps/chosen": -1245.876708984375, |
| "logps/rejected": -1316.34033203125, |
| "loss": 0.6916300654411316, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.0010032318532466888, |
| "rewards/margins": 0.013676752336323261, |
| "rewards/rejected": -0.012673520483076572, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.25274725274725274, |
| "grad_norm": 0.6310904622077942, |
| "learning_rate": 2.101226993865031e-06, |
| "logits/chosen": -0.6055498123168945, |
| "logits/rejected": -0.5735322833061218, |
| "logps/chosen": -1330.6837158203125, |
| "logps/rejected": -1323.1474609375, |
| "loss": 0.6929526925086975, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.03848934546113014, |
| "rewards/margins": 0.0024320981465280056, |
| "rewards/rejected": 0.0360572449862957, |
| "step": 46 |
| }, |
| { |
| "epoch": 0.25824175824175827, |
| "grad_norm": 0.5979546308517456, |
| "learning_rate": 2.085889570552147e-06, |
| "logits/chosen": -0.19976076483726501, |
| "logits/rejected": -0.3477155566215515, |
| "logps/chosen": -1781.2822265625, |
| "logps/rejected": -1497.507080078125, |
| "loss": 0.691817045211792, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.057623643428087234, |
| "rewards/margins": 0.01321370154619217, |
| "rewards/rejected": 0.044409941881895065, |
| "step": 47 |
| }, |
| { |
| "epoch": 0.26373626373626374, |
| "grad_norm": 1.287549376487732, |
| "learning_rate": 2.070552147239264e-06, |
| "logits/chosen": -0.09788006544113159, |
| "logits/rejected": -0.1335160732269287, |
| "logps/chosen": -1499.1783447265625, |
| "logps/rejected": -1211.41064453125, |
| "loss": 0.6919518113136292, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.03057347610592842, |
| "rewards/margins": 0.01761637255549431, |
| "rewards/rejected": 0.012957105413079262, |
| "step": 48 |
| }, |
| { |
| "epoch": 0.2692307692307692, |
| "grad_norm": 0.6202033758163452, |
| "learning_rate": 2.0552147239263804e-06, |
| "logits/chosen": -0.31665268540382385, |
| "logits/rejected": -0.3232308626174927, |
| "logps/chosen": -1485.8748779296875, |
| "logps/rejected": -1326.1854248046875, |
| "loss": 0.6928448677062988, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.07038865238428116, |
| "rewards/margins": 0.009158019907772541, |
| "rewards/rejected": 0.061230629682540894, |
| "step": 49 |
| }, |
| { |
| "epoch": 0.27472527472527475, |
| "grad_norm": 0.5841394066810608, |
| "learning_rate": 2.039877300613497e-06, |
| "logits/chosen": -0.11269676685333252, |
| "logits/rejected": -0.12115436792373657, |
| "logps/chosen": -1302.87255859375, |
| "logps/rejected": -1420.4437255859375, |
| "loss": 0.6917054057121277, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.04229995980858803, |
| "rewards/margins": 0.0015316582284867764, |
| "rewards/rejected": 0.04076829552650452, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.2802197802197802, |
| "grad_norm": 0.7443718910217285, |
| "learning_rate": 2.0245398773006137e-06, |
| "logits/chosen": -0.49943816661834717, |
| "logits/rejected": -0.5221395492553711, |
| "logps/chosen": -1615.616943359375, |
| "logps/rejected": -1565.77783203125, |
| "loss": 0.6914383172988892, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.04047242924571037, |
| "rewards/margins": 0.011208095587790012, |
| "rewards/rejected": 0.029264334589242935, |
| "step": 51 |
| }, |
| { |
| "epoch": 0.2857142857142857, |
| "grad_norm": 1.0812468528747559, |
| "learning_rate": 2.0092024539877303e-06, |
| "logits/chosen": -0.5125001072883606, |
| "logits/rejected": -0.5849840641021729, |
| "logps/chosen": -1389.961181640625, |
| "logps/rejected": -1265.2818603515625, |
| "loss": 0.6922411322593689, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.008559723384678364, |
| "rewards/margins": 0.001604671822860837, |
| "rewards/rejected": 0.006955050863325596, |
| "step": 52 |
| }, |
| { |
| "epoch": 0.29120879120879123, |
| "grad_norm": 1.38298761844635, |
| "learning_rate": 1.9938650306748465e-06, |
| "logits/chosen": -0.8314374685287476, |
| "logits/rejected": -0.8688368797302246, |
| "logps/chosen": -1503.9605712890625, |
| "logps/rejected": -1391.91357421875, |
| "loss": 0.6904184818267822, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.0023605534806847572, |
| "rewards/margins": 0.010243010707199574, |
| "rewards/rejected": -0.01260356418788433, |
| "step": 53 |
| }, |
| { |
| "epoch": 0.2967032967032967, |
| "grad_norm": 0.7907947301864624, |
| "learning_rate": 1.9785276073619635e-06, |
| "logits/chosen": -0.4663585126399994, |
| "logits/rejected": -0.5531838536262512, |
| "logps/chosen": -1565.290771484375, |
| "logps/rejected": -1368.54833984375, |
| "loss": 0.6908788681030273, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.04032035544514656, |
| "rewards/margins": 0.011873043142259121, |
| "rewards/rejected": 0.028447313234210014, |
| "step": 54 |
| }, |
| { |
| "epoch": 0.3021978021978022, |
| "grad_norm": 1.203404426574707, |
| "learning_rate": 1.96319018404908e-06, |
| "logits/chosen": -0.38841575384140015, |
| "logits/rejected": -0.43689727783203125, |
| "logps/chosen": -1580.765625, |
| "logps/rejected": -1455.60498046875, |
| "loss": 0.6918849945068359, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.011032314039766788, |
| "rewards/margins": 0.009747497737407684, |
| "rewards/rejected": 0.0012848172336816788, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.3076923076923077, |
| "grad_norm": 0.7074547410011292, |
| "learning_rate": 1.9478527607361967e-06, |
| "logits/chosen": -0.3431719243526459, |
| "logits/rejected": -0.2189161628484726, |
| "logps/chosen": -1363.068603515625, |
| "logps/rejected": -1273.5313720703125, |
| "loss": 0.6918089389801025, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.029067430645227432, |
| "rewards/margins": 0.013189269229769707, |
| "rewards/rejected": 0.015878159552812576, |
| "step": 56 |
| }, |
| { |
| "epoch": 0.3131868131868132, |
| "grad_norm": 0.8552976250648499, |
| "learning_rate": 1.932515337423313e-06, |
| "logits/chosen": -0.3242160379886627, |
| "logits/rejected": -0.36963948607444763, |
| "logps/chosen": -1408.054443359375, |
| "logps/rejected": -1515.0474853515625, |
| "loss": 0.691418468952179, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.02214316464960575, |
| "rewards/margins": 0.008005449548363686, |
| "rewards/rejected": 0.01413771416991949, |
| "step": 57 |
| }, |
| { |
| "epoch": 0.31868131868131866, |
| "grad_norm": 0.9317906498908997, |
| "learning_rate": 1.9171779141104296e-06, |
| "logits/chosen": -0.651016354560852, |
| "logits/rejected": -0.646873414516449, |
| "logps/chosen": -1354.4619140625, |
| "logps/rejected": -1355.021728515625, |
| "loss": 0.6914571523666382, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.009268360212445259, |
| "rewards/margins": 0.00419324915856123, |
| "rewards/rejected": 0.005075111519545317, |
| "step": 58 |
| }, |
| { |
| "epoch": 0.3241758241758242, |
| "grad_norm": 0.6546081304550171, |
| "learning_rate": 1.9018404907975464e-06, |
| "logits/chosen": -0.4001083970069885, |
| "logits/rejected": -0.34092509746551514, |
| "logps/chosen": -1422.7691650390625, |
| "logps/rejected": -1336.7745361328125, |
| "loss": 0.6927464008331299, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.04447000473737717, |
| "rewards/margins": 0.011220641434192657, |
| "rewards/rejected": 0.03324935957789421, |
| "step": 59 |
| }, |
| { |
| "epoch": 0.32967032967032966, |
| "grad_norm": 1.2057020664215088, |
| "learning_rate": 1.8865030674846626e-06, |
| "logits/chosen": -0.40792492032051086, |
| "logits/rejected": -0.41026824712753296, |
| "logps/chosen": -1616.24462890625, |
| "logps/rejected": -1496.211669921875, |
| "loss": 0.6902381181716919, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.019096756353974342, |
| "rewards/margins": 0.012553432956337929, |
| "rewards/rejected": 0.00654332060366869, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.33516483516483514, |
| "grad_norm": 0.8189549446105957, |
| "learning_rate": 1.8711656441717794e-06, |
| "logits/chosen": -0.40066254138946533, |
| "logits/rejected": -0.5438181757926941, |
| "logps/chosen": -1495.165771484375, |
| "logps/rejected": -1468.3994140625, |
| "loss": 0.6911718845367432, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.026379263028502464, |
| "rewards/margins": 0.010899864137172699, |
| "rewards/rejected": 0.01547939982265234, |
| "step": 61 |
| }, |
| { |
| "epoch": 0.34065934065934067, |
| "grad_norm": 0.7142252922058105, |
| "learning_rate": 1.855828220858896e-06, |
| "logits/chosen": -0.4213395416736603, |
| "logits/rejected": -0.4220888316631317, |
| "logps/chosen": -1588.5791015625, |
| "logps/rejected": -1450.305908203125, |
| "loss": 0.6918849945068359, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.044086188077926636, |
| "rewards/margins": 0.021000908687710762, |
| "rewards/rejected": 0.023085277527570724, |
| "step": 62 |
| }, |
| { |
| "epoch": 0.34615384615384615, |
| "grad_norm": 0.931616485118866, |
| "learning_rate": 1.8404907975460124e-06, |
| "logits/chosen": -0.4536420404911041, |
| "logits/rejected": -0.553905189037323, |
| "logps/chosen": -1479.9383544921875, |
| "logps/rejected": -1247.5972900390625, |
| "loss": 0.6907938122749329, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0278279110789299, |
| "rewards/margins": 0.017392883077263832, |
| "rewards/rejected": 0.010435027070343494, |
| "step": 63 |
| }, |
| { |
| "epoch": 0.3516483516483517, |
| "grad_norm": 1.1059670448303223, |
| "learning_rate": 1.825153374233129e-06, |
| "logits/chosen": -0.1391632854938507, |
| "logits/rejected": -0.29146745800971985, |
| "logps/chosen": -1544.81103515625, |
| "logps/rejected": -1467.8116455078125, |
| "loss": 0.6943617463111877, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": 0.004199332557618618, |
| "rewards/margins": -0.0030631115660071373, |
| "rewards/rejected": 0.007262444589287043, |
| "step": 64 |
| }, |
| { |
| "epoch": 0.35714285714285715, |
| "grad_norm": 2.532100200653076, |
| "learning_rate": 1.8098159509202457e-06, |
| "logits/chosen": -0.4176465570926666, |
| "logits/rejected": -0.47569915652275085, |
| "logps/chosen": -1727.229736328125, |
| "logps/rejected": -1618.333984375, |
| "loss": 0.6903217434883118, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.008169951848685741, |
| "rewards/margins": 0.0118106072768569, |
| "rewards/rejected": -0.01998055912554264, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.3626373626373626, |
| "grad_norm": 0.8319610953330994, |
| "learning_rate": 1.794478527607362e-06, |
| "logits/chosen": -0.5535542964935303, |
| "logits/rejected": -0.45863211154937744, |
| "logps/chosen": -1296.4996337890625, |
| "logps/rejected": -1305.611083984375, |
| "loss": 0.6914780139923096, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.003835830604657531, |
| "rewards/margins": 0.00748834665864706, |
| "rewards/rejected": -0.0036525153554975986, |
| "step": 66 |
| }, |
| { |
| "epoch": 0.36813186813186816, |
| "grad_norm": 1.074711799621582, |
| "learning_rate": 1.7791411042944787e-06, |
| "logits/chosen": -0.6282920241355896, |
| "logits/rejected": -0.6052903532981873, |
| "logps/chosen": -1367.955078125, |
| "logps/rejected": -1418.4644775390625, |
| "loss": 0.691602349281311, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.008508214727044106, |
| "rewards/margins": 0.0025658695958554745, |
| "rewards/rejected": -0.011074084788560867, |
| "step": 67 |
| }, |
| { |
| "epoch": 0.37362637362637363, |
| "grad_norm": 1.048543930053711, |
| "learning_rate": 1.7638036809815953e-06, |
| "logits/chosen": -0.41451916098594666, |
| "logits/rejected": -0.44973787665367126, |
| "logps/chosen": -1594.5008544921875, |
| "logps/rejected": -1379.6549072265625, |
| "loss": 0.6915735602378845, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.013657140545547009, |
| "rewards/margins": 0.010285510681569576, |
| "rewards/rejected": 0.003371629398316145, |
| "step": 68 |
| }, |
| { |
| "epoch": 0.3791208791208791, |
| "grad_norm": 1.4657976627349854, |
| "learning_rate": 1.7484662576687117e-06, |
| "logits/chosen": -0.42228737473487854, |
| "logits/rejected": -0.5371153354644775, |
| "logps/chosen": -1440.2550048828125, |
| "logps/rejected": -1317.087890625, |
| "loss": 0.6919211745262146, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.004323349334299564, |
| "rewards/margins": 0.007630740292370319, |
| "rewards/rejected": -0.0033073904924094677, |
| "step": 69 |
| }, |
| { |
| "epoch": 0.38461538461538464, |
| "grad_norm": 0.7367867827415466, |
| "learning_rate": 1.7331288343558283e-06, |
| "logits/chosen": -0.741744875907898, |
| "logits/rejected": -0.6823490858078003, |
| "logps/chosen": -1317.0145263671875, |
| "logps/rejected": -1323.838623046875, |
| "loss": 0.6901740431785583, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.011306552216410637, |
| "rewards/margins": 0.013367671519517899, |
| "rewards/rejected": -0.0020611193031072617, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.3901098901098901, |
| "grad_norm": 1.0581424236297607, |
| "learning_rate": 1.717791411042945e-06, |
| "logits/chosen": -0.36263424158096313, |
| "logits/rejected": -0.37243887782096863, |
| "logps/chosen": -1136.444580078125, |
| "logps/rejected": -1088.967529296875, |
| "loss": 0.6923285722732544, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.009710215963423252, |
| "rewards/margins": 0.0008770937565714121, |
| "rewards/rejected": 0.008833122439682484, |
| "step": 71 |
| }, |
| { |
| "epoch": 0.3956043956043956, |
| "grad_norm": 0.9798324704170227, |
| "learning_rate": 1.7024539877300614e-06, |
| "logits/chosen": -0.3991851508617401, |
| "logits/rejected": -0.3783729374408722, |
| "logps/chosen": -1416.0003662109375, |
| "logps/rejected": -1323.023681640625, |
| "loss": 0.6924745440483093, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.013836746104061604, |
| "rewards/margins": 0.005006732419133186, |
| "rewards/rejected": 0.008830012753605843, |
| "step": 72 |
| }, |
| { |
| "epoch": 0.4010989010989011, |
| "grad_norm": 0.8338879346847534, |
| "learning_rate": 1.687116564417178e-06, |
| "logits/chosen": -0.19355978071689606, |
| "logits/rejected": -0.2498997300863266, |
| "logps/chosen": -1318.318359375, |
| "logps/rejected": -1202.7623291015625, |
| "loss": 0.6920049786567688, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.010977883823215961, |
| "rewards/margins": 0.0031725564040243626, |
| "rewards/rejected": 0.0078053283505141735, |
| "step": 73 |
| }, |
| { |
| "epoch": 0.4065934065934066, |
| "grad_norm": 0.8426941633224487, |
| "learning_rate": 1.6717791411042946e-06, |
| "logits/chosen": -0.46479836106300354, |
| "logits/rejected": -0.42796429991722107, |
| "logps/chosen": -951.9476928710938, |
| "logps/rejected": -1005.59521484375, |
| "loss": 0.6904844045639038, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.011304330080747604, |
| "rewards/margins": 0.0059456657618284225, |
| "rewards/rejected": 0.005358664318919182, |
| "step": 74 |
| }, |
| { |
| "epoch": 0.41208791208791207, |
| "grad_norm": 1.5550216436386108, |
| "learning_rate": 1.656441717791411e-06, |
| "logits/chosen": -0.6027710437774658, |
| "logits/rejected": -0.632501482963562, |
| "logps/chosen": -1343.49755859375, |
| "logps/rejected": -1283.693115234375, |
| "loss": 0.6909563541412354, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.006782836746424437, |
| "rewards/margins": 0.009988861158490181, |
| "rewards/rejected": -0.003206023946404457, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.4175824175824176, |
| "grad_norm": 0.9994601607322693, |
| "learning_rate": 1.6411042944785276e-06, |
| "logits/chosen": -0.5772875547409058, |
| "logits/rejected": -0.5767691731452942, |
| "logps/chosen": -1284.95263671875, |
| "logps/rejected": -1317.3885498046875, |
| "loss": 0.6914191842079163, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.002038019709289074, |
| "rewards/margins": 0.01685798540711403, |
| "rewards/rejected": -0.01481996476650238, |
| "step": 76 |
| }, |
| { |
| "epoch": 0.4230769230769231, |
| "grad_norm": 1.4171066284179688, |
| "learning_rate": 1.6257668711656445e-06, |
| "logits/chosen": -0.4665534794330597, |
| "logits/rejected": -0.432694673538208, |
| "logps/chosen": -1695.040283203125, |
| "logps/rejected": -1551.968505859375, |
| "loss": 0.6929216384887695, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.02615145593881607, |
| "rewards/margins": -0.0027590510435402393, |
| "rewards/rejected": -0.02339240349829197, |
| "step": 77 |
| }, |
| { |
| "epoch": 0.42857142857142855, |
| "grad_norm": 1.1727432012557983, |
| "learning_rate": 1.6104294478527606e-06, |
| "logits/chosen": -0.5112898349761963, |
| "logits/rejected": -0.5043646097183228, |
| "logps/chosen": -1548.087646484375, |
| "logps/rejected": -1400.1192626953125, |
| "loss": 0.6917024254798889, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.05165726691484451, |
| "rewards/margins": -0.012301253154873848, |
| "rewards/rejected": -0.039356015622615814, |
| "step": 78 |
| }, |
| { |
| "epoch": 0.4340659340659341, |
| "grad_norm": 0.4238358736038208, |
| "learning_rate": 1.5950920245398775e-06, |
| "logits/chosen": -0.30220383405685425, |
| "logits/rejected": -0.300650417804718, |
| "logps/chosen": -1494.92333984375, |
| "logps/rejected": -1436.374267578125, |
| "loss": 0.6916438341140747, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.05291643738746643, |
| "rewards/margins": -0.008324610069394112, |
| "rewards/rejected": -0.04459183290600777, |
| "step": 79 |
| }, |
| { |
| "epoch": 0.43956043956043955, |
| "grad_norm": 0.9021973013877869, |
| "learning_rate": 1.579754601226994e-06, |
| "logits/chosen": -0.10122132301330566, |
| "logits/rejected": -0.2252417355775833, |
| "logps/chosen": -1179.5601806640625, |
| "logps/rejected": -1048.575439453125, |
| "loss": 0.6910136342048645, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.021670779213309288, |
| "rewards/margins": 0.0014739391626790166, |
| "rewards/rejected": -0.023144718259572983, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.44505494505494503, |
| "grad_norm": 0.9659970998764038, |
| "learning_rate": 1.5644171779141107e-06, |
| "logits/chosen": -0.5470824837684631, |
| "logits/rejected": -0.48158833384513855, |
| "logps/chosen": -1314.2874755859375, |
| "logps/rejected": -1259.9593505859375, |
| "loss": 0.6916049718856812, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.01713727042078972, |
| "rewards/margins": 0.005170955788344145, |
| "rewards/rejected": -0.02230822481215, |
| "step": 81 |
| }, |
| { |
| "epoch": 0.45054945054945056, |
| "grad_norm": 1.1842788457870483, |
| "learning_rate": 1.5490797546012271e-06, |
| "logits/chosen": -0.33465588092803955, |
| "logits/rejected": -0.33280038833618164, |
| "logps/chosen": -1350.5224609375, |
| "logps/rejected": -1351.6611328125, |
| "loss": 0.6924982666969299, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.011164035648107529, |
| "rewards/margins": 0.004191071260720491, |
| "rewards/rejected": -0.015355108305811882, |
| "step": 82 |
| }, |
| { |
| "epoch": 0.45604395604395603, |
| "grad_norm": 1.148301124572754, |
| "learning_rate": 1.5337423312883437e-06, |
| "logits/chosen": -0.4244360625743866, |
| "logits/rejected": -0.49099138379096985, |
| "logps/chosen": -1331.2615966796875, |
| "logps/rejected": -1181.3885498046875, |
| "loss": 0.6928058862686157, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.007538508623838425, |
| "rewards/margins": 0.0015701393131166697, |
| "rewards/rejected": -0.00910864770412445, |
| "step": 83 |
| }, |
| { |
| "epoch": 0.46153846153846156, |
| "grad_norm": 1.7485556602478027, |
| "learning_rate": 1.5184049079754604e-06, |
| "logits/chosen": -0.42688897252082825, |
| "logits/rejected": -0.5053452253341675, |
| "logps/chosen": -1439.1773681640625, |
| "logps/rejected": -1262.5604248046875, |
| "loss": 0.6921294331550598, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.0023087309673428535, |
| "rewards/margins": 0.0033039902336895466, |
| "rewards/rejected": -0.0056127216666936874, |
| "step": 84 |
| }, |
| { |
| "epoch": 0.46703296703296704, |
| "grad_norm": 1.6599191427230835, |
| "learning_rate": 1.5030674846625768e-06, |
| "logits/chosen": -0.43120938539505005, |
| "logits/rejected": -0.6010267734527588, |
| "logps/chosen": -1558.959228515625, |
| "logps/rejected": -1472.342529296875, |
| "loss": 0.6921494007110596, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.022558651864528656, |
| "rewards/margins": 0.005641758441925049, |
| "rewards/rejected": 0.016916893422603607, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.4725274725274725, |
| "grad_norm": 1.0252467393875122, |
| "learning_rate": 1.4877300613496934e-06, |
| "logits/chosen": -0.37089625000953674, |
| "logits/rejected": -0.35030075907707214, |
| "logps/chosen": -1299.443115234375, |
| "logps/rejected": -1176.6553955078125, |
| "loss": 0.6907151341438293, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.024281539022922516, |
| "rewards/margins": 0.009798645041882992, |
| "rewards/rejected": 0.014482893981039524, |
| "step": 86 |
| }, |
| { |
| "epoch": 0.47802197802197804, |
| "grad_norm": 1.026886224746704, |
| "learning_rate": 1.47239263803681e-06, |
| "logits/chosen": -0.601810872554779, |
| "logits/rejected": -0.5252466201782227, |
| "logps/chosen": -1495.2203369140625, |
| "logps/rejected": -1310.950927734375, |
| "loss": 0.6917718648910522, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0073541309684515, |
| "rewards/margins": 0.007022920064628124, |
| "rewards/rejected": 0.00033121160231530666, |
| "step": 87 |
| }, |
| { |
| "epoch": 0.4835164835164835, |
| "grad_norm": 0.6109346151351929, |
| "learning_rate": 1.4570552147239264e-06, |
| "logits/chosen": -0.3959222435951233, |
| "logits/rejected": -0.2995205819606781, |
| "logps/chosen": -1450.761474609375, |
| "logps/rejected": -1450.51953125, |
| "loss": 0.6905621886253357, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.0416596420109272, |
| "rewards/margins": 0.013335322961211205, |
| "rewards/rejected": 0.028324317187070847, |
| "step": 88 |
| }, |
| { |
| "epoch": 0.489010989010989, |
| "grad_norm": 0.8334896564483643, |
| "learning_rate": 1.441717791411043e-06, |
| "logits/chosen": -0.3118741810321808, |
| "logits/rejected": -0.4128867983818054, |
| "logps/chosen": -1281.40478515625, |
| "logps/rejected": -1296.41357421875, |
| "loss": 0.6907057762145996, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.014409005641937256, |
| "rewards/margins": 0.014574131928384304, |
| "rewards/rejected": -0.0001651242928346619, |
| "step": 89 |
| }, |
| { |
| "epoch": 0.4945054945054945, |
| "grad_norm": 0.8609052300453186, |
| "learning_rate": 1.4263803680981596e-06, |
| "logits/chosen": -0.2428097128868103, |
| "logits/rejected": -0.23511645197868347, |
| "logps/chosen": -1411.8843994140625, |
| "logps/rejected": -1272.73876953125, |
| "loss": 0.6919518709182739, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.026493338868021965, |
| "rewards/margins": 0.014896931126713753, |
| "rewards/rejected": 0.011596407741308212, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.5, |
| "grad_norm": 0.7259723544120789, |
| "learning_rate": 1.411042944785276e-06, |
| "logits/chosen": -0.11918177455663681, |
| "logits/rejected": -0.07845693826675415, |
| "logps/chosen": -1513.873779296875, |
| "logps/rejected": -1462.4080810546875, |
| "loss": 0.6918542981147766, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.0520310252904892, |
| "rewards/margins": 0.011454623192548752, |
| "rewards/rejected": 0.040576402097940445, |
| "step": 91 |
| }, |
| { |
| "epoch": 0.5054945054945055, |
| "grad_norm": 1.1002850532531738, |
| "learning_rate": 1.3957055214723927e-06, |
| "logits/chosen": -0.2559316158294678, |
| "logits/rejected": -0.200018048286438, |
| "logps/chosen": -1493.306396484375, |
| "logps/rejected": -1425.4647216796875, |
| "loss": 0.6930555105209351, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.02612033672630787, |
| "rewards/margins": -0.002100848825648427, |
| "rewards/rejected": 0.028221188113093376, |
| "step": 92 |
| }, |
| { |
| "epoch": 0.510989010989011, |
| "grad_norm": 0.7148520946502686, |
| "learning_rate": 1.3803680981595095e-06, |
| "logits/chosen": -0.367002010345459, |
| "logits/rejected": -0.4066450893878937, |
| "logps/chosen": -1357.5050048828125, |
| "logps/rejected": -1381.8837890625, |
| "loss": 0.6922594308853149, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.04670805484056473, |
| "rewards/margins": 0.009145058691501617, |
| "rewards/rejected": 0.03756299987435341, |
| "step": 93 |
| }, |
| { |
| "epoch": 0.5164835164835165, |
| "grad_norm": 1.1037565469741821, |
| "learning_rate": 1.3650306748466257e-06, |
| "logits/chosen": -0.5499804615974426, |
| "logits/rejected": -0.45308274030685425, |
| "logps/chosen": -1548.565185546875, |
| "logps/rejected": -1470.4267578125, |
| "loss": 0.6900733709335327, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.038844555616378784, |
| "rewards/margins": 0.01671256124973297, |
| "rewards/rejected": 0.022131996229290962, |
| "step": 94 |
| }, |
| { |
| "epoch": 0.521978021978022, |
| "grad_norm": 0.8843666911125183, |
| "learning_rate": 1.3496932515337425e-06, |
| "logits/chosen": 0.04877326637506485, |
| "logits/rejected": -0.005727946758270264, |
| "logps/chosen": -1560.8104248046875, |
| "logps/rejected": -1314.3270263671875, |
| "loss": 0.6904133558273315, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.030398786067962646, |
| "rewards/margins": 0.02400778792798519, |
| "rewards/rejected": 0.006391000468283892, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.5274725274725275, |
| "grad_norm": 1.0202571153640747, |
| "learning_rate": 1.3343558282208591e-06, |
| "logits/chosen": -0.12934473156929016, |
| "logits/rejected": -0.175532728433609, |
| "logps/chosen": -1101.5224609375, |
| "logps/rejected": -1186.7508544921875, |
| "loss": 0.6912416219711304, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.01369690801948309, |
| "rewards/margins": 0.005995034705847502, |
| "rewards/rejected": 0.007701873779296875, |
| "step": 96 |
| }, |
| { |
| "epoch": 0.532967032967033, |
| "grad_norm": 1.3046600818634033, |
| "learning_rate": 1.3190184049079755e-06, |
| "logits/chosen": -0.3259139955043793, |
| "logits/rejected": -0.36783546209335327, |
| "logps/chosen": -1740.535888671875, |
| "logps/rejected": -1574.8447265625, |
| "loss": 0.6932768821716309, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.02200869657099247, |
| "rewards/margins": 0.005453357473015785, |
| "rewards/rejected": 0.016555337235331535, |
| "step": 97 |
| }, |
| { |
| "epoch": 0.5384615384615384, |
| "grad_norm": 1.0123155117034912, |
| "learning_rate": 1.3036809815950922e-06, |
| "logits/chosen": -0.13210992515087128, |
| "logits/rejected": -0.04369845241308212, |
| "logps/chosen": -1536.9061279296875, |
| "logps/rejected": -1519.1163330078125, |
| "loss": 0.6910783648490906, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.01258872076869011, |
| "rewards/margins": 0.010054588317871094, |
| "rewards/rejected": 0.002534131519496441, |
| "step": 98 |
| }, |
| { |
| "epoch": 0.5439560439560439, |
| "grad_norm": 1.3854718208312988, |
| "learning_rate": 1.2883435582822088e-06, |
| "logits/chosen": -0.18825362622737885, |
| "logits/rejected": -0.24444834887981415, |
| "logps/chosen": -1398.3173828125, |
| "logps/rejected": -1364.36865234375, |
| "loss": 0.6922256946563721, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.01920771598815918, |
| "rewards/margins": 0.014828749001026154, |
| "rewards/rejected": 0.004378966521471739, |
| "step": 99 |
| }, |
| { |
| "epoch": 0.5494505494505495, |
| "grad_norm": 1.2779775857925415, |
| "learning_rate": 1.2730061349693252e-06, |
| "logits/chosen": -0.4430152475833893, |
| "logits/rejected": -0.35647666454315186, |
| "logps/chosen": -1487.2095947265625, |
| "logps/rejected": -1470.6143798828125, |
| "loss": 0.6919921040534973, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.009396498091518879, |
| "rewards/margins": 0.005829436704516411, |
| "rewards/rejected": -0.01522593479603529, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.554945054945055, |
| "grad_norm": 0.6213805675506592, |
| "learning_rate": 1.2576687116564418e-06, |
| "logits/chosen": -0.22846508026123047, |
| "logits/rejected": -0.21416452527046204, |
| "logps/chosen": -1408.79833984375, |
| "logps/rejected": -1421.5936279296875, |
| "loss": 0.6927986145019531, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.04689623415470123, |
| "rewards/margins": -0.009503163397312164, |
| "rewards/rejected": -0.03739307448267937, |
| "step": 101 |
| }, |
| { |
| "epoch": 0.5604395604395604, |
| "grad_norm": 0.9802994132041931, |
| "learning_rate": 1.2423312883435584e-06, |
| "logits/chosen": -0.505018949508667, |
| "logits/rejected": -0.5704793334007263, |
| "logps/chosen": -1257.908447265625, |
| "logps/rejected": -1249.787353515625, |
| "loss": 0.6931822896003723, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.025434236973524094, |
| "rewards/margins": -0.002385740401223302, |
| "rewards/rejected": -0.02304849587380886, |
| "step": 102 |
| }, |
| { |
| "epoch": 0.5659340659340659, |
| "grad_norm": 0.8993610739707947, |
| "learning_rate": 1.226993865030675e-06, |
| "logits/chosen": -0.41527795791625977, |
| "logits/rejected": -0.29129812121391296, |
| "logps/chosen": -1268.831787109375, |
| "logps/rejected": -1143.24609375, |
| "loss": 0.69354248046875, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.008909551426768303, |
| "rewards/margins": 0.0018367673037573695, |
| "rewards/rejected": -0.010746316984295845, |
| "step": 103 |
| }, |
| { |
| "epoch": 0.5714285714285714, |
| "grad_norm": 1.2592899799346924, |
| "learning_rate": 1.2116564417177914e-06, |
| "logits/chosen": -0.34576109051704407, |
| "logits/rejected": -0.3690977990627289, |
| "logps/chosen": -1669.214599609375, |
| "logps/rejected": -1526.4136962890625, |
| "loss": 0.6926941275596619, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.0013259842526167631, |
| "rewards/margins": 0.004025318659842014, |
| "rewards/rejected": -0.005351303145289421, |
| "step": 104 |
| }, |
| { |
| "epoch": 0.5769230769230769, |
| "grad_norm": 1.046607494354248, |
| "learning_rate": 1.196319018404908e-06, |
| "logits/chosen": -0.15831628441810608, |
| "logits/rejected": -0.37302541732788086, |
| "logps/chosen": -1464.148193359375, |
| "logps/rejected": -1303.965576171875, |
| "loss": 0.6931281685829163, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.02865537628531456, |
| "rewards/margins": -0.001078385510481894, |
| "rewards/rejected": -0.027576986700296402, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.5824175824175825, |
| "grad_norm": 0.426277756690979, |
| "learning_rate": 1.1809815950920247e-06, |
| "logits/chosen": -0.5624100565910339, |
| "logits/rejected": -0.5374657511711121, |
| "logps/chosen": -1273.535888671875, |
| "logps/rejected": -1335.310791015625, |
| "loss": 0.6924761533737183, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.0322088897228241, |
| "rewards/margins": -0.0029357431922107935, |
| "rewards/rejected": -0.029273148626089096, |
| "step": 106 |
| }, |
| { |
| "epoch": 0.5879120879120879, |
| "grad_norm": 0.6133700013160706, |
| "learning_rate": 1.165644171779141e-06, |
| "logits/chosen": -0.20700356364250183, |
| "logits/rejected": -0.22031204402446747, |
| "logps/chosen": -1394.43017578125, |
| "logps/rejected": -1308.563720703125, |
| "loss": 0.6926645636558533, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.04462941363453865, |
| "rewards/margins": -0.008419161662459373, |
| "rewards/rejected": -0.03621025010943413, |
| "step": 107 |
| }, |
| { |
| "epoch": 0.5934065934065934, |
| "grad_norm": 0.6847041845321655, |
| "learning_rate": 1.1503067484662577e-06, |
| "logits/chosen": -0.14995786547660828, |
| "logits/rejected": -0.14485234022140503, |
| "logps/chosen": -1369.47998046875, |
| "logps/rejected": -1361.496337890625, |
| "loss": 0.6923136115074158, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.03646060824394226, |
| "rewards/margins": 0.0012728970032185316, |
| "rewards/rejected": -0.03773351013660431, |
| "step": 108 |
| }, |
| { |
| "epoch": 0.5989010989010989, |
| "grad_norm": 0.6556602120399475, |
| "learning_rate": 1.1349693251533743e-06, |
| "logits/chosen": -0.17063872516155243, |
| "logits/rejected": -0.25048452615737915, |
| "logps/chosen": -1319.7943115234375, |
| "logps/rejected": -1256.775634765625, |
| "loss": 0.6931478977203369, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.025922460481524467, |
| "rewards/margins": -0.00031948136165738106, |
| "rewards/rejected": -0.025602979585528374, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.6043956043956044, |
| "grad_norm": 0.8980839848518372, |
| "learning_rate": 1.119631901840491e-06, |
| "logits/chosen": -0.3595108091831207, |
| "logits/rejected": -0.34528863430023193, |
| "logps/chosen": -1376.7236328125, |
| "logps/rejected": -1430.8155517578125, |
| "loss": 0.693233847618103, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.022598477080464363, |
| "rewards/margins": 0.000155882618855685, |
| "rewards/rejected": -0.022754356265068054, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.6098901098901099, |
| "grad_norm": 1.0571579933166504, |
| "learning_rate": 1.1042944785276075e-06, |
| "logits/chosen": -0.37562742829322815, |
| "logits/rejected": -0.4181535243988037, |
| "logps/chosen": -1414.00732421875, |
| "logps/rejected": -1326.544677734375, |
| "loss": 0.6914387345314026, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.012199820019304752, |
| "rewards/margins": 0.0014734650030732155, |
| "rewards/rejected": -0.013673287816345692, |
| "step": 111 |
| }, |
| { |
| "epoch": 0.6153846153846154, |
| "grad_norm": 1.7735028266906738, |
| "learning_rate": 1.088957055214724e-06, |
| "logits/chosen": -0.4293566346168518, |
| "logits/rejected": -0.4803142547607422, |
| "logps/chosen": -1466.178955078125, |
| "logps/rejected": -1471.9337158203125, |
| "loss": 0.6919817924499512, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.010825528763234615, |
| "rewards/margins": 0.010156643576920033, |
| "rewards/rejected": -0.020982172340154648, |
| "step": 112 |
| }, |
| { |
| "epoch": 0.6208791208791209, |
| "grad_norm": 1.0000537633895874, |
| "learning_rate": 1.0736196319018406e-06, |
| "logits/chosen": -0.5714986324310303, |
| "logits/rejected": -0.5201742053031921, |
| "logps/chosen": -1415.6246337890625, |
| "logps/rejected": -1385.709716796875, |
| "loss": 0.6931148767471313, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.03592289984226227, |
| "rewards/margins": 0.010911607183516026, |
| "rewards/rejected": 0.02501128986477852, |
| "step": 113 |
| }, |
| { |
| "epoch": 0.6263736263736264, |
| "grad_norm": 0.8961329460144043, |
| "learning_rate": 1.0582822085889572e-06, |
| "logits/chosen": -0.45078492164611816, |
| "logits/rejected": -0.4087766408920288, |
| "logps/chosen": -1351.2510986328125, |
| "logps/rejected": -1375.3194580078125, |
| "loss": 0.6920664310455322, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.029718702659010887, |
| "rewards/margins": 0.006335887126624584, |
| "rewards/rejected": 0.02338281460106373, |
| "step": 114 |
| }, |
| { |
| "epoch": 0.6318681318681318, |
| "grad_norm": 0.9695339202880859, |
| "learning_rate": 1.0429447852760736e-06, |
| "logits/chosen": -0.5634657144546509, |
| "logits/rejected": -0.6761491894721985, |
| "logps/chosen": -1333.12109375, |
| "logps/rejected": -1317.586669921875, |
| "loss": 0.6914012432098389, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.01340350229293108, |
| "rewards/margins": 0.008562865667045116, |
| "rewards/rejected": 0.0048406366258859634, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.6373626373626373, |
| "grad_norm": 0.8852068185806274, |
| "learning_rate": 1.0276073619631902e-06, |
| "logits/chosen": -0.2876928150653839, |
| "logits/rejected": -0.397905558347702, |
| "logps/chosen": -1615.9324951171875, |
| "logps/rejected": -1533.5203857421875, |
| "loss": 0.6928586959838867, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.00988774187862873, |
| "rewards/margins": 0.0017955403309315443, |
| "rewards/rejected": -0.011683283373713493, |
| "step": 116 |
| }, |
| { |
| "epoch": 0.6428571428571429, |
| "grad_norm": 0.9956200122833252, |
| "learning_rate": 1.0122699386503068e-06, |
| "logits/chosen": -0.7941329479217529, |
| "logits/rejected": -0.8391545414924622, |
| "logps/chosen": -1485.06884765625, |
| "logps/rejected": -1504.8533935546875, |
| "loss": 0.6929861307144165, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.005250845104455948, |
| "rewards/margins": -0.0009261607192456722, |
| "rewards/rejected": -0.004324684850871563, |
| "step": 117 |
| }, |
| { |
| "epoch": 0.6483516483516484, |
| "grad_norm": 0.8900634050369263, |
| "learning_rate": 9.969325153374232e-07, |
| "logits/chosen": -0.4403144121170044, |
| "logits/rejected": -0.3117816746234894, |
| "logps/chosen": -1124.2105712890625, |
| "logps/rejected": -1162.626220703125, |
| "loss": 0.6915737390518188, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.018135830760002136, |
| "rewards/margins": 0.004251901991665363, |
| "rewards/rejected": -0.022387731820344925, |
| "step": 118 |
| }, |
| { |
| "epoch": 0.6538461538461539, |
| "grad_norm": 1.519210934638977, |
| "learning_rate": 9.8159509202454e-07, |
| "logits/chosen": -0.3049584925174713, |
| "logits/rejected": -0.348927766084671, |
| "logps/chosen": -1318.882568359375, |
| "logps/rejected": -1179.972900390625, |
| "loss": 0.6907892227172852, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.009297370910644531, |
| "rewards/margins": 0.0032487385906279087, |
| "rewards/rejected": -0.012546109966933727, |
| "step": 119 |
| }, |
| { |
| "epoch": 0.6593406593406593, |
| "grad_norm": 1.4839621782302856, |
| "learning_rate": 9.662576687116565e-07, |
| "logits/chosen": -0.7371286153793335, |
| "logits/rejected": -0.813791036605835, |
| "logps/chosen": -1551.0048828125, |
| "logps/rejected": -1457.481201171875, |
| "loss": 0.692111074924469, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.006462344899773598, |
| "rewards/margins": 0.0030792569741606712, |
| "rewards/rejected": -0.009541602805256844, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.6648351648351648, |
| "grad_norm": 1.420235514640808, |
| "learning_rate": 9.509202453987732e-07, |
| "logits/chosen": -0.3957596719264984, |
| "logits/rejected": -0.44786208868026733, |
| "logps/chosen": -1695.80517578125, |
| "logps/rejected": -1655.417236328125, |
| "loss": 0.6925563812255859, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.018932923674583435, |
| "rewards/margins": 0.0009399798000231385, |
| "rewards/rejected": -0.01987290196120739, |
| "step": 121 |
| }, |
| { |
| "epoch": 0.6703296703296703, |
| "grad_norm": 1.4338594675064087, |
| "learning_rate": 9.355828220858897e-07, |
| "logits/chosen": -0.6365727186203003, |
| "logits/rejected": -0.6079616546630859, |
| "logps/chosen": -1449.3353271484375, |
| "logps/rejected": -1306.079345703125, |
| "loss": 0.6926460862159729, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.01147129014134407, |
| "rewards/margins": 0.005524273030459881, |
| "rewards/rejected": -0.016995562240481377, |
| "step": 122 |
| }, |
| { |
| "epoch": 0.6758241758241759, |
| "grad_norm": 1.0156636238098145, |
| "learning_rate": 9.202453987730062e-07, |
| "logits/chosen": -0.2907084822654724, |
| "logits/rejected": -0.3623642325401306, |
| "logps/chosen": -1366.35009765625, |
| "logps/rejected": -1161.46875, |
| "loss": 0.6904911398887634, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.017240682616829872, |
| "rewards/margins": 0.014017186127603054, |
| "rewards/rejected": 0.0032234953250736, |
| "step": 123 |
| }, |
| { |
| "epoch": 0.6813186813186813, |
| "grad_norm": 1.0583997964859009, |
| "learning_rate": 9.049079754601228e-07, |
| "logits/chosen": -0.7642356157302856, |
| "logits/rejected": -0.7943760752677917, |
| "logps/chosen": -1484.6651611328125, |
| "logps/rejected": -1499.005126953125, |
| "loss": 0.6919360160827637, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.029063954949378967, |
| "rewards/margins": 0.013380720280110836, |
| "rewards/rejected": 0.015683233737945557, |
| "step": 124 |
| }, |
| { |
| "epoch": 0.6868131868131868, |
| "grad_norm": 1.5796548128128052, |
| "learning_rate": 8.895705521472393e-07, |
| "logits/chosen": -0.727140486240387, |
| "logits/rejected": -0.7532109022140503, |
| "logps/chosen": -1440.1063232421875, |
| "logps/rejected": -1624.1826171875, |
| "loss": 0.6913582682609558, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -2.9935792554169893e-05, |
| "rewards/margins": 0.007877510972321033, |
| "rewards/rejected": -0.007907447405159473, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.6923076923076923, |
| "grad_norm": 0.86751389503479, |
| "learning_rate": 8.742331288343559e-07, |
| "logits/chosen": -0.389473021030426, |
| "logits/rejected": -0.41523274779319763, |
| "logps/chosen": -1585.4542236328125, |
| "logps/rejected": -1302.420166015625, |
| "loss": 0.6916378736495972, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.022927483543753624, |
| "rewards/margins": -0.0011927932500839233, |
| "rewards/rejected": -0.02173468843102455, |
| "step": 126 |
| }, |
| { |
| "epoch": 0.6978021978021978, |
| "grad_norm": 0.7973591089248657, |
| "learning_rate": 8.588957055214725e-07, |
| "logits/chosen": -0.3697919547557831, |
| "logits/rejected": -0.32609784603118896, |
| "logps/chosen": -1386.21240234375, |
| "logps/rejected": -1433.26171875, |
| "loss": 0.6924079656600952, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.024928748607635498, |
| "rewards/margins": 0.0017317293677479029, |
| "rewards/rejected": -0.026660479605197906, |
| "step": 127 |
| }, |
| { |
| "epoch": 0.7032967032967034, |
| "grad_norm": 0.8267586827278137, |
| "learning_rate": 8.43558282208589e-07, |
| "logits/chosen": -0.39392054080963135, |
| "logits/rejected": -0.4094931483268738, |
| "logps/chosen": -1449.90380859375, |
| "logps/rejected": -1335.430908203125, |
| "loss": 0.6908578872680664, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.037380997091531754, |
| "rewards/margins": 0.0011315422598272562, |
| "rewards/rejected": -0.038512542843818665, |
| "step": 128 |
| }, |
| { |
| "epoch": 0.7087912087912088, |
| "grad_norm": 1.491594910621643, |
| "learning_rate": 8.282208588957055e-07, |
| "logits/chosen": -0.3594258725643158, |
| "logits/rejected": -0.37132909893989563, |
| "logps/chosen": -1539.3756103515625, |
| "logps/rejected": -1565.0859375, |
| "loss": 0.6924813985824585, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.011002148501574993, |
| "rewards/margins": 0.0007907864637672901, |
| "rewards/rejected": -0.011792936362326145, |
| "step": 129 |
| }, |
| { |
| "epoch": 0.7142857142857143, |
| "grad_norm": 0.961751401424408, |
| "learning_rate": 8.128834355828222e-07, |
| "logits/chosen": -0.5297249555587769, |
| "logits/rejected": -0.45530810952186584, |
| "logps/chosen": -1412.378173828125, |
| "logps/rejected": -1533.008544921875, |
| "loss": 0.6914486885070801, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.0015569772804155946, |
| "rewards/margins": 0.0011243524495512247, |
| "rewards/rejected": 0.0004326249472796917, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.7197802197802198, |
| "grad_norm": 1.0202986001968384, |
| "learning_rate": 7.975460122699387e-07, |
| "logits/chosen": -0.2956511974334717, |
| "logits/rejected": -0.3015546500682831, |
| "logps/chosen": -1375.919921875, |
| "logps/rejected": -1224.4910888671875, |
| "loss": 0.6927995681762695, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.004763889592140913, |
| "rewards/margins": -0.001240363228134811, |
| "rewards/rejected": -0.003523525781929493, |
| "step": 131 |
| }, |
| { |
| "epoch": 0.7252747252747253, |
| "grad_norm": 1.1129013299942017, |
| "learning_rate": 7.822085889570554e-07, |
| "logits/chosen": -0.33333620429039, |
| "logits/rejected": -0.3021361827850342, |
| "logps/chosen": -1697.37109375, |
| "logps/rejected": -1630.856689453125, |
| "loss": 0.6946622729301453, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.0037074373103678226, |
| "rewards/margins": -0.003561133984476328, |
| "rewards/rejected": -0.00014630379155278206, |
| "step": 132 |
| }, |
| { |
| "epoch": 0.7307692307692307, |
| "grad_norm": 0.7518520355224609, |
| "learning_rate": 7.668711656441719e-07, |
| "logits/chosen": -0.8502654433250427, |
| "logits/rejected": -0.8670777678489685, |
| "logps/chosen": -1536.0880126953125, |
| "logps/rejected": -1519.32666015625, |
| "loss": 0.6922348737716675, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.001666030497290194, |
| "rewards/margins": 0.007148093543946743, |
| "rewards/rejected": -0.005482063163071871, |
| "step": 133 |
| }, |
| { |
| "epoch": 0.7362637362637363, |
| "grad_norm": 1.1008754968643188, |
| "learning_rate": 7.515337423312884e-07, |
| "logits/chosen": -0.4814603924751282, |
| "logits/rejected": -0.43353089690208435, |
| "logps/chosen": -1490.4459228515625, |
| "logps/rejected": -1474.66162109375, |
| "loss": 0.6931823492050171, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.004679413046687841, |
| "rewards/margins": 0.005811995826661587, |
| "rewards/rejected": -0.010491408407688141, |
| "step": 134 |
| }, |
| { |
| "epoch": 0.7417582417582418, |
| "grad_norm": 1.0830036401748657, |
| "learning_rate": 7.36196319018405e-07, |
| "logits/chosen": -0.5737816691398621, |
| "logits/rejected": -0.5043767690658569, |
| "logps/chosen": -1319.7476806640625, |
| "logps/rejected": -1288.7066650390625, |
| "loss": 0.6926530599594116, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.0063733747228980064, |
| "rewards/margins": -0.0013508339179679751, |
| "rewards/rejected": 0.007724207825958729, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.7472527472527473, |
| "grad_norm": 1.152644157409668, |
| "learning_rate": 7.208588957055215e-07, |
| "logits/chosen": -0.4295804798603058, |
| "logits/rejected": -0.39793089032173157, |
| "logps/chosen": -1226.8475341796875, |
| "logps/rejected": -1343.85107421875, |
| "loss": 0.6914076805114746, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.03230779990553856, |
| "rewards/margins": 0.011389164254069328, |
| "rewards/rejected": 0.02091863378882408, |
| "step": 136 |
| }, |
| { |
| "epoch": 0.7527472527472527, |
| "grad_norm": 0.779612123966217, |
| "learning_rate": 7.05521472392638e-07, |
| "logits/chosen": -0.689027726650238, |
| "logits/rejected": -0.6783583164215088, |
| "logps/chosen": -1387.3411865234375, |
| "logps/rejected": -1337.02685546875, |
| "loss": 0.6930306553840637, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.029504049569368362, |
| "rewards/margins": 0.007792205549776554, |
| "rewards/rejected": 0.021711843088269234, |
| "step": 137 |
| }, |
| { |
| "epoch": 0.7582417582417582, |
| "grad_norm": 0.6696953773498535, |
| "learning_rate": 6.901840490797547e-07, |
| "logits/chosen": -0.4422488212585449, |
| "logits/rejected": -0.40494176745414734, |
| "logps/chosen": -1538.20556640625, |
| "logps/rejected": -1443.9151611328125, |
| "loss": 0.6923375725746155, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.03989330306649208, |
| "rewards/margins": 0.011228306218981743, |
| "rewards/rejected": 0.02866499498486519, |
| "step": 138 |
| }, |
| { |
| "epoch": 0.7637362637362637, |
| "grad_norm": 1.1540745496749878, |
| "learning_rate": 6.748466257668713e-07, |
| "logits/chosen": -0.40560609102249146, |
| "logits/rejected": -0.41341060400009155, |
| "logps/chosen": -1528.6448974609375, |
| "logps/rejected": -1515.0938720703125, |
| "loss": 0.6929686069488525, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.017328977584838867, |
| "rewards/margins": 0.005272225942462683, |
| "rewards/rejected": 0.012056750245392323, |
| "step": 139 |
| }, |
| { |
| "epoch": 0.7692307692307693, |
| "grad_norm": 0.9930852651596069, |
| "learning_rate": 6.595092024539878e-07, |
| "logits/chosen": -0.07321417331695557, |
| "logits/rejected": -0.08877704292535782, |
| "logps/chosen": -1580.5616455078125, |
| "logps/rejected": -1326.6715087890625, |
| "loss": 0.6931515336036682, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.02595691569149494, |
| "rewards/margins": 0.0032394053414463997, |
| "rewards/rejected": 0.022717511281371117, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.7747252747252747, |
| "grad_norm": 1.2813407182693481, |
| "learning_rate": 6.441717791411044e-07, |
| "logits/chosen": -0.3404533267021179, |
| "logits/rejected": -0.39101195335388184, |
| "logps/chosen": -1059.5224609375, |
| "logps/rejected": -1201.876953125, |
| "loss": 0.6932750940322876, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.011713255196809769, |
| "rewards/margins": 0.004767118953168392, |
| "rewards/rejected": 0.0069461362436413765, |
| "step": 141 |
| }, |
| { |
| "epoch": 0.7802197802197802, |
| "grad_norm": 0.8233911991119385, |
| "learning_rate": 6.288343558282209e-07, |
| "logits/chosen": -0.4058606028556824, |
| "logits/rejected": -0.2887468934059143, |
| "logps/chosen": -993.4666137695312, |
| "logps/rejected": -1229.0377197265625, |
| "loss": 0.6933331489562988, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.021297745406627655, |
| "rewards/margins": 0.007736450061202049, |
| "rewards/rejected": 0.01356129627674818, |
| "step": 142 |
| }, |
| { |
| "epoch": 0.7857142857142857, |
| "grad_norm": 0.9581381678581238, |
| "learning_rate": 6.134969325153375e-07, |
| "logits/chosen": -0.08297372609376907, |
| "logits/rejected": -0.16077230870723724, |
| "logps/chosen": -1555.29931640625, |
| "logps/rejected": -1433.138916015625, |
| "loss": 0.6934653520584106, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.03898211941123009, |
| "rewards/margins": 0.010908895172178745, |
| "rewards/rejected": 0.028073228895664215, |
| "step": 143 |
| }, |
| { |
| "epoch": 0.7912087912087912, |
| "grad_norm": 1.3117345571517944, |
| "learning_rate": 5.98159509202454e-07, |
| "logits/chosen": -0.2987571656703949, |
| "logits/rejected": -0.33382365107536316, |
| "logps/chosen": -1226.4559326171875, |
| "logps/rejected": -1198.4481201171875, |
| "loss": 0.6925820112228394, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.0050286054611206055, |
| "rewards/margins": 0.005197577178478241, |
| "rewards/rejected": -0.0001689721830189228, |
| "step": 144 |
| }, |
| { |
| "epoch": 0.7967032967032966, |
| "grad_norm": 1.4236594438552856, |
| "learning_rate": 5.828220858895705e-07, |
| "logits/chosen": -0.49541574716567993, |
| "logits/rejected": -0.4821420907974243, |
| "logps/chosen": -1369.834716796875, |
| "logps/rejected": -1285.265625, |
| "loss": 0.6933659315109253, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.005504836328327656, |
| "rewards/margins": 0.00019674794748425484, |
| "rewards/rejected": -0.005701584275811911, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.8021978021978022, |
| "grad_norm": 1.0803322792053223, |
| "learning_rate": 5.674846625766872e-07, |
| "logits/chosen": -0.41038578748703003, |
| "logits/rejected": -0.22637057304382324, |
| "logps/chosen": -1731.6552734375, |
| "logps/rejected": -1467.64599609375, |
| "loss": 0.6921501755714417, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.021399000659585, |
| "rewards/margins": 0.0027879904955625534, |
| "rewards/rejected": -0.024186991155147552, |
| "step": 146 |
| }, |
| { |
| "epoch": 0.8076923076923077, |
| "grad_norm": 1.168318748474121, |
| "learning_rate": 5.521472392638038e-07, |
| "logits/chosen": -0.034145765006542206, |
| "logits/rejected": -0.19365057349205017, |
| "logps/chosen": -1311.8702392578125, |
| "logps/rejected": -1272.918701171875, |
| "loss": 0.6925601959228516, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.009253215976059437, |
| "rewards/margins": 0.007820549421012402, |
| "rewards/rejected": 0.0014326669042930007, |
| "step": 147 |
| }, |
| { |
| "epoch": 0.8131868131868132, |
| "grad_norm": 1.2571437358856201, |
| "learning_rate": 5.368098159509203e-07, |
| "logits/chosen": -0.6215896010398865, |
| "logits/rejected": -0.6399307250976562, |
| "logps/chosen": -1441.660888671875, |
| "logps/rejected": -1511.095947265625, |
| "loss": 0.6926032304763794, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.008588919416069984, |
| "rewards/margins": 0.0011720561888068914, |
| "rewards/rejected": 0.007416863460093737, |
| "step": 148 |
| }, |
| { |
| "epoch": 0.8186813186813187, |
| "grad_norm": 1.3910409212112427, |
| "learning_rate": 5.214723926380368e-07, |
| "logits/chosen": -0.7271393537521362, |
| "logits/rejected": -0.761420726776123, |
| "logps/chosen": -1419.0078125, |
| "logps/rejected": -1475.87353515625, |
| "loss": 0.6915566325187683, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.022086886689066887, |
| "rewards/margins": 0.0090789794921875, |
| "rewards/rejected": 0.013007907196879387, |
| "step": 149 |
| }, |
| { |
| "epoch": 0.8241758241758241, |
| "grad_norm": 1.4809271097183228, |
| "learning_rate": 5.061349693251534e-07, |
| "logits/chosen": -0.7618609666824341, |
| "logits/rejected": -0.7325096130371094, |
| "logps/chosen": -1367.3497314453125, |
| "logps/rejected": -1415.16943359375, |
| "loss": 0.6933605074882507, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.014094429090619087, |
| "rewards/margins": 0.0031739422120153904, |
| "rewards/rejected": 0.010920487344264984, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.8296703296703297, |
| "grad_norm": 1.8276501893997192, |
| "learning_rate": 4.9079754601227e-07, |
| "logits/chosen": -0.8397077322006226, |
| "logits/rejected": -0.7813823223114014, |
| "logps/chosen": -1705.48486328125, |
| "logps/rejected": -1569.8170166015625, |
| "loss": 0.6919160485267639, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.012699166312813759, |
| "rewards/margins": 0.012700291350483894, |
| "rewards/rejected": -1.1255033314228058e-06, |
| "step": 151 |
| }, |
| { |
| "epoch": 0.8351648351648352, |
| "grad_norm": 1.1467257738113403, |
| "learning_rate": 4.754601226993866e-07, |
| "logits/chosen": -0.6820580959320068, |
| "logits/rejected": -0.8008350729942322, |
| "logps/chosen": -1449.619140625, |
| "logps/rejected": -1359.572021484375, |
| "loss": 0.6915912628173828, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.024284247308969498, |
| "rewards/margins": 0.009465246461331844, |
| "rewards/rejected": 0.014819002710282803, |
| "step": 152 |
| }, |
| { |
| "epoch": 0.8406593406593407, |
| "grad_norm": 1.1940540075302124, |
| "learning_rate": 4.601226993865031e-07, |
| "logits/chosen": -0.4987485110759735, |
| "logits/rejected": -0.4857948124408722, |
| "logps/chosen": -1300.3497314453125, |
| "logps/rejected": -1322.68017578125, |
| "loss": 0.6913399696350098, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.018459968268871307, |
| "rewards/margins": 0.00594030786305666, |
| "rewards/rejected": 0.012519659474492073, |
| "step": 153 |
| }, |
| { |
| "epoch": 0.8461538461538461, |
| "grad_norm": 1.2268966436386108, |
| "learning_rate": 4.447852760736197e-07, |
| "logits/chosen": -0.6148173213005066, |
| "logits/rejected": -0.5686666965484619, |
| "logps/chosen": -1196.4783935546875, |
| "logps/rejected": -1201.1766357421875, |
| "loss": 0.6927800178527832, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.009037494659423828, |
| "rewards/margins": 0.0022115707397460938, |
| "rewards/rejected": -0.011249064467847347, |
| "step": 154 |
| }, |
| { |
| "epoch": 0.8516483516483516, |
| "grad_norm": 1.4325149059295654, |
| "learning_rate": 4.2944785276073624e-07, |
| "logits/chosen": -0.09442466497421265, |
| "logits/rejected": 0.06447166204452515, |
| "logps/chosen": -1344.0181884765625, |
| "logps/rejected": -1390.199462890625, |
| "loss": 0.6922627091407776, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.014286688528954983, |
| "rewards/margins": -0.0005080983391962945, |
| "rewards/rejected": -0.013778590597212315, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.8571428571428571, |
| "grad_norm": 1.6210157871246338, |
| "learning_rate": 4.1411042944785275e-07, |
| "logits/chosen": -0.5195028781890869, |
| "logits/rejected": -0.581871509552002, |
| "logps/chosen": -1449.6171875, |
| "logps/rejected": -1445.11962890625, |
| "loss": 0.6920541524887085, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.007332286797463894, |
| "rewards/margins": 0.004986848682165146, |
| "rewards/rejected": -0.012319136410951614, |
| "step": 156 |
| }, |
| { |
| "epoch": 0.8626373626373627, |
| "grad_norm": 1.2568283081054688, |
| "learning_rate": 3.9877300613496937e-07, |
| "logits/chosen": -0.39413487911224365, |
| "logits/rejected": -0.3989291191101074, |
| "logps/chosen": -1560.168212890625, |
| "logps/rejected": -1546.84326171875, |
| "loss": 0.6931201219558716, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.03144290670752525, |
| "rewards/margins": -0.005113277584314346, |
| "rewards/rejected": -0.026329632848501205, |
| "step": 157 |
| }, |
| { |
| "epoch": 0.8681318681318682, |
| "grad_norm": 0.5363301634788513, |
| "learning_rate": 3.8343558282208593e-07, |
| "logits/chosen": -0.32399898767471313, |
| "logits/rejected": -0.2612525522708893, |
| "logps/chosen": -1289.2220458984375, |
| "logps/rejected": -1368.276123046875, |
| "loss": 0.6930078268051147, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.020566385239362717, |
| "rewards/margins": 0.007330484688282013, |
| "rewards/rejected": -0.02789686992764473, |
| "step": 158 |
| }, |
| { |
| "epoch": 0.8736263736263736, |
| "grad_norm": 1.1271432638168335, |
| "learning_rate": 3.680981595092025e-07, |
| "logits/chosen": -0.8580625057220459, |
| "logits/rejected": -0.8349483013153076, |
| "logps/chosen": -1564.2672119140625, |
| "logps/rejected": -1513.3536376953125, |
| "loss": 0.6922256946563721, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.021702442318201065, |
| "rewards/margins": 0.0017409119755029678, |
| "rewards/rejected": -0.023443354293704033, |
| "step": 159 |
| }, |
| { |
| "epoch": 0.8791208791208791, |
| "grad_norm": 1.249580979347229, |
| "learning_rate": 3.52760736196319e-07, |
| "logits/chosen": -0.24156060814857483, |
| "logits/rejected": -0.3282901644706726, |
| "logps/chosen": -1468.79931640625, |
| "logps/rejected": -1360.5396728515625, |
| "loss": 0.6932394504547119, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.016846224665641785, |
| "rewards/margins": -0.0005967399920336902, |
| "rewards/rejected": -0.016249485313892365, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.8846153846153846, |
| "grad_norm": 1.7817763090133667, |
| "learning_rate": 3.3742331288343563e-07, |
| "logits/chosen": -0.4720512926578522, |
| "logits/rejected": -0.47025078535079956, |
| "logps/chosen": -1514.5360107421875, |
| "logps/rejected": -1491.627197265625, |
| "loss": 0.6926375031471252, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.020506229251623154, |
| "rewards/margins": 0.008882999420166016, |
| "rewards/rejected": -0.02938922867178917, |
| "step": 161 |
| }, |
| { |
| "epoch": 0.8901098901098901, |
| "grad_norm": 0.959372878074646, |
| "learning_rate": 3.220858895705522e-07, |
| "logits/chosen": -0.35722965002059937, |
| "logits/rejected": -0.41712379455566406, |
| "logps/chosen": -1411.179931640625, |
| "logps/rejected": -1289.341796875, |
| "loss": 0.6943350434303284, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": -0.024521712213754654, |
| "rewards/margins": -0.0049790311604738235, |
| "rewards/rejected": -0.01954268291592598, |
| "step": 162 |
| }, |
| { |
| "epoch": 0.8956043956043956, |
| "grad_norm": 1.2075046300888062, |
| "learning_rate": 3.0674846625766876e-07, |
| "logits/chosen": -0.8041161298751831, |
| "logits/rejected": -0.8109002113342285, |
| "logps/chosen": -1505.929931640625, |
| "logps/rejected": -1477.200927734375, |
| "loss": 0.6926769614219666, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.028787925839424133, |
| "rewards/margins": -0.00552023109048605, |
| "rewards/rejected": -0.023267695680260658, |
| "step": 163 |
| }, |
| { |
| "epoch": 0.9010989010989011, |
| "grad_norm": 1.387386441230774, |
| "learning_rate": 2.9141104294478527e-07, |
| "logits/chosen": -0.44806283712387085, |
| "logits/rejected": -0.61948162317276, |
| "logps/chosen": -1527.060302734375, |
| "logps/rejected": -1494.800537109375, |
| "loss": 0.692619264125824, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.011695308610796928, |
| "rewards/margins": -0.0015909149078652263, |
| "rewards/rejected": -0.01010439358651638, |
| "step": 164 |
| }, |
| { |
| "epoch": 0.9065934065934066, |
| "grad_norm": 1.484518051147461, |
| "learning_rate": 2.760736196319019e-07, |
| "logits/chosen": -0.622553288936615, |
| "logits/rejected": -0.7360720634460449, |
| "logps/chosen": -1395.18798828125, |
| "logps/rejected": -1419.4432373046875, |
| "loss": 0.6941399574279785, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.008678913116455078, |
| "rewards/margins": 0.001724362256936729, |
| "rewards/rejected": -0.010403275489807129, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.9120879120879121, |
| "grad_norm": 0.7226335406303406, |
| "learning_rate": 2.607361963190184e-07, |
| "logits/chosen": -0.43059980869293213, |
| "logits/rejected": -0.5786530375480652, |
| "logps/chosen": -1415.7808837890625, |
| "logps/rejected": -1407.481201171875, |
| "loss": 0.6924831867218018, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.02000749483704567, |
| "rewards/margins": -0.002660846570506692, |
| "rewards/rejected": -0.01734665036201477, |
| "step": 166 |
| }, |
| { |
| "epoch": 0.9175824175824175, |
| "grad_norm": 1.1836820840835571, |
| "learning_rate": 2.45398773006135e-07, |
| "logits/chosen": -0.3440017104148865, |
| "logits/rejected": -0.3202272355556488, |
| "logps/chosen": -1425.4095458984375, |
| "logps/rejected": -1382.1885986328125, |
| "loss": 0.6946048140525818, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.013712071813642979, |
| "rewards/margins": -0.006773996166884899, |
| "rewards/rejected": -0.0069380756467580795, |
| "step": 167 |
| }, |
| { |
| "epoch": 0.9230769230769231, |
| "grad_norm": 2.162140369415283, |
| "learning_rate": 2.3006134969325155e-07, |
| "logits/chosen": -0.6371763944625854, |
| "logits/rejected": -0.7157647609710693, |
| "logps/chosen": -1485.0994873046875, |
| "logps/rejected": -1396.60693359375, |
| "loss": 0.6927782297134399, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.014371508732438087, |
| "rewards/margins": 0.0014920367393642664, |
| "rewards/rejected": -0.01586354523897171, |
| "step": 168 |
| }, |
| { |
| "epoch": 0.9285714285714286, |
| "grad_norm": 1.5967764854431152, |
| "learning_rate": 2.1472392638036812e-07, |
| "logits/chosen": -0.5323659777641296, |
| "logits/rejected": -0.5269442200660706, |
| "logps/chosen": -1347.4677734375, |
| "logps/rejected": -1475.41064453125, |
| "loss": 0.6943572163581848, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.0019814111292362213, |
| "rewards/margins": -0.00141085684299469, |
| "rewards/rejected": -0.0005705545772798359, |
| "step": 169 |
| }, |
| { |
| "epoch": 0.9340659340659341, |
| "grad_norm": 1.2801460027694702, |
| "learning_rate": 1.9938650306748468e-07, |
| "logits/chosen": -0.7320770621299744, |
| "logits/rejected": -0.7078484892845154, |
| "logps/chosen": -1049.31298828125, |
| "logps/rejected": -1216.61083984375, |
| "loss": 0.6933168172836304, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.009347095154225826, |
| "rewards/margins": -0.0006960107129998505, |
| "rewards/rejected": -0.008651085197925568, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.9395604395604396, |
| "grad_norm": 1.4801154136657715, |
| "learning_rate": 1.8404907975460125e-07, |
| "logits/chosen": -0.8624351620674133, |
| "logits/rejected": -0.6278290748596191, |
| "logps/chosen": -1554.202392578125, |
| "logps/rejected": -1768.3101806640625, |
| "loss": 0.6939010620117188, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.011780891567468643, |
| "rewards/margins": -0.0007463262882083654, |
| "rewards/rejected": -0.011034565046429634, |
| "step": 171 |
| }, |
| { |
| "epoch": 0.945054945054945, |
| "grad_norm": 1.2581945657730103, |
| "learning_rate": 1.6871165644171781e-07, |
| "logits/chosen": -0.8042242527008057, |
| "logits/rejected": -0.8509213328361511, |
| "logps/chosen": -1742.9061279296875, |
| "logps/rejected": -1663.30078125, |
| "loss": 0.6920545101165771, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.009515208192169666, |
| "rewards/margins": 0.006152589339762926, |
| "rewards/rejected": -0.01566779799759388, |
| "step": 172 |
| }, |
| { |
| "epoch": 0.9505494505494505, |
| "grad_norm": 1.6917072534561157, |
| "learning_rate": 1.5337423312883438e-07, |
| "logits/chosen": -0.677159309387207, |
| "logits/rejected": -0.708314836025238, |
| "logps/chosen": -1458.487548828125, |
| "logps/rejected": -1540.4312744140625, |
| "loss": 0.6939187049865723, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.008841114118695259, |
| "rewards/margins": -0.0041299439035356045, |
| "rewards/rejected": -0.004711170680820942, |
| "step": 173 |
| }, |
| { |
| "epoch": 0.9560439560439561, |
| "grad_norm": 1.562403917312622, |
| "learning_rate": 1.3803680981595094e-07, |
| "logits/chosen": -0.8315936923027039, |
| "logits/rejected": -0.8156816959381104, |
| "logps/chosen": -1405.054931640625, |
| "logps/rejected": -1378.0836181640625, |
| "loss": 0.6930129528045654, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.008151168935000896, |
| "rewards/margins": 0.0004372410476207733, |
| "rewards/rejected": -0.008588409051299095, |
| "step": 174 |
| }, |
| { |
| "epoch": 0.9615384615384616, |
| "grad_norm": 1.8198566436767578, |
| "learning_rate": 1.226993865030675e-07, |
| "logits/chosen": -0.35468000173568726, |
| "logits/rejected": -0.46720781922340393, |
| "logps/chosen": -1518.917236328125, |
| "logps/rejected": -1435.27783203125, |
| "loss": 0.6903528571128845, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.005114269442856312, |
| "rewards/margins": 0.011224641464650631, |
| "rewards/rejected": -0.006110372021794319, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.967032967032967, |
| "grad_norm": 2.1096837520599365, |
| "learning_rate": 1.0736196319018406e-07, |
| "logits/chosen": -0.6721988320350647, |
| "logits/rejected": -0.6404861211776733, |
| "logps/chosen": -1544.8468017578125, |
| "logps/rejected": -1587.005615234375, |
| "loss": 0.6921041011810303, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.010166358202695847, |
| "rewards/margins": 0.003511047223582864, |
| "rewards/rejected": -0.013677406124770641, |
| "step": 176 |
| }, |
| { |
| "epoch": 0.9725274725274725, |
| "grad_norm": 1.5950981378555298, |
| "learning_rate": 9.202453987730062e-08, |
| "logits/chosen": -0.867381751537323, |
| "logits/rejected": -0.753036379814148, |
| "logps/chosen": -1559.811767578125, |
| "logps/rejected": -1612.46826171875, |
| "loss": 0.6940016150474548, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.014099950902163982, |
| "rewards/margins": -0.0024051289074122906, |
| "rewards/rejected": -0.011694822460412979, |
| "step": 177 |
| }, |
| { |
| "epoch": 0.978021978021978, |
| "grad_norm": 1.7663509845733643, |
| "learning_rate": 7.668711656441719e-08, |
| "logits/chosen": -0.49028313159942627, |
| "logits/rejected": -0.6348991990089417, |
| "logps/chosen": -1656.97216796875, |
| "logps/rejected": -1574.3087158203125, |
| "loss": 0.6930438876152039, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.0007067109690979123, |
| "rewards/margins": 0.002308547031134367, |
| "rewards/rejected": -0.0030152583494782448, |
| "step": 178 |
| }, |
| { |
| "epoch": 0.9835164835164835, |
| "grad_norm": 2.180283308029175, |
| "learning_rate": 6.134969325153375e-08, |
| "logits/chosen": -0.3077111840248108, |
| "logits/rejected": -0.3495293855667114, |
| "logps/chosen": -1459.8404541015625, |
| "logps/rejected": -1447.5089111328125, |
| "loss": 0.6924480199813843, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.008424444124102592, |
| "rewards/margins": 0.0035952467005699873, |
| "rewards/rejected": -0.012019690126180649, |
| "step": 179 |
| }, |
| { |
| "epoch": 0.989010989010989, |
| "grad_norm": 1.005254864692688, |
| "learning_rate": 4.601226993865031e-08, |
| "logits/chosen": -0.6914213299751282, |
| "logits/rejected": -0.6896047592163086, |
| "logps/chosen": -1281.80419921875, |
| "logps/rejected": -1390.142578125, |
| "loss": 0.692180871963501, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.00413126964122057, |
| "rewards/margins": 0.0011504649883136153, |
| "rewards/rejected": -0.0052817342802882195, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.9945054945054945, |
| "grad_norm": 1.4081807136535645, |
| "learning_rate": 3.067484662576688e-08, |
| "logits/chosen": -0.8277162313461304, |
| "logits/rejected": -0.8068577647209167, |
| "logps/chosen": -1470.6419677734375, |
| "logps/rejected": -1466.707763671875, |
| "loss": 0.6934786438941956, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.0031227758154273033, |
| "rewards/margins": -0.004466504789888859, |
| "rewards/rejected": 0.0013437292072921991, |
| "step": 181 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 1.1653679609298706, |
| "learning_rate": 1.533742331288344e-08, |
| "logits/chosen": -0.3629406690597534, |
| "logits/rejected": -0.33546456694602966, |
| "logps/chosen": -1682.532470703125, |
| "logps/rejected": -1659.32470703125, |
| "loss": 0.6933432221412659, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.00286815594881773, |
| "rewards/margins": 0.000943202874623239, |
| "rewards/rejected": 0.0019249536562711, |
| "step": 182 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 182, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
