| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 19.37219730941704, | |
| "eval_steps": 500, | |
| "global_step": 540, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.35874439461883406, | |
| "grad_norm": 0.5037005543708801, | |
| "learning_rate": 9.259259259259259e-07, | |
| "logits/chosen": 1.851180076599121, | |
| "logits/rejected": 1.8187497854232788, | |
| "logps/chosen": -71.38922119140625, | |
| "logps/rejected": -77.1716537475586, | |
| "loss": 0.6947, | |
| "rewards/accuracies": 0.36250001192092896, | |
| "rewards/chosen": -0.0033944465685635805, | |
| "rewards/margins": -0.004582419525831938, | |
| "rewards/rejected": 0.0011879729572683573, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.7174887892376681, | |
| "grad_norm": 0.4899679124355316, | |
| "learning_rate": 1.8518518518518519e-06, | |
| "logits/chosen": 1.804616928100586, | |
| "logits/rejected": 1.8522075414657593, | |
| "logps/chosen": -68.71460723876953, | |
| "logps/rejected": -92.10017395019531, | |
| "loss": 0.6935, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": 0.0006334683275781572, | |
| "rewards/margins": 0.002181120216846466, | |
| "rewards/rejected": -0.0015476513653993607, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 1.0762331838565022, | |
| "grad_norm": 0.4610757529735565, | |
| "learning_rate": 2.7777777777777783e-06, | |
| "logits/chosen": 1.7368440628051758, | |
| "logits/rejected": 1.747293472290039, | |
| "logps/chosen": -86.10456848144531, | |
| "logps/rejected": -94.18408203125, | |
| "loss": 0.6923, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.004750942811369896, | |
| "rewards/margins": -3.7298166716936976e-05, | |
| "rewards/rejected": -0.0047136456705629826, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 1.4349775784753362, | |
| "grad_norm": 0.5546987056732178, | |
| "learning_rate": 3.7037037037037037e-06, | |
| "logits/chosen": 1.760947585105896, | |
| "logits/rejected": 1.7774131298065186, | |
| "logps/chosen": -72.0299301147461, | |
| "logps/rejected": -75.53373718261719, | |
| "loss": 0.6938, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": 0.004539580084383488, | |
| "rewards/margins": 0.004153046756982803, | |
| "rewards/rejected": 0.0003865335020236671, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 1.7937219730941703, | |
| "grad_norm": 0.5019991397857666, | |
| "learning_rate": 4.62962962962963e-06, | |
| "logits/chosen": 1.6594927310943604, | |
| "logits/rejected": 1.7026996612548828, | |
| "logps/chosen": -74.72074890136719, | |
| "logps/rejected": -80.34657287597656, | |
| "loss": 0.694, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": 0.0014741375343874097, | |
| "rewards/margins": 0.00692129647359252, | |
| "rewards/rejected": -0.00544715765863657, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 2.1524663677130045, | |
| "grad_norm": 0.5497386455535889, | |
| "learning_rate": 4.998119881260576e-06, | |
| "logits/chosen": 1.845393180847168, | |
| "logits/rejected": 1.8288724422454834, | |
| "logps/chosen": -85.8925552368164, | |
| "logps/rejected": -79.84625244140625, | |
| "loss": 0.6939, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": 0.0007343249162659049, | |
| "rewards/margins": 0.0027797420043498278, | |
| "rewards/rejected": -0.0020454167388379574, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 2.5112107623318387, | |
| "grad_norm": 0.53052818775177, | |
| "learning_rate": 4.9866405060165044e-06, | |
| "logits/chosen": 1.7219512462615967, | |
| "logits/rejected": 1.739393949508667, | |
| "logps/chosen": -82.69403076171875, | |
| "logps/rejected": -85.24137878417969, | |
| "loss": 0.6912, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": 0.004084458574652672, | |
| "rewards/margins": 8.91583040356636e-05, | |
| "rewards/rejected": 0.003995300270617008, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 2.8699551569506725, | |
| "grad_norm": 0.5697340965270996, | |
| "learning_rate": 4.964774158361991e-06, | |
| "logits/chosen": 1.688643217086792, | |
| "logits/rejected": 1.7375280857086182, | |
| "logps/chosen": -77.02912902832031, | |
| "logps/rejected": -84.49137115478516, | |
| "loss": 0.6923, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.0026642463635653257, | |
| "rewards/margins": -0.0012511021923273802, | |
| "rewards/rejected": -0.0014131448697298765, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 3.2286995515695067, | |
| "grad_norm": 0.6337418556213379, | |
| "learning_rate": 4.93261217644956e-06, | |
| "logits/chosen": 1.7215973138809204, | |
| "logits/rejected": 1.6856582164764404, | |
| "logps/chosen": -81.94705963134766, | |
| "logps/rejected": -79.82647705078125, | |
| "loss": 0.6916, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": 0.004837460815906525, | |
| "rewards/margins": 0.0014790964778512716, | |
| "rewards/rejected": 0.003358363639563322, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 3.587443946188341, | |
| "grad_norm": 0.6216289401054382, | |
| "learning_rate": 4.8902889044347e-06, | |
| "logits/chosen": 1.8092563152313232, | |
| "logits/rejected": 1.7624406814575195, | |
| "logps/chosen": -91.27610778808594, | |
| "logps/rejected": -76.58167266845703, | |
| "loss": 0.6879, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": 0.004790940321981907, | |
| "rewards/margins": 0.018742190673947334, | |
| "rewards/rejected": -0.013951249420642853, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 3.9461883408071747, | |
| "grad_norm": 0.6686798930168152, | |
| "learning_rate": 4.837981131305475e-06, | |
| "logits/chosen": 1.8108171224594116, | |
| "logits/rejected": 1.8439037799835205, | |
| "logps/chosen": -68.89733123779297, | |
| "logps/rejected": -90.71812438964844, | |
| "loss": 0.686, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": 0.0019698950927704573, | |
| "rewards/margins": 0.011438245885074139, | |
| "rewards/rejected": -0.009468351490795612, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 4.304932735426009, | |
| "grad_norm": 0.690719723701477, | |
| "learning_rate": 4.775907352415367e-06, | |
| "logits/chosen": 1.7892429828643799, | |
| "logits/rejected": 1.7746537923812866, | |
| "logps/chosen": -76.5858383178711, | |
| "logps/rejected": -70.5009765625, | |
| "loss": 0.6841, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.00402367627248168, | |
| "rewards/margins": 0.02823176048696041, | |
| "rewards/rejected": -0.024208081886172295, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 4.663677130044843, | |
| "grad_norm": 0.6445876955986023, | |
| "learning_rate": 4.70432685680402e-06, | |
| "logits/chosen": 1.7731869220733643, | |
| "logits/rejected": 1.70940363407135, | |
| "logps/chosen": -80.2589111328125, | |
| "logps/rejected": -69.99195861816406, | |
| "loss": 0.6807, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": 0.0028332143556326628, | |
| "rewards/margins": 0.027303647249937057, | |
| "rewards/rejected": -0.024470435455441475, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 5.022421524663677, | |
| "grad_norm": 0.7485262751579285, | |
| "learning_rate": 4.623538644118244e-06, | |
| "logits/chosen": 1.707404375076294, | |
| "logits/rejected": 1.7899534702301025, | |
| "logps/chosen": -85.12259674072266, | |
| "logps/rejected": -96.5855712890625, | |
| "loss": 0.6717, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": 0.0005208252114243805, | |
| "rewards/margins": 0.046183306723833084, | |
| "rewards/rejected": -0.04566247761249542, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 5.381165919282511, | |
| "grad_norm": 0.7476359009742737, | |
| "learning_rate": 4.533880175657419e-06, | |
| "logits/chosen": 1.8374496698379517, | |
| "logits/rejected": 1.8710906505584717, | |
| "logps/chosen": -91.67071533203125, | |
| "logps/rejected": -89.70133209228516, | |
| "loss": 0.6683, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.004734884016215801, | |
| "rewards/margins": 0.06081797555088997, | |
| "rewards/rejected": -0.05608309060335159, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 5.739910313901345, | |
| "grad_norm": 0.8249401450157166, | |
| "learning_rate": 4.435725964760331e-06, | |
| "logits/chosen": 1.7922887802124023, | |
| "logits/rejected": 1.766998291015625, | |
| "logps/chosen": -90.3626708984375, | |
| "logps/rejected": -78.34722900390625, | |
| "loss": 0.6615, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.0012385320151224732, | |
| "rewards/margins": 0.06155504658818245, | |
| "rewards/rejected": -0.06279357522726059, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 6.098654708520179, | |
| "grad_norm": 0.8427699208259583, | |
| "learning_rate": 4.329486012421531e-06, | |
| "logits/chosen": 1.8337100744247437, | |
| "logits/rejected": 1.8013321161270142, | |
| "logps/chosen": -88.58504486083984, | |
| "logps/rejected": -82.3107681274414, | |
| "loss": 0.6575, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.028140008449554443, | |
| "rewards/margins": 0.06505288183689117, | |
| "rewards/rejected": -0.09319288283586502, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 6.457399103139013, | |
| "grad_norm": 0.8900613784790039, | |
| "learning_rate": 4.215604094671835e-06, | |
| "logits/chosen": 1.8928254842758179, | |
| "logits/rejected": 1.8716427087783813, | |
| "logps/chosen": -73.64405822753906, | |
| "logps/rejected": -82.36759948730469, | |
| "loss": 0.6486, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.008158551529049873, | |
| "rewards/margins": 0.10160098224878311, | |
| "rewards/rejected": -0.10975953191518784, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 6.816143497757848, | |
| "grad_norm": 0.8851564526557922, | |
| "learning_rate": 4.094555908876765e-06, | |
| "logits/chosen": 1.8064581155776978, | |
| "logits/rejected": 1.7510322332382202, | |
| "logps/chosen": -81.51691436767578, | |
| "logps/rejected": -80.2887954711914, | |
| "loss": 0.6397, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.010233854874968529, | |
| "rewards/margins": 0.15614357590675354, | |
| "rewards/rejected": -0.166377454996109, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 7.174887892376682, | |
| "grad_norm": 0.9349417090415955, | |
| "learning_rate": 3.966847086696045e-06, | |
| "logits/chosen": 1.869329810142517, | |
| "logits/rejected": 1.8965225219726562, | |
| "logps/chosen": -83.17039489746094, | |
| "logps/rejected": -83.12516021728516, | |
| "loss": 0.6421, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.06402869522571564, | |
| "rewards/margins": 0.11340844631195068, | |
| "rewards/rejected": -0.17743715643882751, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 7.533632286995516, | |
| "grad_norm": 0.9500117301940918, | |
| "learning_rate": 3.833011082004229e-06, | |
| "logits/chosen": 1.74019455909729, | |
| "logits/rejected": 1.8087282180786133, | |
| "logps/chosen": -69.24908447265625, | |
| "logps/rejected": -78.79796600341797, | |
| "loss": 0.6373, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.09633999317884445, | |
| "rewards/margins": 0.10314120352268219, | |
| "rewards/rejected": -0.19948120415210724, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 7.8923766816143495, | |
| "grad_norm": 0.9928329586982727, | |
| "learning_rate": 3.693606942594873e-06, | |
| "logits/chosen": 1.7647511959075928, | |
| "logits/rejected": 1.688582420349121, | |
| "logps/chosen": -85.75634002685547, | |
| "logps/rejected": -84.30413818359375, | |
| "loss": 0.6193, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.08094713091850281, | |
| "rewards/margins": 0.20030252635478973, | |
| "rewards/rejected": -0.28124964237213135, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 8.251121076233185, | |
| "grad_norm": 0.9496206045150757, | |
| "learning_rate": 3.549216974976073e-06, | |
| "logits/chosen": 1.6640609502792358, | |
| "logits/rejected": 1.7074018716812134, | |
| "logps/chosen": -78.85441589355469, | |
| "logps/rejected": -101.50064849853516, | |
| "loss": 0.6187, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.10971230268478394, | |
| "rewards/margins": 0.16489508748054504, | |
| "rewards/rejected": -0.2746073603630066, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 8.609865470852018, | |
| "grad_norm": 0.8651049137115479, | |
| "learning_rate": 3.400444312011776e-06, | |
| "logits/chosen": 1.8204492330551147, | |
| "logits/rejected": 1.8322817087173462, | |
| "logps/chosen": -80.65580749511719, | |
| "logps/rejected": -85.63023376464844, | |
| "loss": 0.6167, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.09459066390991211, | |
| "rewards/margins": 0.2738259434700012, | |
| "rewards/rejected": -0.3684166371822357, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 8.968609865470851, | |
| "grad_norm": 1.070734977722168, | |
| "learning_rate": 3.2479103935691047e-06, | |
| "logits/chosen": 1.6801395416259766, | |
| "logits/rejected": 1.7755155563354492, | |
| "logps/chosen": -79.02301788330078, | |
| "logps/rejected": -99.99031066894531, | |
| "loss": 0.5929, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.14009204506874084, | |
| "rewards/margins": 0.26308172941207886, | |
| "rewards/rejected": -0.4031738340854645, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 9.327354260089686, | |
| "grad_norm": 0.9541939496994019, | |
| "learning_rate": 3.092252370695298e-06, | |
| "logits/chosen": 1.662615418434143, | |
| "logits/rejected": 1.7456607818603516, | |
| "logps/chosen": -75.20834350585938, | |
| "logps/rejected": -113.8222885131836, | |
| "loss": 0.5945, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.12744423747062683, | |
| "rewards/margins": 0.22806742787361145, | |
| "rewards/rejected": -0.3555116653442383, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 9.68609865470852, | |
| "grad_norm": 1.0955382585525513, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "logits/chosen": 1.7580053806304932, | |
| "logits/rejected": 1.7044979333877563, | |
| "logps/chosen": -74.88780212402344, | |
| "logps/rejected": -77.73652648925781, | |
| "loss": 0.5913, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.20694907009601593, | |
| "rewards/margins": 0.28244781494140625, | |
| "rewards/rejected": -0.489396870136261, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 10.044843049327355, | |
| "grad_norm": 1.0972706079483032, | |
| "learning_rate": 2.7741751485313295e-06, | |
| "logits/chosen": 1.6702816486358643, | |
| "logits/rejected": 1.861555814743042, | |
| "logps/chosen": -76.36530303955078, | |
| "logps/rejected": -107.99542236328125, | |
| "loss": 0.596, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.19078974425792694, | |
| "rewards/margins": 0.31978002190589905, | |
| "rewards/rejected": -0.5105697512626648, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 10.403587443946188, | |
| "grad_norm": 1.180747628211975, | |
| "learning_rate": 2.6130845929767662e-06, | |
| "logits/chosen": 1.7563337087631226, | |
| "logits/rejected": 1.8194200992584229, | |
| "logps/chosen": -74.66185760498047, | |
| "logps/rejected": -93.19525146484375, | |
| "loss": 0.5767, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.15936018526554108, | |
| "rewards/margins": 0.2494836151599884, | |
| "rewards/rejected": -0.4088438153266907, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 10.762331838565022, | |
| "grad_norm": 1.1262431144714355, | |
| "learning_rate": 2.4515216705704396e-06, | |
| "logits/chosen": 1.6524626016616821, | |
| "logits/rejected": 1.6555395126342773, | |
| "logps/chosen": -76.6367416381836, | |
| "logps/rejected": -79.79283905029297, | |
| "loss": 0.5847, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.17207466065883636, | |
| "rewards/margins": 0.3527255058288574, | |
| "rewards/rejected": -0.524800181388855, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 11.121076233183857, | |
| "grad_norm": 1.0337167978286743, | |
| "learning_rate": 2.290161247507733e-06, | |
| "logits/chosen": 1.5576703548431396, | |
| "logits/rejected": 1.6868317127227783, | |
| "logps/chosen": -82.24876403808594, | |
| "logps/rejected": -97.26077270507812, | |
| "loss": 0.5878, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.2861223816871643, | |
| "rewards/margins": 0.3279086947441101, | |
| "rewards/rejected": -0.6140309572219849, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 11.47982062780269, | |
| "grad_norm": 1.0992393493652344, | |
| "learning_rate": 2.129677344121879e-06, | |
| "logits/chosen": 1.767163872718811, | |
| "logits/rejected": 1.7726752758026123, | |
| "logps/chosen": -85.95258331298828, | |
| "logps/rejected": -95.29878234863281, | |
| "loss": 0.5674, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.25968822836875916, | |
| "rewards/margins": 0.37743932008743286, | |
| "rewards/rejected": -0.6371275186538696, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 11.838565022421525, | |
| "grad_norm": 1.082541823387146, | |
| "learning_rate": 1.970740319426474e-06, | |
| "logits/chosen": 1.6448657512664795, | |
| "logits/rejected": 1.6428959369659424, | |
| "logps/chosen": -103.04365539550781, | |
| "logps/rejected": -82.7887954711914, | |
| "loss": 0.5632, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.20720317959785461, | |
| "rewards/margins": 0.2542288899421692, | |
| "rewards/rejected": -0.4614320695400238, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 12.197309417040358, | |
| "grad_norm": 1.061893105506897, | |
| "learning_rate": 1.8140140709517467e-06, | |
| "logits/chosen": 1.7359678745269775, | |
| "logits/rejected": 1.7358520030975342, | |
| "logps/chosen": -85.93545532226562, | |
| "logps/rejected": -89.08082580566406, | |
| "loss": 0.5727, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.2792442739009857, | |
| "rewards/margins": 0.22401955723762512, | |
| "rewards/rejected": -0.5032638311386108, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 12.556053811659194, | |
| "grad_norm": 1.107885479927063, | |
| "learning_rate": 1.6601532615711452e-06, | |
| "logits/chosen": 1.585384726524353, | |
| "logits/rejected": 1.636023759841919, | |
| "logps/chosen": -93.92401123046875, | |
| "logps/rejected": -90.39390563964844, | |
| "loss": 0.5611, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.1979474574327469, | |
| "rewards/margins": 0.33891761302948, | |
| "rewards/rejected": -0.5368650555610657, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 12.914798206278027, | |
| "grad_norm": 1.0166430473327637, | |
| "learning_rate": 1.509800584902108e-06, | |
| "logits/chosen": 1.6831719875335693, | |
| "logits/rejected": 1.624122977256775, | |
| "logps/chosen": -93.83638763427734, | |
| "logps/rejected": -85.26428985595703, | |
| "loss": 0.5576, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.20380835235118866, | |
| "rewards/margins": 0.33769887685775757, | |
| "rewards/rejected": -0.5415073037147522, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 13.27354260089686, | |
| "grad_norm": 1.039067268371582, | |
| "learning_rate": 1.3635840807037487e-06, | |
| "logits/chosen": 1.6376430988311768, | |
| "logits/rejected": 1.747945785522461, | |
| "logps/chosen": -66.59056091308594, | |
| "logps/rejected": -86.75489044189453, | |
| "loss": 0.5502, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.27494093775749207, | |
| "rewards/margins": 0.4173588156700134, | |
| "rewards/rejected": -0.6922997832298279, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 13.632286995515695, | |
| "grad_norm": 1.0018366575241089, | |
| "learning_rate": 1.2221145114853172e-06, | |
| "logits/chosen": 1.6222606897354126, | |
| "logits/rejected": 1.6619726419448853, | |
| "logps/chosen": -87.45683288574219, | |
| "logps/rejected": -83.59837341308594, | |
| "loss": 0.5595, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.17884480953216553, | |
| "rewards/margins": 0.2639870047569275, | |
| "rewards/rejected": -0.4428318440914154, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 13.991031390134529, | |
| "grad_norm": 1.0804483890533447, | |
| "learning_rate": 1.085982811283654e-06, | |
| "logits/chosen": 1.63021981716156, | |
| "logits/rejected": 1.6384109258651733, | |
| "logps/chosen": -91.65565490722656, | |
| "logps/rejected": -92.39521789550781, | |
| "loss": 0.5517, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.2314184606075287, | |
| "rewards/margins": 0.3744989335536957, | |
| "rewards/rejected": -0.6059174537658691, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 14.349775784753364, | |
| "grad_norm": 1.126531958580017, | |
| "learning_rate": 9.557576172663577e-07, | |
| "logits/chosen": 1.6363227367401123, | |
| "logits/rejected": 1.6808557510375977, | |
| "logps/chosen": -84.4040298461914, | |
| "logps/rejected": -89.61891174316406, | |
| "loss": 0.5479, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.19463828206062317, | |
| "rewards/margins": 0.4325459599494934, | |
| "rewards/rejected": -0.627184271812439, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 14.708520179372197, | |
| "grad_norm": 1.229477882385254, | |
| "learning_rate": 8.319828944714508e-07, | |
| "logits/chosen": 1.7398450374603271, | |
| "logits/rejected": 1.7130982875823975, | |
| "logps/chosen": -79.57427978515625, | |
| "logps/rejected": -91.69161224365234, | |
| "loss": 0.5551, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.23391035199165344, | |
| "rewards/margins": 0.42068368196487427, | |
| "rewards/rejected": -0.6545940637588501, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 15.067264573991032, | |
| "grad_norm": 1.0864663124084473, | |
| "learning_rate": 7.151756636052529e-07, | |
| "logits/chosen": 1.6776914596557617, | |
| "logits/rejected": 1.6496366262435913, | |
| "logps/chosen": -80.69108581542969, | |
| "logps/rejected": -81.94221496582031, | |
| "loss": 0.5469, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.23782162368297577, | |
| "rewards/margins": 0.47778135538101196, | |
| "rewards/rejected": -0.7156028747558594, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 15.426008968609866, | |
| "grad_norm": 1.0339457988739014, | |
| "learning_rate": 6.058238413897052e-07, | |
| "logits/chosen": 1.7183252573013306, | |
| "logits/rejected": 1.7133119106292725, | |
| "logps/chosen": -100.38890075683594, | |
| "logps/rejected": -101.6832046508789, | |
| "loss": 0.551, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.34573277831077576, | |
| "rewards/margins": 0.3820987939834595, | |
| "rewards/rejected": -0.7278315424919128, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 15.784753363228699, | |
| "grad_norm": 1.2006279230117798, | |
| "learning_rate": 5.043842024802675e-07, | |
| "logits/chosen": 1.7278945446014404, | |
| "logits/rejected": 1.7016990184783936, | |
| "logps/chosen": -89.6147232055664, | |
| "logps/rejected": -92.5606918334961, | |
| "loss": 0.5419, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.24864788353443146, | |
| "rewards/margins": 0.3481907546520233, | |
| "rewards/rejected": -0.5968385934829712, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 16.143497757847534, | |
| "grad_norm": 1.1825172901153564, | |
| "learning_rate": 4.1128047146765936e-07, | |
| "logits/chosen": 1.5679899454116821, | |
| "logits/rejected": 1.5998270511627197, | |
| "logps/chosen": -81.74020385742188, | |
| "logps/rejected": -86.86679077148438, | |
| "loss": 0.5492, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.2699211537837982, | |
| "rewards/margins": 0.4290841221809387, | |
| "rewards/rejected": -0.6990053057670593, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 16.50224215246637, | |
| "grad_norm": 1.1816792488098145, | |
| "learning_rate": 3.269015529333805e-07, | |
| "logits/chosen": 1.621124029159546, | |
| "logits/rejected": 1.749467134475708, | |
| "logps/chosen": -73.30225372314453, | |
| "logps/rejected": -92.98735046386719, | |
| "loss": 0.5364, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.29340410232543945, | |
| "rewards/margins": 0.39086267352104187, | |
| "rewards/rejected": -0.6842667460441589, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 16.8609865470852, | |
| "grad_norm": 1.107458472251892, | |
| "learning_rate": 2.515999069522676e-07, | |
| "logits/chosen": 1.7169005870819092, | |
| "logits/rejected": 1.6980698108673096, | |
| "logps/chosen": -102.9745864868164, | |
| "logps/rejected": -77.38671875, | |
| "loss": 0.5485, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.22492234408855438, | |
| "rewards/margins": 0.4119078516960144, | |
| "rewards/rejected": -0.63683021068573, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 17.219730941704036, | |
| "grad_norm": 1.1910994052886963, | |
| "learning_rate": 1.8569007682777417e-07, | |
| "logits/chosen": 1.5466203689575195, | |
| "logits/rejected": 1.5265737771987915, | |
| "logps/chosen": -81.35516357421875, | |
| "logps/rejected": -79.12394714355469, | |
| "loss": 0.5546, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.2879922688007355, | |
| "rewards/margins": 0.2313261479139328, | |
| "rewards/rejected": -0.5193184614181519, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 17.57847533632287, | |
| "grad_norm": 1.156771183013916, | |
| "learning_rate": 1.2944737520980883e-07, | |
| "logits/chosen": 1.7166872024536133, | |
| "logits/rejected": 1.8277479410171509, | |
| "logps/chosen": -89.5985336303711, | |
| "logps/rejected": -97.35225677490234, | |
| "loss": 0.5312, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.27265459299087524, | |
| "rewards/margins": 0.3236234784126282, | |
| "rewards/rejected": -0.5962780714035034, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 17.937219730941703, | |
| "grad_norm": 1.5837684869766235, | |
| "learning_rate": 8.310673408334496e-08, | |
| "logits/chosen": 1.693872094154358, | |
| "logits/rejected": 1.6753771305084229, | |
| "logps/chosen": -84.9941177368164, | |
| "logps/rejected": -78.13436126708984, | |
| "loss": 0.528, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.1711592972278595, | |
| "rewards/margins": 0.4582854211330414, | |
| "rewards/rejected": -0.6294447183609009, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 18.295964125560538, | |
| "grad_norm": 1.1619194746017456, | |
| "learning_rate": 4.6861723431538273e-08, | |
| "logits/chosen": 1.6101446151733398, | |
| "logits/rejected": 1.793236494064331, | |
| "logps/chosen": -80.81233215332031, | |
| "logps/rejected": -107.17619323730469, | |
| "loss": 0.5495, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.2168944776058197, | |
| "rewards/margins": 0.5115703344345093, | |
| "rewards/rejected": -0.7284647822380066, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 18.654708520179373, | |
| "grad_norm": 1.2213138341903687, | |
| "learning_rate": 2.0863742672497244e-08, | |
| "logits/chosen": 1.6612660884857178, | |
| "logits/rejected": 1.6667835712432861, | |
| "logps/chosen": -89.79667663574219, | |
| "logps/rejected": -92.6375732421875, | |
| "loss": 0.5483, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.21362046897411346, | |
| "rewards/margins": 0.5911002159118652, | |
| "rewards/rejected": -0.8047205805778503, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 19.013452914798208, | |
| "grad_norm": 0.9736039042472839, | |
| "learning_rate": 5.221388247169945e-09, | |
| "logits/chosen": 1.5504584312438965, | |
| "logits/rejected": 1.4831935167312622, | |
| "logps/chosen": -100.22550201416016, | |
| "logps/rejected": -77.9913558959961, | |
| "loss": 0.5351, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -0.3059537410736084, | |
| "rewards/margins": 0.30310431122779846, | |
| "rewards/rejected": -0.6090580224990845, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 19.37219730941704, | |
| "grad_norm": 1.2035579681396484, | |
| "learning_rate": 0.0, | |
| "logits/chosen": 1.761300802230835, | |
| "logits/rejected": 1.7381843328475952, | |
| "logps/chosen": -83.55279541015625, | |
| "logps/rejected": -88.41242980957031, | |
| "loss": 0.5356, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.21133939921855927, | |
| "rewards/margins": 0.5088428258895874, | |
| "rewards/rejected": -0.7201822400093079, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 19.37219730941704, | |
| "step": 540, | |
| "total_flos": 1.8378090112404685e+18, | |
| "train_loss": 0.6078463554382324, | |
| "train_runtime": 3976.7885, | |
| "train_samples_per_second": 8.972, | |
| "train_steps_per_second": 0.136 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 540, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 20, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 1.8378090112404685e+18, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |