{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 40.0, "eval_steps": 500, "global_step": 1600, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.25, "grad_norm": 26.865100860595703, "learning_rate": 6.25e-07, "logits/chosen": -3.0250370502471924, "logits/rejected": -3.0283076763153076, "logps/chosen": -291.21490478515625, "logps/rejected": -308.63470458984375, "loss": 0.6929, "rewards/accuracies": 0.5, "rewards/chosen": 0.0004953003372065723, "rewards/margins": 0.00046472548274323344, "rewards/rejected": 3.057479625567794e-05, "step": 10 }, { "epoch": 0.5, "grad_norm": 30.28512191772461, "learning_rate": 1.25e-06, "logits/chosen": -3.051795482635498, "logits/rejected": -3.0552544593811035, "logps/chosen": -275.4926452636719, "logps/rejected": -295.6332702636719, "loss": 0.6913, "rewards/accuracies": 0.824999988079071, "rewards/chosen": 0.004581451416015625, "rewards/margins": 0.003644981188699603, "rewards/rejected": 0.0009364699944853783, "step": 20 }, { "epoch": 0.75, "grad_norm": 22.003541946411133, "learning_rate": 1.8750000000000003e-06, "logits/chosen": -3.0227792263031006, "logits/rejected": -3.037844657897949, "logps/chosen": -288.5640563964844, "logps/rejected": -305.0924072265625, "loss": 0.6858, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 0.01881382055580616, "rewards/margins": 0.014787979423999786, "rewards/rejected": 0.004025841131806374, "step": 30 }, { "epoch": 1.0, "grad_norm": 25.61740493774414, "learning_rate": 2.5e-06, "logits/chosen": -3.075989246368408, "logits/rejected": -3.09385347366333, "logps/chosen": -326.1593322753906, "logps/rejected": -336.47247314453125, "loss": 0.6792, "rewards/accuracies": 1.0, "rewards/chosen": 0.03061000630259514, "rewards/margins": 0.02814296819269657, "rewards/rejected": 0.0024670413695275784, "step": 40 }, { "epoch": 1.25, "grad_norm": 18.726919174194336, "learning_rate": 3.125e-06, "logits/chosen": -3.045722246170044, "logits/rejected": -3.021096706390381, "logps/chosen": -300.9041748046875, "logps/rejected": -342.43634033203125, "loss": 0.6553, "rewards/accuracies": 1.0, "rewards/chosen": 0.06766779720783234, "rewards/margins": 0.0775221437215805, "rewards/rejected": -0.009854355826973915, "step": 50 }, { "epoch": 1.5, "grad_norm": 23.078794479370117, "learning_rate": 3.7500000000000005e-06, "logits/chosen": -2.990593433380127, "logits/rejected": -3.0149338245391846, "logps/chosen": -277.4351501464844, "logps/rejected": -289.46417236328125, "loss": 0.6346, "rewards/accuracies": 1.0, "rewards/chosen": 0.10617438703775406, "rewards/margins": 0.12149196863174438, "rewards/rejected": -0.015317574143409729, "step": 60 }, { "epoch": 1.75, "grad_norm": 27.955883026123047, "learning_rate": 4.3750000000000005e-06, "logits/chosen": -3.044400691986084, "logits/rejected": -3.057063102722168, "logps/chosen": -293.9222717285156, "logps/rejected": -292.18939208984375, "loss": 0.6276, "rewards/accuracies": 1.0, "rewards/chosen": 0.13079899549484253, "rewards/margins": 0.13752365112304688, "rewards/rejected": -0.006724664010107517, "step": 70 }, { "epoch": 2.0, "grad_norm": 19.496564865112305, "learning_rate": 5e-06, "logits/chosen": -3.0755558013916016, "logits/rejected": -3.110250473022461, "logps/chosen": -304.8177795410156, "logps/rejected": -323.0760803222656, "loss": 0.596, "rewards/accuracies": 1.0, "rewards/chosen": 0.19854378700256348, "rewards/margins": 0.2069414108991623, "rewards/rejected": -0.008397617377340794, "step": 80 }, { "epoch": 2.25, "grad_norm": 18.082189559936523, "learning_rate": 5.625e-06, "logits/chosen": -2.9778950214385986, "logits/rejected": -2.996166944503784, "logps/chosen": -300.85528564453125, "logps/rejected": -329.5995178222656, "loss": 0.5514, "rewards/accuracies": 1.0, "rewards/chosen": 0.2628921568393707, "rewards/margins": 0.31406083703041077, "rewards/rejected": -0.05116865038871765, "step": 90 }, { "epoch": 2.5, "grad_norm": 21.41973114013672, "learning_rate": 6.25e-06, "logits/chosen": -3.006256580352783, "logits/rejected": -3.015749454498291, "logps/chosen": -285.61602783203125, "logps/rejected": -294.6287536621094, "loss": 0.5194, "rewards/accuracies": 1.0, "rewards/chosen": 0.33982062339782715, "rewards/margins": 0.3938543200492859, "rewards/rejected": -0.05403373762965202, "step": 100 }, { "epoch": 2.75, "grad_norm": 19.096920013427734, "learning_rate": 6.875e-06, "logits/chosen": -3.0787596702575684, "logits/rejected": -3.0709357261657715, "logps/chosen": -284.648193359375, "logps/rejected": -322.32989501953125, "loss": 0.4882, "rewards/accuracies": 1.0, "rewards/chosen": 0.4202825129032135, "rewards/margins": 0.4840998649597168, "rewards/rejected": -0.06381740421056747, "step": 110 }, { "epoch": 3.0, "grad_norm": 20.947290420532227, "learning_rate": 7.500000000000001e-06, "logits/chosen": -3.1311302185058594, "logits/rejected": -3.158562183380127, "logps/chosen": -298.94390869140625, "logps/rejected": -304.8675842285156, "loss": 0.4482, "rewards/accuracies": 1.0, "rewards/chosen": 0.541038990020752, "rewards/margins": 0.5876237750053406, "rewards/rejected": -0.04658476263284683, "step": 120 }, { "epoch": 3.25, "grad_norm": 15.755688667297363, "learning_rate": 8.125000000000001e-06, "logits/chosen": -3.037346124649048, "logits/rejected": -3.062375783920288, "logps/chosen": -275.81427001953125, "logps/rejected": -284.2325744628906, "loss": 0.3734, "rewards/accuracies": 1.0, "rewards/chosen": 0.7324575781822205, "rewards/margins": 0.8292361497879028, "rewards/rejected": -0.09677852690219879, "step": 130 }, { "epoch": 3.5, "grad_norm": 16.660192489624023, "learning_rate": 8.750000000000001e-06, "logits/chosen": -3.072263240814209, "logits/rejected": -3.080423355102539, "logps/chosen": -301.5035095214844, "logps/rejected": -340.21142578125, "loss": 0.3471, "rewards/accuracies": 1.0, "rewards/chosen": 0.7654526829719543, "rewards/margins": 0.9343374967575073, "rewards/rejected": -0.16888491809368134, "step": 140 }, { "epoch": 3.75, "grad_norm": 14.524090766906738, "learning_rate": 9.375000000000001e-06, "logits/chosen": -3.060041904449463, "logits/rejected": -3.0739455223083496, "logps/chosen": -307.4438171386719, "logps/rejected": -317.702392578125, "loss": 0.2956, "rewards/accuracies": 1.0, "rewards/chosen": 1.040612816810608, "rewards/margins": 1.1534035205841064, "rewards/rejected": -0.11279074847698212, "step": 150 }, { "epoch": 4.0, "grad_norm": 16.867460250854492, "learning_rate": 1e-05, "logits/chosen": -3.006394863128662, "logits/rejected": -3.005350112915039, "logps/chosen": -261.86370849609375, "logps/rejected": -307.3501892089844, "loss": 0.2868, "rewards/accuracies": 1.0, "rewards/chosen": 1.0526248216629028, "rewards/margins": 1.1865119934082031, "rewards/rejected": -0.13388730585575104, "step": 160 }, { "epoch": 4.25, "grad_norm": 12.979177474975586, "learning_rate": 9.998810135399545e-06, "logits/chosen": -3.0648646354675293, "logits/rejected": -3.0790963172912598, "logps/chosen": -300.56158447265625, "logps/rejected": -350.8419189453125, "loss": 0.1818, "rewards/accuracies": 1.0, "rewards/chosen": 1.3793233633041382, "rewards/margins": 1.7721487283706665, "rewards/rejected": -0.39282527565956116, "step": 170 }, { "epoch": 4.5, "grad_norm": 17.333553314208984, "learning_rate": 9.99524110790929e-06, "logits/chosen": -2.979598045349121, "logits/rejected": -3.0077521800994873, "logps/chosen": -284.83245849609375, "logps/rejected": -317.96502685546875, "loss": 0.1726, "rewards/accuracies": 1.0, "rewards/chosen": 1.4392335414886475, "rewards/margins": 1.8440691232681274, "rewards/rejected": -0.4048355221748352, "step": 180 }, { "epoch": 4.75, "grad_norm": 10.082865715026855, "learning_rate": 9.989294616193018e-06, "logits/chosen": -3.0077311992645264, "logits/rejected": -3.0012829303741455, "logps/chosen": -264.1474609375, "logps/rejected": -299.96075439453125, "loss": 0.146, "rewards/accuracies": 1.0, "rewards/chosen": 1.6937824487686157, "rewards/margins": 2.088609218597412, "rewards/rejected": -0.39482688903808594, "step": 190 }, { "epoch": 5.0, "grad_norm": 7.095228672027588, "learning_rate": 9.980973490458728e-06, "logits/chosen": -3.10638165473938, "logits/rejected": -3.1016831398010254, "logps/chosen": -267.3155212402344, "logps/rejected": -291.3060607910156, "loss": 0.1178, "rewards/accuracies": 1.0, "rewards/chosen": 1.9219974279403687, "rewards/margins": 2.328953742980957, "rewards/rejected": -0.40695637464523315, "step": 200 }, { "epoch": 5.25, "grad_norm": 6.248754978179932, "learning_rate": 9.970281691111598e-06, "logits/chosen": -3.0690035820007324, "logits/rejected": -3.080627918243408, "logps/chosen": -260.17803955078125, "logps/rejected": -301.79058837890625, "loss": 0.0743, "rewards/accuracies": 1.0, "rewards/chosen": 2.114182233810425, "rewards/margins": 2.8904576301574707, "rewards/rejected": -0.7762753367424011, "step": 210 }, { "epoch": 5.5, "grad_norm": 3.346534490585327, "learning_rate": 9.957224306869053e-06, "logits/chosen": -2.9923551082611084, "logits/rejected": -2.9941794872283936, "logps/chosen": -284.899658203125, "logps/rejected": -333.97760009765625, "loss": 0.0469, "rewards/accuracies": 1.0, "rewards/chosen": 2.5475454330444336, "rewards/margins": 3.5171380043029785, "rewards/rejected": -0.9695925712585449, "step": 220 }, { "epoch": 5.75, "grad_norm": 4.401463985443115, "learning_rate": 9.941807552338805e-06, "logits/chosen": -2.996560573577881, "logits/rejected": -3.0060484409332275, "logps/chosen": -259.8013000488281, "logps/rejected": -321.4071044921875, "loss": 0.0313, "rewards/accuracies": 1.0, "rewards/chosen": 2.493565082550049, "rewards/margins": 3.8989131450653076, "rewards/rejected": -1.4053480625152588, "step": 230 }, { "epoch": 6.0, "grad_norm": 1.5952703952789307, "learning_rate": 9.924038765061042e-06, "logits/chosen": -2.9383039474487305, "logits/rejected": -3.001051664352417, "logps/chosen": -289.7506103515625, "logps/rejected": -342.30413818359375, "loss": 0.0607, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 1.786336898803711, "rewards/margins": 3.688387632369995, "rewards/rejected": -1.9020507335662842, "step": 240 }, { "epoch": 6.25, "grad_norm": 1.3278578519821167, "learning_rate": 9.903926402016153e-06, "logits/chosen": -2.9040274620056152, "logits/rejected": -2.955465316772461, "logps/chosen": -274.72857666015625, "logps/rejected": -317.9063415527344, "loss": 0.0151, "rewards/accuracies": 1.0, "rewards/chosen": 2.4853923320770264, "rewards/margins": 4.935563087463379, "rewards/rejected": -2.4501702785491943, "step": 250 }, { "epoch": 6.5, "grad_norm": 3.8670496940612793, "learning_rate": 9.881480035599667e-06, "logits/chosen": -2.9413228034973145, "logits/rejected": -2.986402988433838, "logps/chosen": -241.05313110351562, "logps/rejected": -335.13330078125, "loss": 0.0131, "rewards/accuracies": 1.0, "rewards/chosen": 2.401592969894409, "rewards/margins": 5.367430686950684, "rewards/rejected": -2.965837240219116, "step": 260 }, { "epoch": 6.75, "grad_norm": 0.7370750904083252, "learning_rate": 9.856710349066307e-06, "logits/chosen": -2.8481552600860596, "logits/rejected": -2.871796131134033, "logps/chosen": -281.4495849609375, "logps/rejected": -375.6628112792969, "loss": 0.0191, "rewards/accuracies": 1.0, "rewards/chosen": 2.0013809204101562, "rewards/margins": 5.650525093078613, "rewards/rejected": -3.6491446495056152, "step": 270 }, { "epoch": 7.0, "grad_norm": 0.38341209292411804, "learning_rate": 9.829629131445342e-06, "logits/chosen": -2.975663661956787, "logits/rejected": -2.9795100688934326, "logps/chosen": -297.9117431640625, "logps/rejected": -355.7571716308594, "loss": 0.005, "rewards/accuracies": 1.0, "rewards/chosen": 1.8948161602020264, "rewards/margins": 6.713313102722168, "rewards/rejected": -4.818497657775879, "step": 280 }, { "epoch": 7.25, "grad_norm": 0.8936977982521057, "learning_rate": 9.800249271929645e-06, "logits/chosen": -2.7922892570495605, "logits/rejected": -2.82979154586792, "logps/chosen": -301.52105712890625, "logps/rejected": -391.2060852050781, "loss": 0.0077, "rewards/accuracies": 1.0, "rewards/chosen": 1.6500043869018555, "rewards/margins": 6.588435173034668, "rewards/rejected": -4.938431262969971, "step": 290 }, { "epoch": 7.5, "grad_norm": 0.1153794676065445, "learning_rate": 9.768584753741134e-06, "logits/chosen": -2.7638492584228516, "logits/rejected": -2.8244481086730957, "logps/chosen": -250.18270874023438, "logps/rejected": -330.7033386230469, "loss": 0.0013, "rewards/accuracies": 1.0, "rewards/chosen": 2.1644153594970703, "rewards/margins": 7.6036481857299805, "rewards/rejected": -5.43923282623291, "step": 300 }, { "epoch": 7.75, "grad_norm": 0.43729519844055176, "learning_rate": 9.73465064747553e-06, "logits/chosen": -2.9230093955993652, "logits/rejected": -2.956589937210083, "logps/chosen": -259.6827392578125, "logps/rejected": -352.92327880859375, "loss": 0.0019, "rewards/accuracies": 1.0, "rewards/chosen": 1.9091575145721436, "rewards/margins": 7.56458044052124, "rewards/rejected": -5.655422687530518, "step": 310 }, { "epoch": 8.0, "grad_norm": 0.09286278486251831, "learning_rate": 9.698463103929542e-06, "logits/chosen": -2.9670042991638184, "logits/rejected": -2.979112386703491, "logps/chosen": -295.2704162597656, "logps/rejected": -391.45263671875, "loss": 0.0011, "rewards/accuracies": 1.0, "rewards/chosen": 1.921958327293396, "rewards/margins": 7.987102508544922, "rewards/rejected": -6.065144062042236, "step": 320 }, { "epoch": 8.25, "grad_norm": 0.04435203596949577, "learning_rate": 9.660039346413994e-06, "logits/chosen": -2.816281795501709, "logits/rejected": -2.8549602031707764, "logps/chosen": -269.79345703125, "logps/rejected": -387.1796875, "loss": 0.0004, "rewards/accuracies": 1.0, "rewards/chosen": 1.8488849401474, "rewards/margins": 8.709232330322266, "rewards/rejected": -6.860346794128418, "step": 330 }, { "epoch": 8.5, "grad_norm": 0.10658630728721619, "learning_rate": 9.619397662556434e-06, "logits/chosen": -2.8702447414398193, "logits/rejected": -2.8935627937316895, "logps/chosen": -267.3102111816406, "logps/rejected": -386.0473937988281, "loss": 0.0006, "rewards/accuracies": 1.0, "rewards/chosen": 1.496031641960144, "rewards/margins": 8.856051445007324, "rewards/rejected": -7.360020637512207, "step": 340 }, { "epoch": 8.75, "grad_norm": 0.34437116980552673, "learning_rate": 9.576557395597237e-06, "logits/chosen": -2.9254894256591797, "logits/rejected": -2.9604105949401855, "logps/chosen": -292.0805969238281, "logps/rejected": -366.92828369140625, "loss": 0.0006, "rewards/accuracies": 1.0, "rewards/chosen": 1.7589994668960571, "rewards/margins": 8.077742576599121, "rewards/rejected": -6.3187432289123535, "step": 350 }, { "epoch": 9.0, "grad_norm": 0.03675313666462898, "learning_rate": 9.531538935183252e-06, "logits/chosen": -2.7603306770324707, "logits/rejected": -2.778339147567749, "logps/chosen": -287.90899658203125, "logps/rejected": -375.89959716796875, "loss": 0.0211, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 1.1417970657348633, "rewards/margins": 8.125410079956055, "rewards/rejected": -6.983613014221191, "step": 360 }, { "epoch": 9.25, "grad_norm": 0.03928977623581886, "learning_rate": 9.484363707663443e-06, "logits/chosen": -2.8726091384887695, "logits/rejected": -2.914290189743042, "logps/chosen": -277.1239929199219, "logps/rejected": -403.6535949707031, "loss": 0.0004, "rewards/accuracies": 1.0, "rewards/chosen": 1.2768957614898682, "rewards/margins": 9.230466842651367, "rewards/rejected": -7.953570365905762, "step": 370 }, { "epoch": 9.5, "grad_norm": 0.07424739003181458, "learning_rate": 9.43505416589111e-06, "logits/chosen": -2.7936153411865234, "logits/rejected": -2.8210067749023438, "logps/chosen": -286.84423828125, "logps/rejected": -358.430908203125, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 1.287247896194458, "rewards/margins": 9.114188194274902, "rewards/rejected": -7.826941013336182, "step": 380 }, { "epoch": 9.75, "grad_norm": 0.16556577384471893, "learning_rate": 9.38363377853754e-06, "logits/chosen": -2.7510828971862793, "logits/rejected": -2.810314655303955, "logps/chosen": -279.12481689453125, "logps/rejected": -388.2037658691406, "loss": 0.0012, "rewards/accuracies": 1.0, "rewards/chosen": 1.3857409954071045, "rewards/margins": 8.737149238586426, "rewards/rejected": -7.3514084815979, "step": 390 }, { "epoch": 10.0, "grad_norm": 0.04436686635017395, "learning_rate": 9.330127018922195e-06, "logits/chosen": -2.8653323650360107, "logits/rejected": -2.8766775131225586, "logps/chosen": -288.20709228515625, "logps/rejected": -406.8284606933594, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": 1.3324298858642578, "rewards/margins": 9.166605949401855, "rewards/rejected": -7.834176540374756, "step": 400 }, { "epoch": 10.25, "grad_norm": 0.05012385919690132, "learning_rate": 9.274559353364734e-06, "logits/chosen": -2.864193916320801, "logits/rejected": -2.883277177810669, "logps/chosen": -267.9872131347656, "logps/rejected": -386.8029479980469, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": 0.9138404726982117, "rewards/margins": 9.733392715454102, "rewards/rejected": -8.819551467895508, "step": 410 }, { "epoch": 10.5, "grad_norm": 0.43551239371299744, "learning_rate": 9.21695722906443e-06, "logits/chosen": -2.7738356590270996, "logits/rejected": -2.8102309703826904, "logps/chosen": -263.6973571777344, "logps/rejected": -381.28717041015625, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": 1.2245395183563232, "rewards/margins": 9.739076614379883, "rewards/rejected": -8.514535903930664, "step": 420 }, { "epoch": 10.75, "grad_norm": 0.13912050426006317, "learning_rate": 9.157348061512728e-06, "logits/chosen": -2.7845568656921387, "logits/rejected": -2.8310136795043945, "logps/chosen": -286.35858154296875, "logps/rejected": -424.60162353515625, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 0.6671217679977417, "rewards/margins": 9.726836204528809, "rewards/rejected": -9.059713363647461, "step": 430 }, { "epoch": 11.0, "grad_norm": 0.06393156945705414, "learning_rate": 9.09576022144496e-06, "logits/chosen": -2.7752254009246826, "logits/rejected": -2.788267135620117, "logps/chosen": -336.1651306152344, "logps/rejected": -411.5840759277344, "loss": 0.0082, "rewards/accuracies": 1.0, "rewards/chosen": 0.1248699277639389, "rewards/margins": 9.496256828308105, "rewards/rejected": -9.37138843536377, "step": 440 }, { "epoch": 11.25, "grad_norm": 0.03681237995624542, "learning_rate": 9.032223021337415e-06, "logits/chosen": -2.8105359077453613, "logits/rejected": -2.839118242263794, "logps/chosen": -295.88751220703125, "logps/rejected": -395.9588928222656, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 1.1100804805755615, "rewards/margins": 9.81136417388916, "rewards/rejected": -8.701284408569336, "step": 450 }, { "epoch": 11.5, "grad_norm": 0.03024367056787014, "learning_rate": 8.966766701456177e-06, "logits/chosen": -2.8440585136413574, "logits/rejected": -2.879945993423462, "logps/chosen": -313.1834411621094, "logps/rejected": -407.3936462402344, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 0.3773784041404724, "rewards/margins": 9.822793960571289, "rewards/rejected": -9.445414543151855, "step": 460 }, { "epoch": 11.75, "grad_norm": 0.017395537346601486, "learning_rate": 8.899422415464409e-06, "logits/chosen": -2.8262779712677, "logits/rejected": -2.868819236755371, "logps/chosen": -275.37896728515625, "logps/rejected": -413.3304748535156, "loss": 0.0004, "rewards/accuracies": 1.0, "rewards/chosen": 1.2674087285995483, "rewards/margins": 9.833629608154297, "rewards/rejected": -8.566221237182617, "step": 470 }, { "epoch": 12.0, "grad_norm": 0.015860358253121376, "learning_rate": 8.83022221559489e-06, "logits/chosen": -2.698403835296631, "logits/rejected": -2.7310996055603027, "logps/chosen": -270.9880676269531, "logps/rejected": -397.07965087890625, "loss": 0.0135, "rewards/accuracies": 1.0, "rewards/chosen": 0.23779204487800598, "rewards/margins": 10.131186485290527, "rewards/rejected": -9.893393516540527, "step": 480 }, { "epoch": 12.25, "grad_norm": 0.011888382956385612, "learning_rate": 8.759199037394888e-06, "logits/chosen": -2.770529270172119, "logits/rejected": -2.8385539054870605, "logps/chosen": -293.5708312988281, "logps/rejected": -416.44195556640625, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.0723686218261719, "rewards/margins": 10.268576622009277, "rewards/rejected": -9.196208000183105, "step": 490 }, { "epoch": 12.5, "grad_norm": 0.03234002739191055, "learning_rate": 8.68638668405062e-06, "logits/chosen": -2.7817087173461914, "logits/rejected": -2.8032031059265137, "logps/chosen": -312.52459716796875, "logps/rejected": -415.45654296875, "loss": 0.0128, "rewards/accuracies": 1.0, "rewards/chosen": 0.5771347880363464, "rewards/margins": 9.832048416137695, "rewards/rejected": -9.254913330078125, "step": 500 }, { "epoch": 12.75, "grad_norm": 0.02184719778597355, "learning_rate": 8.611819810298778e-06, "logits/chosen": -2.790955066680908, "logits/rejected": -2.8012702465057373, "logps/chosen": -270.79864501953125, "logps/rejected": -401.427490234375, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.1068525314331055, "rewards/margins": 10.041081428527832, "rewards/rejected": -8.934228897094727, "step": 510 }, { "epoch": 13.0, "grad_norm": 0.051279619336128235, "learning_rate": 8.535533905932739e-06, "logits/chosen": -2.8458592891693115, "logits/rejected": -2.880188465118408, "logps/chosen": -275.39886474609375, "logps/rejected": -387.335693359375, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.7447389364242554, "rewards/margins": 10.408342361450195, "rewards/rejected": -9.663604736328125, "step": 520 }, { "epoch": 13.25, "grad_norm": 0.044555239379405975, "learning_rate": 8.457565278911349e-06, "logits/chosen": -2.7830259799957275, "logits/rejected": -2.8375871181488037, "logps/chosen": -270.60235595703125, "logps/rejected": -393.5851135253906, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.0359405279159546, "rewards/margins": 10.306293487548828, "rewards/rejected": -9.270353317260742, "step": 530 }, { "epoch": 13.5, "grad_norm": 0.17373812198638916, "learning_rate": 8.377951038078303e-06, "logits/chosen": -2.7552318572998047, "logits/rejected": -2.8055858612060547, "logps/chosen": -307.6959533691406, "logps/rejected": -438.8316955566406, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.6580104827880859, "rewards/margins": 10.06629467010498, "rewards/rejected": -9.408284187316895, "step": 540 }, { "epoch": 13.75, "grad_norm": 0.007444189395755529, "learning_rate": 8.296729075500345e-06, "logits/chosen": -2.933079481124878, "logits/rejected": -2.9362237453460693, "logps/chosen": -296.7786560058594, "logps/rejected": -411.70782470703125, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.005584955215454, "rewards/margins": 10.901617050170898, "rewards/rejected": -9.89603328704834, "step": 550 }, { "epoch": 14.0, "grad_norm": 0.01406220905482769, "learning_rate": 8.213938048432697e-06, "logits/chosen": -2.6971287727355957, "logits/rejected": -2.7503409385681152, "logps/chosen": -270.8581237792969, "logps/rejected": -379.1615295410156, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.0409095287322998, "rewards/margins": 10.008934020996094, "rewards/rejected": -8.968024253845215, "step": 560 }, { "epoch": 14.25, "grad_norm": 0.012559090740978718, "learning_rate": 8.129617360920297e-06, "logits/chosen": -2.891970157623291, "logits/rejected": -2.8930327892303467, "logps/chosen": -257.08953857421875, "logps/rejected": -426.03045654296875, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.4467694759368896, "rewards/margins": 10.829238891601562, "rewards/rejected": -9.382468223571777, "step": 570 }, { "epoch": 14.5, "grad_norm": 0.00887581892311573, "learning_rate": 8.043807145043604e-06, "logits/chosen": -2.800017833709717, "logits/rejected": -2.8574788570404053, "logps/chosen": -300.4933166503906, "logps/rejected": -405.90228271484375, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.45210862159729004, "rewards/margins": 10.489278793334961, "rewards/rejected": -10.037171363830566, "step": 580 }, { "epoch": 14.75, "grad_norm": 0.03181132301688194, "learning_rate": 7.956548241817914e-06, "logits/chosen": -2.773524761199951, "logits/rejected": -2.8105709552764893, "logps/chosen": -297.21917724609375, "logps/rejected": -382.3047790527344, "loss": 0.0026, "rewards/accuracies": 1.0, "rewards/chosen": 0.5754307508468628, "rewards/margins": 10.027421951293945, "rewards/rejected": -9.451990127563477, "step": 590 }, { "epoch": 15.0, "grad_norm": 0.01683628000319004, "learning_rate": 7.86788218175523e-06, "logits/chosen": -2.6840453147888184, "logits/rejected": -2.7441771030426025, "logps/chosen": -299.78656005859375, "logps/rejected": -423.99713134765625, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.4864146113395691, "rewards/margins": 10.617002487182617, "rewards/rejected": -10.130589485168457, "step": 600 }, { "epoch": 15.25, "grad_norm": 0.008264761418104172, "learning_rate": 7.777851165098012e-06, "logits/chosen": -2.758063793182373, "logits/rejected": -2.8015329837799072, "logps/chosen": -288.71295166015625, "logps/rejected": -417.6280212402344, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.4645349085330963, "rewards/margins": 10.90323257446289, "rewards/rejected": -10.43869686126709, "step": 610 }, { "epoch": 15.5, "grad_norm": 0.030291497707366943, "learning_rate": 7.686498041734121e-06, "logits/chosen": -2.782529830932617, "logits/rejected": -2.8300535678863525, "logps/chosen": -270.43450927734375, "logps/rejected": -401.63824462890625, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 1.1531331539154053, "rewards/margins": 10.583002090454102, "rewards/rejected": -9.429868698120117, "step": 620 }, { "epoch": 15.75, "grad_norm": 0.009538864716887474, "learning_rate": 7.593866290802608e-06, "logits/chosen": -2.7459161281585693, "logits/rejected": -2.788944721221924, "logps/chosen": -280.4390869140625, "logps/rejected": -394.48883056640625, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.5682659149169922, "rewards/margins": 10.8987455368042, "rewards/rejected": -10.330477714538574, "step": 630 }, { "epoch": 16.0, "grad_norm": 0.02041240781545639, "learning_rate": 7.500000000000001e-06, "logits/chosen": -2.795477867126465, "logits/rejected": -2.820490598678589, "logps/chosen": -318.35418701171875, "logps/rejected": -434.80987548828125, "loss": 0.0049, "rewards/accuracies": 1.0, "rewards/chosen": 0.37515130639076233, "rewards/margins": 10.356420516967773, "rewards/rejected": -9.981268882751465, "step": 640 }, { "epoch": 16.25, "grad_norm": 0.012305938638746738, "learning_rate": 7.404943844596939e-06, "logits/chosen": -2.665245771408081, "logits/rejected": -2.72597074508667, "logps/chosen": -330.48480224609375, "logps/rejected": -456.65411376953125, "loss": 0.0067, "rewards/accuracies": 1.0, "rewards/chosen": 0.4235839247703552, "rewards/margins": 9.822153091430664, "rewards/rejected": -9.398569107055664, "step": 650 }, { "epoch": 16.5, "grad_norm": 0.01604386977851391, "learning_rate": 7.308743066175172e-06, "logits/chosen": -2.8210482597351074, "logits/rejected": -2.8381505012512207, "logps/chosen": -291.52581787109375, "logps/rejected": -412.03546142578125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.3690633773803711, "rewards/margins": 11.374852180480957, "rewards/rejected": -11.005789756774902, "step": 660 }, { "epoch": 16.75, "grad_norm": 0.019596703350543976, "learning_rate": 7.211443451095007e-06, "logits/chosen": -2.756493091583252, "logits/rejected": -2.818694829940796, "logps/chosen": -252.20590209960938, "logps/rejected": -381.78485107421875, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.5581167936325073, "rewards/margins": 10.99728775024414, "rewards/rejected": -10.439170837402344, "step": 670 }, { "epoch": 17.0, "grad_norm": 0.0072501422837376595, "learning_rate": 7.113091308703498e-06, "logits/chosen": -2.8325161933898926, "logits/rejected": -2.866009473800659, "logps/chosen": -289.85833740234375, "logps/rejected": -412.3795471191406, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.4296005368232727, "rewards/margins": 11.440092086791992, "rewards/rejected": -11.010492324829102, "step": 680 }, { "epoch": 17.25, "grad_norm": 0.020742328837513924, "learning_rate": 7.0137334492936875e-06, "logits/chosen": -2.702772378921509, "logits/rejected": -2.769951105117798, "logps/chosen": -286.2898254394531, "logps/rejected": -432.13653564453125, "loss": 0.0012, "rewards/accuracies": 1.0, "rewards/chosen": 0.33315685391426086, "rewards/margins": 10.437726974487305, "rewards/rejected": -10.104570388793945, "step": 690 }, { "epoch": 17.5, "grad_norm": 0.025510217994451523, "learning_rate": 6.913417161825449e-06, "logits/chosen": -2.7644479274749756, "logits/rejected": -2.7879385948181152, "logps/chosen": -268.5244140625, "logps/rejected": -382.07073974609375, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.7151921987533569, "rewards/margins": 11.053590774536133, "rewards/rejected": -10.338397979736328, "step": 700 }, { "epoch": 17.75, "grad_norm": 0.002281380584463477, "learning_rate": 6.812190191418508e-06, "logits/chosen": -2.785850763320923, "logits/rejected": -2.8239619731903076, "logps/chosen": -306.57843017578125, "logps/rejected": -423.93157958984375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.2157173603773117, "rewards/margins": 10.968465805053711, "rewards/rejected": -10.752748489379883, "step": 710 }, { "epoch": 18.0, "grad_norm": 0.003572087734937668, "learning_rate": 6.710100716628345e-06, "logits/chosen": -2.803100109100342, "logits/rejected": -2.836097240447998, "logps/chosen": -304.5684509277344, "logps/rejected": -437.28302001953125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6064807772636414, "rewards/margins": 12.001432418823242, "rewards/rejected": -11.394950866699219, "step": 720 }, { "epoch": 18.25, "grad_norm": 0.008005160838365555, "learning_rate": 6.607197326515808e-06, "logits/chosen": -2.7193379402160645, "logits/rejected": -2.7719767093658447, "logps/chosen": -296.9802551269531, "logps/rejected": -437.65142822265625, "loss": 0.0008, "rewards/accuracies": 1.0, "rewards/chosen": 0.9086353182792664, "rewards/margins": 10.867486000061035, "rewards/rejected": -9.958850860595703, "step": 730 }, { "epoch": 18.5, "grad_norm": 0.009184204041957855, "learning_rate": 6.503528997521365e-06, "logits/chosen": -2.7671329975128174, "logits/rejected": -2.8140933513641357, "logps/chosen": -288.43389892578125, "logps/rejected": -418.6834411621094, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.7397573590278625, "rewards/margins": 11.15060806274414, "rewards/rejected": -10.410850524902344, "step": 740 }, { "epoch": 18.75, "grad_norm": 0.008301551453769207, "learning_rate": 6.399145070154962e-06, "logits/chosen": -2.791762113571167, "logits/rejected": -2.8275296688079834, "logps/chosen": -266.60394287109375, "logps/rejected": -389.6926574707031, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.2849633991718292, "rewards/margins": 11.423474311828613, "rewards/rejected": -11.138509750366211, "step": 750 }, { "epoch": 19.0, "grad_norm": 0.015563180670142174, "learning_rate": 6.294095225512604e-06, "logits/chosen": -2.7435309886932373, "logits/rejected": -2.7662229537963867, "logps/chosen": -313.7065734863281, "logps/rejected": -435.94647216796875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.47863197326660156, "rewards/margins": 11.74044418334961, "rewards/rejected": -12.219076156616211, "step": 760 }, { "epoch": 19.25, "grad_norm": 0.010865326970815659, "learning_rate": 6.188429461630866e-06, "logits/chosen": -2.7403242588043213, "logits/rejected": -2.7853188514709473, "logps/chosen": -310.6920166015625, "logps/rejected": -450.71630859375, "loss": 0.0147, "rewards/accuracies": 1.0, "rewards/chosen": 0.3375871777534485, "rewards/margins": 11.067597389221191, "rewards/rejected": -10.730010032653809, "step": 770 }, { "epoch": 19.5, "grad_norm": 0.011109190993010998, "learning_rate": 6.0821980696905145e-06, "logits/chosen": -2.7623090744018555, "logits/rejected": -2.8018345832824707, "logps/chosen": -275.25665283203125, "logps/rejected": -388.6085205078125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.39552536606788635, "rewards/margins": 11.198080062866211, "rewards/rejected": -10.802556037902832, "step": 780 }, { "epoch": 19.75, "grad_norm": 0.13688191771507263, "learning_rate": 5.975451610080643e-06, "logits/chosen": -2.792748212814331, "logits/rejected": -2.8350272178649902, "logps/chosen": -280.85430908203125, "logps/rejected": -433.8482971191406, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.1213034838438034, "rewards/margins": 11.554020881652832, "rewards/rejected": -11.432718276977539, "step": 790 }, { "epoch": 20.0, "grad_norm": 0.012268920429050922, "learning_rate": 5.8682408883346535e-06, "logits/chosen": -2.7681257724761963, "logits/rejected": -2.7897846698760986, "logps/chosen": -304.73272705078125, "logps/rejected": -409.9903869628906, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6212397813796997, "rewards/margins": 11.226862907409668, "rewards/rejected": -10.605623245239258, "step": 800 }, { "epoch": 20.25, "grad_norm": 0.012932064011693, "learning_rate": 5.760616930949584e-06, "logits/chosen": -2.770331859588623, "logits/rejected": -2.7890381813049316, "logps/chosen": -306.2134704589844, "logps/rejected": -418.7654724121094, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.3504090905189514, "rewards/margins": 11.633293151855469, "rewards/rejected": -11.28288459777832, "step": 810 }, { "epoch": 20.5, "grad_norm": 0.006929120514541864, "learning_rate": 5.65263096110026e-06, "logits/chosen": -2.7397124767303467, "logits/rejected": -2.7778358459472656, "logps/chosen": -308.2324523925781, "logps/rejected": -470.67791748046875, "loss": 0.0006, "rewards/accuracies": 1.0, "rewards/chosen": 0.12212464958429337, "rewards/margins": 10.697164535522461, "rewards/rejected": -10.575040817260742, "step": 820 }, { "epoch": 20.75, "grad_norm": 0.0058187334798276424, "learning_rate": 5.544334374259823e-06, "logits/chosen": -2.820895195007324, "logits/rejected": -2.853609085083008, "logps/chosen": -296.0104064941406, "logps/rejected": -399.3705749511719, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6735066175460815, "rewards/margins": 11.60952091217041, "rewards/rejected": -10.936014175415039, "step": 830 }, { "epoch": 21.0, "grad_norm": 0.00335478107444942, "learning_rate": 5.435778713738292e-06, "logits/chosen": -2.7048516273498535, "logits/rejected": -2.756258726119995, "logps/chosen": -254.4524688720703, "logps/rejected": -396.0684814453125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6581499576568604, "rewards/margins": 11.730459213256836, "rewards/rejected": -11.072309494018555, "step": 840 }, { "epoch": 21.25, "grad_norm": 0.018709037452936172, "learning_rate": 5.327015646150716e-06, "logits/chosen": -2.6166112422943115, "logits/rejected": -2.677870273590088, "logps/chosen": -297.18731689453125, "logps/rejected": -451.8585510253906, "loss": 0.0059, "rewards/accuracies": 1.0, "rewards/chosen": 0.0066245137713849545, "rewards/margins": 10.767498970031738, "rewards/rejected": -10.760875701904297, "step": 850 }, { "epoch": 21.5, "grad_norm": 0.00640977593138814, "learning_rate": 5.218096936826681e-06, "logits/chosen": -2.7767691612243652, "logits/rejected": -2.782492160797119, "logps/chosen": -281.61224365234375, "logps/rejected": -401.4766845703125, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.07375653833150864, "rewards/margins": 11.642614364624023, "rewards/rejected": -11.568860054016113, "step": 860 }, { "epoch": 21.75, "grad_norm": 0.018667172640562057, "learning_rate": 5.109074425172806e-06, "logits/chosen": -2.754488468170166, "logits/rejected": -2.797163486480713, "logps/chosen": -284.8846435546875, "logps/rejected": -415.6575622558594, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.540486216545105, "rewards/margins": 11.744099617004395, "rewards/rejected": -11.20361328125, "step": 870 }, { "epoch": 22.0, "grad_norm": 0.003822013735771179, "learning_rate": 5e-06, "logits/chosen": -2.859811782836914, "logits/rejected": -2.891792058944702, "logps/chosen": -310.1468200683594, "logps/rejected": -433.6126403808594, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.2926860451698303, "rewards/margins": 12.372417449951172, "rewards/rejected": -12.079731941223145, "step": 880 }, { "epoch": 22.25, "grad_norm": 0.010801389813423157, "learning_rate": 4.890925574827195e-06, "logits/chosen": -2.7312240600585938, "logits/rejected": -2.7544617652893066, "logps/chosen": -293.9632568359375, "logps/rejected": -422.1595764160156, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.030094634741544724, "rewards/margins": 11.75137996673584, "rewards/rejected": -11.721285820007324, "step": 890 }, { "epoch": 22.5, "grad_norm": 0.00307653215713799, "learning_rate": 4.781903063173321e-06, "logits/chosen": -2.7293999195098877, "logits/rejected": -2.801996946334839, "logps/chosen": -290.69244384765625, "logps/rejected": -423.0887756347656, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5350134372711182, "rewards/margins": 12.045049667358398, "rewards/rejected": -11.510037422180176, "step": 900 }, { "epoch": 22.75, "grad_norm": 0.0032190086785703897, "learning_rate": 4.672984353849285e-06, "logits/chosen": -2.7523961067199707, "logits/rejected": -2.7586894035339355, "logps/chosen": -295.1702575683594, "logps/rejected": -440.543701171875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.17444385588169098, "rewards/margins": 12.515422821044922, "rewards/rejected": -12.340980529785156, "step": 910 }, { "epoch": 23.0, "grad_norm": 0.013608491979539394, "learning_rate": 4.564221286261709e-06, "logits/chosen": -2.7768361568450928, "logits/rejected": -2.829939126968384, "logps/chosen": -296.3355712890625, "logps/rejected": -421.0843811035156, "loss": 0.0216, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -0.010329616256058216, "rewards/margins": 10.561881065368652, "rewards/rejected": -10.572213172912598, "step": 920 }, { "epoch": 23.25, "grad_norm": 0.010324417613446712, "learning_rate": 4.4556656257401786e-06, "logits/chosen": -2.814089298248291, "logits/rejected": -2.8441967964172363, "logps/chosen": -245.9521026611328, "logps/rejected": -392.0475158691406, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.9622662663459778, "rewards/margins": 11.689537048339844, "rewards/rejected": -10.72727108001709, "step": 930 }, { "epoch": 23.5, "grad_norm": 0.00877166073769331, "learning_rate": 4.347369038899744e-06, "logits/chosen": -2.763094186782837, "logits/rejected": -2.803144931793213, "logps/chosen": -303.9397277832031, "logps/rejected": -402.18841552734375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5226415991783142, "rewards/margins": 11.711947441101074, "rewards/rejected": -11.18930721282959, "step": 940 }, { "epoch": 23.75, "grad_norm": 0.012196135707199574, "learning_rate": 4.239383069050417e-06, "logits/chosen": -2.7214579582214355, "logits/rejected": -2.751098155975342, "logps/chosen": -308.78851318359375, "logps/rejected": -460.69281005859375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.11721920967102051, "rewards/margins": 12.034148216247559, "rewards/rejected": -12.151368141174316, "step": 950 }, { "epoch": 24.0, "grad_norm": 0.016689885407686234, "learning_rate": 4.131759111665349e-06, "logits/chosen": -2.7126431465148926, "logits/rejected": -2.766502618789673, "logps/chosen": -310.998291015625, "logps/rejected": -442.939697265625, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.4348395764827728, "rewards/margins": 11.327308654785156, "rewards/rejected": -10.892468452453613, "step": 960 }, { "epoch": 24.25, "grad_norm": 0.7392864227294922, "learning_rate": 4.02454838991936e-06, "logits/chosen": -2.6969475746154785, "logits/rejected": -2.702312469482422, "logps/chosen": -306.0939025878906, "logps/rejected": -427.08074951171875, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 0.31883805990219116, "rewards/margins": 11.24242115020752, "rewards/rejected": -10.923582077026367, "step": 970 }, { "epoch": 24.5, "grad_norm": 0.003211956238374114, "learning_rate": 3.917801930309486e-06, "logits/chosen": -2.782787322998047, "logits/rejected": -2.808108329772949, "logps/chosen": -258.2384033203125, "logps/rejected": -389.11798095703125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 1.0415832996368408, "rewards/margins": 11.802536010742188, "rewards/rejected": -10.760952949523926, "step": 980 }, { "epoch": 24.75, "grad_norm": 0.003792324336245656, "learning_rate": 3.8115705383691354e-06, "logits/chosen": -2.756258010864258, "logits/rejected": -2.803229331970215, "logps/chosen": -304.2614440917969, "logps/rejected": -457.0116271972656, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.01773083209991455, "rewards/margins": 12.328390121459961, "rewards/rejected": -12.346120834350586, "step": 990 }, { "epoch": 25.0, "grad_norm": 0.00849173590540886, "learning_rate": 3.705904774487396e-06, "logits/chosen": -2.7826552391052246, "logits/rejected": -2.8513171672821045, "logps/chosen": -300.17535400390625, "logps/rejected": -427.40234375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5104937553405762, "rewards/margins": 11.4827880859375, "rewards/rejected": -10.972294807434082, "step": 1000 }, { "epoch": 25.25, "grad_norm": 0.00654919259250164, "learning_rate": 3.6008549298450403e-06, "logits/chosen": -2.7531707286834717, "logits/rejected": -2.7878074645996094, "logps/chosen": -284.87396240234375, "logps/rejected": -390.9798583984375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6618450880050659, "rewards/margins": 11.684257507324219, "rewards/rejected": -11.022411346435547, "step": 1010 }, { "epoch": 25.5, "grad_norm": 0.0041053337045013905, "learning_rate": 3.4964710024786354e-06, "logits/chosen": -2.6924521923065186, "logits/rejected": -2.7480525970458984, "logps/chosen": -282.3283386230469, "logps/rejected": -394.1541442871094, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.988944411277771, "rewards/margins": 11.823431015014648, "rewards/rejected": -10.83448600769043, "step": 1020 }, { "epoch": 25.75, "grad_norm": 0.004295062739402056, "learning_rate": 3.3928026734841935e-06, "logits/chosen": -2.7896180152893066, "logits/rejected": -2.833456516265869, "logps/chosen": -286.898193359375, "logps/rejected": -469.9169006347656, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": 0.3953208327293396, "rewards/margins": 11.806564331054688, "rewards/rejected": -11.41124153137207, "step": 1030 }, { "epoch": 26.0, "grad_norm": 0.007940116338431835, "learning_rate": 3.289899283371657e-06, "logits/chosen": -2.7841601371765137, "logits/rejected": -2.8034167289733887, "logps/chosen": -310.6112060546875, "logps/rejected": -443.7947692871094, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": -0.23492594063282013, "rewards/margins": 11.810868263244629, "rewards/rejected": -12.045793533325195, "step": 1040 }, { "epoch": 26.25, "grad_norm": 0.006178665440529585, "learning_rate": 3.1878098085814926e-06, "logits/chosen": -2.7314186096191406, "logits/rejected": -2.7590575218200684, "logps/chosen": -291.75933837890625, "logps/rejected": -405.439208984375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.7959985136985779, "rewards/margins": 11.849149703979492, "rewards/rejected": -11.05315113067627, "step": 1050 }, { "epoch": 26.5, "grad_norm": 0.0034364075399935246, "learning_rate": 3.0865828381745515e-06, "logits/chosen": -2.6799168586730957, "logits/rejected": -2.752761125564575, "logps/chosen": -320.683837890625, "logps/rejected": -476.5531311035156, "loss": 0.0017, "rewards/accuracies": 1.0, "rewards/chosen": -0.32047194242477417, "rewards/margins": 11.475412368774414, "rewards/rejected": -11.79588508605957, "step": 1060 }, { "epoch": 26.75, "grad_norm": 0.012047665193676949, "learning_rate": 2.986266550706315e-06, "logits/chosen": -2.823770523071289, "logits/rejected": -2.831331729888916, "logps/chosen": -292.18621826171875, "logps/rejected": -409.607177734375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.3296182155609131, "rewards/margins": 12.108281135559082, "rewards/rejected": -11.77866268157959, "step": 1070 }, { "epoch": 27.0, "grad_norm": 0.00699230283498764, "learning_rate": 2.886908691296504e-06, "logits/chosen": -2.768615961074829, "logits/rejected": -2.8066773414611816, "logps/chosen": -266.96826171875, "logps/rejected": -419.3490295410156, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5609016418457031, "rewards/margins": 12.091551780700684, "rewards/rejected": -11.530649185180664, "step": 1080 }, { "epoch": 27.25, "grad_norm": 0.0021058099810034037, "learning_rate": 2.7885565489049948e-06, "logits/chosen": -2.813713550567627, "logits/rejected": -2.8499045372009277, "logps/chosen": -264.84381103515625, "logps/rejected": -419.009765625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.655460000038147, "rewards/margins": 12.120909690856934, "rewards/rejected": -11.465449333190918, "step": 1090 }, { "epoch": 27.5, "grad_norm": 0.011556684039533138, "learning_rate": 2.6912569338248317e-06, "logits/chosen": -2.765392780303955, "logits/rejected": -2.78332781791687, "logps/chosen": -313.57452392578125, "logps/rejected": -453.7864685058594, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": -0.08190927654504776, "rewards/margins": 12.152857780456543, "rewards/rejected": -12.23476791381836, "step": 1100 }, { "epoch": 27.75, "grad_norm": 0.01144924946129322, "learning_rate": 2.595056155403063e-06, "logits/chosen": -2.7482686042785645, "logits/rejected": -2.7814669609069824, "logps/chosen": -288.89404296875, "logps/rejected": -417.9188537597656, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6152423620223999, "rewards/margins": 12.10200309753418, "rewards/rejected": -11.486761093139648, "step": 1110 }, { "epoch": 28.0, "grad_norm": 0.007494707591831684, "learning_rate": 2.5000000000000015e-06, "logits/chosen": -2.678860902786255, "logits/rejected": -2.7341535091400146, "logps/chosen": -302.2644958496094, "logps/rejected": -424.6878967285156, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.14390479028224945, "rewards/margins": 11.76207160949707, "rewards/rejected": -11.618168830871582, "step": 1120 }, { "epoch": 28.25, "grad_norm": 0.00261974660679698, "learning_rate": 2.406133709197392e-06, "logits/chosen": -2.8185958862304688, "logits/rejected": -2.8240275382995605, "logps/chosen": -288.8084411621094, "logps/rejected": -427.37762451171875, "loss": 0.0049, "rewards/accuracies": 1.0, "rewards/chosen": 0.27108827233314514, "rewards/margins": 11.560935020446777, "rewards/rejected": -11.289847373962402, "step": 1130 }, { "epoch": 28.5, "grad_norm": 0.006601645611226559, "learning_rate": 2.3135019582658803e-06, "logits/chosen": -2.759382486343384, "logits/rejected": -2.8069920539855957, "logps/chosen": -298.5120544433594, "logps/rejected": -424.13360595703125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5905256271362305, "rewards/margins": 12.292265892028809, "rewards/rejected": -11.701741218566895, "step": 1140 }, { "epoch": 28.75, "grad_norm": 0.006943762768059969, "learning_rate": 2.2221488349019903e-06, "logits/chosen": -2.6638615131378174, "logits/rejected": -2.73032546043396, "logps/chosen": -294.6612548828125, "logps/rejected": -444.9859313964844, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.28661757707595825, "rewards/margins": 12.06439208984375, "rewards/rejected": -12.3510103225708, "step": 1150 }, { "epoch": 29.0, "grad_norm": 0.0011426012497395277, "learning_rate": 2.132117818244771e-06, "logits/chosen": -2.7360968589782715, "logits/rejected": -2.767688512802124, "logps/chosen": -291.3235168457031, "logps/rejected": -420.4112854003906, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6397272348403931, "rewards/margins": 12.181781768798828, "rewards/rejected": -11.542055130004883, "step": 1160 }, { "epoch": 29.25, "grad_norm": 0.003964864648878574, "learning_rate": 2.0434517581820893e-06, "logits/chosen": -2.6658082008361816, "logits/rejected": -2.717425584793091, "logps/chosen": -343.0668640136719, "logps/rejected": -475.0995178222656, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 0.19656455516815186, "rewards/margins": 11.766244888305664, "rewards/rejected": -11.569681167602539, "step": 1170 }, { "epoch": 29.5, "grad_norm": 0.003464967478066683, "learning_rate": 1.956192854956397e-06, "logits/chosen": -2.7914059162139893, "logits/rejected": -2.8226091861724854, "logps/chosen": -284.6141052246094, "logps/rejected": -396.912109375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.4676376283168793, "rewards/margins": 12.429913520812988, "rewards/rejected": -11.962274551391602, "step": 1180 }, { "epoch": 29.75, "grad_norm": 0.0023241571616381407, "learning_rate": 1.8703826390797047e-06, "logits/chosen": -2.740171432495117, "logits/rejected": -2.7767977714538574, "logps/chosen": -265.77099609375, "logps/rejected": -446.67431640625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.4891475737094879, "rewards/margins": 12.44828987121582, "rewards/rejected": -11.959141731262207, "step": 1190 }, { "epoch": 30.0, "grad_norm": 0.0072005875408649445, "learning_rate": 1.7860619515673034e-06, "logits/chosen": -2.7672040462493896, "logits/rejected": -2.8059945106506348, "logps/chosen": -281.13922119140625, "logps/rejected": -401.81390380859375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.1551550179719925, "rewards/margins": 11.701306343078613, "rewards/rejected": -11.856461524963379, "step": 1200 }, { "epoch": 30.25, "grad_norm": 0.002375040901824832, "learning_rate": 1.7032709244996559e-06, "logits/chosen": -2.78956937789917, "logits/rejected": -2.806525468826294, "logps/chosen": -312.67327880859375, "logps/rejected": -466.92034912109375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.4753616452217102, "rewards/margins": 12.765235900878906, "rewards/rejected": -13.24059772491455, "step": 1210 }, { "epoch": 30.5, "grad_norm": 0.007024294696748257, "learning_rate": 1.6220489619216988e-06, "logits/chosen": -2.6667380332946777, "logits/rejected": -2.7224926948547363, "logps/chosen": -296.1967468261719, "logps/rejected": -395.61114501953125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.27107173204421997, "rewards/margins": 11.369800567626953, "rewards/rejected": -11.098730087280273, "step": 1220 }, { "epoch": 30.75, "grad_norm": 0.002501589013263583, "learning_rate": 1.5424347210886538e-06, "logits/chosen": -2.7434449195861816, "logits/rejected": -2.7846179008483887, "logps/chosen": -303.44464111328125, "logps/rejected": -444.24127197265625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6443105340003967, "rewards/margins": 12.676385879516602, "rewards/rejected": -12.032075881958008, "step": 1230 }, { "epoch": 31.0, "grad_norm": 0.005482690874487162, "learning_rate": 1.4644660940672628e-06, "logits/chosen": -2.7875418663024902, "logits/rejected": -2.7959892749786377, "logps/chosen": -263.8461608886719, "logps/rejected": -416.28662109375, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 0.32634425163269043, "rewards/margins": 11.619983673095703, "rewards/rejected": -11.293639183044434, "step": 1240 }, { "epoch": 31.25, "grad_norm": 0.006386056076735258, "learning_rate": 1.3881801897012225e-06, "logits/chosen": -2.666128635406494, "logits/rejected": -2.6980326175689697, "logps/chosen": -307.4803771972656, "logps/rejected": -438.99853515625, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": 0.005978202912956476, "rewards/margins": 11.530052185058594, "rewards/rejected": -11.524072647094727, "step": 1250 }, { "epoch": 31.5, "grad_norm": 0.010775255970656872, "learning_rate": 1.3136133159493803e-06, "logits/chosen": -2.7379791736602783, "logits/rejected": -2.7717361450195312, "logps/chosen": -262.4990539550781, "logps/rejected": -398.7112731933594, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6824764609336853, "rewards/margins": 12.771679878234863, "rewards/rejected": -12.089202880859375, "step": 1260 }, { "epoch": 31.75, "grad_norm": 0.005956208799034357, "learning_rate": 1.2408009626051137e-06, "logits/chosen": -2.8342833518981934, "logits/rejected": -2.8574931621551514, "logps/chosen": -311.24884033203125, "logps/rejected": -467.5516662597656, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.5595420002937317, "rewards/margins": 12.868734359741211, "rewards/rejected": -13.428276062011719, "step": 1270 }, { "epoch": 32.0, "grad_norm": 0.00918492116034031, "learning_rate": 1.1697777844051105e-06, "logits/chosen": -2.725320339202881, "logits/rejected": -2.775146245956421, "logps/chosen": -297.0390319824219, "logps/rejected": -427.73724365234375, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.23792123794555664, "rewards/margins": 11.733413696289062, "rewards/rejected": -11.495491981506348, "step": 1280 }, { "epoch": 32.25, "grad_norm": 0.0025609093718230724, "learning_rate": 1.100577584535592e-06, "logits/chosen": -2.6968932151794434, "logits/rejected": -2.7568459510803223, "logps/chosen": -276.00592041015625, "logps/rejected": -420.32989501953125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.28203171491622925, "rewards/margins": 12.333124160766602, "rewards/rejected": -12.051092147827148, "step": 1290 }, { "epoch": 32.5, "grad_norm": 0.002115601906552911, "learning_rate": 1.0332332985438248e-06, "logits/chosen": -2.7094054222106934, "logits/rejected": -2.7540245056152344, "logps/chosen": -278.0862121582031, "logps/rejected": -412.1825256347656, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.8267415165901184, "rewards/margins": 12.288800239562988, "rewards/rejected": -11.462059020996094, "step": 1300 }, { "epoch": 32.75, "grad_norm": 0.010958091355860233, "learning_rate": 9.677769786625869e-07, "logits/chosen": -2.8227927684783936, "logits/rejected": -2.853522539138794, "logps/chosen": -327.1927490234375, "logps/rejected": -470.41546630859375, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": -0.44436079263687134, "rewards/margins": 11.060840606689453, "rewards/rejected": -11.505200386047363, "step": 1310 }, { "epoch": 33.0, "grad_norm": 0.001769970403984189, "learning_rate": 9.042397785550405e-07, "logits/chosen": -2.719733715057373, "logits/rejected": -2.772249460220337, "logps/chosen": -299.26641845703125, "logps/rejected": -427.08612060546875, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": -0.377460777759552, "rewards/margins": 13.097183227539062, "rewards/rejected": -13.474644660949707, "step": 1320 }, { "epoch": 33.25, "grad_norm": 0.017327116802334785, "learning_rate": 8.426519384872733e-07, "logits/chosen": -2.7387938499450684, "logits/rejected": -2.7465500831604004, "logps/chosen": -316.33941650390625, "logps/rejected": -463.70294189453125, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": -0.29718923568725586, "rewards/margins": 12.517646789550781, "rewards/rejected": -12.814834594726562, "step": 1330 }, { "epoch": 33.5, "grad_norm": 0.015168960206210613, "learning_rate": 7.830427709355726e-07, "logits/chosen": -2.7458043098449707, "logits/rejected": -2.7986690998077393, "logps/chosen": -282.633056640625, "logps/rejected": -415.61865234375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.6882933378219604, "rewards/margins": 12.167409896850586, "rewards/rejected": -11.479117393493652, "step": 1340 }, { "epoch": 33.75, "grad_norm": 0.0029403967782855034, "learning_rate": 7.254406466352682e-07, "logits/chosen": -2.751441240310669, "logits/rejected": -2.7536258697509766, "logps/chosen": -271.40545654296875, "logps/rejected": -417.96044921875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.024983251467347145, "rewards/margins": 12.195524215698242, "rewards/rejected": -12.220507621765137, "step": 1350 }, { "epoch": 34.0, "grad_norm": 0.008198041468858719, "learning_rate": 6.698729810778065e-07, "logits/chosen": -2.764251470565796, "logits/rejected": -2.802863836288452, "logps/chosen": -311.47430419921875, "logps/rejected": -441.355224609375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.037785936146974564, "rewards/margins": 12.428993225097656, "rewards/rejected": -12.466779708862305, "step": 1360 }, { "epoch": 34.25, "grad_norm": 0.01094425655901432, "learning_rate": 6.163662214624616e-07, "logits/chosen": -2.7676146030426025, "logits/rejected": -2.7861106395721436, "logps/chosen": -317.7796936035156, "logps/rejected": -462.99884033203125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.8035726547241211, "rewards/margins": 13.26586627960205, "rewards/rejected": -14.069439888000488, "step": 1370 }, { "epoch": 34.5, "grad_norm": 0.014173522591590881, "learning_rate": 5.649458341088915e-07, "logits/chosen": -2.7046866416931152, "logits/rejected": -2.780492067337036, "logps/chosen": -266.5248718261719, "logps/rejected": -408.0611267089844, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.9178122282028198, "rewards/margins": 11.700557708740234, "rewards/rejected": -10.782745361328125, "step": 1380 }, { "epoch": 34.75, "grad_norm": 0.005380965769290924, "learning_rate": 5.156362923365587e-07, "logits/chosen": -2.804652214050293, "logits/rejected": -2.8121368885040283, "logps/chosen": -317.160888671875, "logps/rejected": -424.66339111328125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.029048467054963112, "rewards/margins": 12.284951210021973, "rewards/rejected": -12.314001083374023, "step": 1390 }, { "epoch": 35.0, "grad_norm": 0.0023389519192278385, "learning_rate": 4.6846106481675035e-07, "logits/chosen": -2.6911251544952393, "logits/rejected": -2.7417197227478027, "logps/chosen": -279.67474365234375, "logps/rejected": -443.27264404296875, "loss": 0.0001, "rewards/accuracies": 1.0, "rewards/chosen": 0.21463599801063538, "rewards/margins": 12.136545181274414, "rewards/rejected": -11.921908378601074, "step": 1400 }, { "epoch": 35.25, "grad_norm": 0.017130231484770775, "learning_rate": 4.2344260440276455e-07, "logits/chosen": -2.705173969268799, "logits/rejected": -2.755000591278076, "logps/chosen": -279.12811279296875, "logps/rejected": -430.97509765625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5486839413642883, "rewards/margins": 12.646631240844727, "rewards/rejected": -12.09794807434082, "step": 1410 }, { "epoch": 35.5, "grad_norm": 0.007070457097142935, "learning_rate": 3.8060233744356634e-07, "logits/chosen": -2.706882953643799, "logits/rejected": -2.758007287979126, "logps/chosen": -290.89569091796875, "logps/rejected": -439.9056091308594, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.09179888665676117, "rewards/margins": 11.889338493347168, "rewards/rejected": -11.797539710998535, "step": 1420 }, { "epoch": 35.75, "grad_norm": 0.0010205187136307359, "learning_rate": 3.399606535860078e-07, "logits/chosen": -2.761723279953003, "logits/rejected": -2.7938809394836426, "logps/chosen": -297.20263671875, "logps/rejected": -408.91900634765625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.1678989827632904, "rewards/margins": 12.045710563659668, "rewards/rejected": -12.21360969543457, "step": 1430 }, { "epoch": 36.0, "grad_norm": 0.0035463483072817326, "learning_rate": 3.015368960704584e-07, "logits/chosen": -2.7846503257751465, "logits/rejected": -2.818844795227051, "logps/chosen": -316.3269958496094, "logps/rejected": -458.6512145996094, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": -0.409940242767334, "rewards/margins": 12.515970230102539, "rewards/rejected": -12.925909042358398, "step": 1440 }, { "epoch": 36.25, "grad_norm": 0.002209126716479659, "learning_rate": 2.653493525244721e-07, "logits/chosen": -2.8237838745117188, "logits/rejected": -2.8457577228546143, "logps/chosen": -266.9129333496094, "logps/rejected": -445.2333984375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5144069790840149, "rewards/margins": 12.918063163757324, "rewards/rejected": -12.403654098510742, "step": 1450 }, { "epoch": 36.5, "grad_norm": 0.006079660728573799, "learning_rate": 2.314152462588659e-07, "logits/chosen": -2.733881711959839, "logits/rejected": -2.7485454082489014, "logps/chosen": -292.0540466308594, "logps/rejected": -413.36590576171875, "loss": 0.0191, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": 0.3418157994747162, "rewards/margins": 11.966304779052734, "rewards/rejected": -11.624489784240723, "step": 1460 }, { "epoch": 36.75, "grad_norm": 0.004013635218143463, "learning_rate": 1.99750728070357e-07, "logits/chosen": -2.6816630363464355, "logits/rejected": -2.748070001602173, "logps/chosen": -331.69964599609375, "logps/rejected": -471.43798828125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.7932103872299194, "rewards/margins": 11.995916366577148, "rewards/rejected": -12.789125442504883, "step": 1470 }, { "epoch": 37.0, "grad_norm": 0.010304308496415615, "learning_rate": 1.7037086855465902e-07, "logits/chosen": -2.7218892574310303, "logits/rejected": -2.7555019855499268, "logps/chosen": -291.52740478515625, "logps/rejected": -406.4840087890625, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.12983234226703644, "rewards/margins": 12.319366455078125, "rewards/rejected": -12.189533233642578, "step": 1480 }, { "epoch": 37.25, "grad_norm": 0.0003127313684672117, "learning_rate": 1.4328965093369284e-07, "logits/chosen": -2.6925010681152344, "logits/rejected": -2.7279093265533447, "logps/chosen": -284.3682556152344, "logps/rejected": -427.28729248046875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.09465457499027252, "rewards/margins": 12.547832489013672, "rewards/rejected": -12.453178405761719, "step": 1490 }, { "epoch": 37.5, "grad_norm": 0.007795688696205616, "learning_rate": 1.185199644003332e-07, "logits/chosen": -2.79750657081604, "logits/rejected": -2.8254566192626953, "logps/chosen": -292.19561767578125, "logps/rejected": -425.38128662109375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.0007894396549090743, "rewards/margins": 12.570943832397461, "rewards/rejected": -12.570155143737793, "step": 1500 }, { "epoch": 37.75, "grad_norm": 0.004515082575380802, "learning_rate": 9.607359798384785e-08, "logits/chosen": -2.8021745681762695, "logits/rejected": -2.8447844982147217, "logps/chosen": -306.9266357421875, "logps/rejected": -404.08831787109375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.2106831967830658, "rewards/margins": 12.148092269897461, "rewards/rejected": -12.358774185180664, "step": 1510 }, { "epoch": 38.0, "grad_norm": 0.2741380035877228, "learning_rate": 7.59612349389599e-08, "logits/chosen": -2.702080488204956, "logits/rejected": -2.7362682819366455, "logps/chosen": -303.89068603515625, "logps/rejected": -479.22119140625, "loss": 0.0012, "rewards/accuracies": 1.0, "rewards/chosen": 0.2993970811367035, "rewards/margins": 11.771306991577148, "rewards/rejected": -11.47191047668457, "step": 1520 }, { "epoch": 38.25, "grad_norm": 0.005564592778682709, "learning_rate": 5.8192447661196694e-08, "logits/chosen": -2.7832558155059814, "logits/rejected": -2.825497627258301, "logps/chosen": -267.9512939453125, "logps/rejected": -420.0101623535156, "loss": 0.0046, "rewards/accuracies": 1.0, "rewards/chosen": 0.6914939284324646, "rewards/margins": 11.328948020935059, "rewards/rejected": -10.63745403289795, "step": 1530 }, { "epoch": 38.5, "grad_norm": 0.006241649389266968, "learning_rate": 4.2775693130948094e-08, "logits/chosen": -2.7838566303253174, "logits/rejected": -2.7983555793762207, "logps/chosen": -312.9120788574219, "logps/rejected": -482.3827209472656, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.37507203221321106, "rewards/margins": 13.01745319366455, "rewards/rejected": -13.392524719238281, "step": 1540 }, { "epoch": 38.75, "grad_norm": 0.0031968543771654367, "learning_rate": 2.971830888840177e-08, "logits/chosen": -2.751046657562256, "logits/rejected": -2.774620532989502, "logps/chosen": -289.15106201171875, "logps/rejected": -391.25018310546875, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.36087581515312195, "rewards/margins": 12.353649139404297, "rewards/rejected": -12.71452522277832, "step": 1550 }, { "epoch": 39.0, "grad_norm": 0.0021725529804825783, "learning_rate": 1.9026509541272276e-08, "logits/chosen": -2.6909258365631104, "logits/rejected": -2.746695041656494, "logps/chosen": -310.7889709472656, "logps/rejected": -442.99005126953125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.3208140730857849, "rewards/margins": 12.539921760559082, "rewards/rejected": -12.219108581542969, "step": 1560 }, { "epoch": 39.25, "grad_norm": 0.005507487803697586, "learning_rate": 1.0705383806982606e-08, "logits/chosen": -2.7035202980041504, "logits/rejected": -2.7346391677856445, "logps/chosen": -311.97955322265625, "logps/rejected": -452.98773193359375, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.07614202797412872, "rewards/margins": 12.285137176513672, "rewards/rejected": -12.361278533935547, "step": 1570 }, { "epoch": 39.5, "grad_norm": 0.007295460440218449, "learning_rate": 4.758892090711009e-09, "logits/chosen": -2.7759146690368652, "logits/rejected": -2.8210902214050293, "logps/chosen": -297.54779052734375, "logps/rejected": -436.24365234375, "loss": 0.0018, "rewards/accuracies": 1.0, "rewards/chosen": -0.14304715394973755, "rewards/margins": 11.92789077758789, "rewards/rejected": -12.070939064025879, "step": 1580 }, { "epoch": 39.75, "grad_norm": 0.003202753607183695, "learning_rate": 1.189864600454338e-09, "logits/chosen": -2.77467679977417, "logits/rejected": -2.812164783477783, "logps/chosen": -293.31451416015625, "logps/rejected": -415.6048889160156, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": -0.15772318840026855, "rewards/margins": 12.923543930053711, "rewards/rejected": -13.081266403198242, "step": 1590 }, { "epoch": 40.0, "grad_norm": 0.004265669733285904, "learning_rate": 0.0, "logits/chosen": -2.672964572906494, "logits/rejected": -2.7406535148620605, "logps/chosen": -279.08856201171875, "logps/rejected": -434.34405517578125, "loss": 0.0, "rewards/accuracies": 1.0, "rewards/chosen": 0.5602867603302002, "rewards/margins": 12.123626708984375, "rewards/rejected": -11.56334114074707, "step": 1600 } ], "logging_steps": 10, "max_steps": 1600, "num_input_tokens_seen": 0, "num_train_epochs": 40, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }