{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 3.0, "eval_steps": 500, "global_step": 1407, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.010666666666666666, "grad_norm": 3.066758394241333, "learning_rate": 4.651162790697675e-07, "logits/chosen": -1.3197795152664185, "logits/rejected": -1.2224810123443604, "logps/chosen": -204.17431640625, "logps/rejected": -269.80377197265625, "loss": 0.6924, "rewards/accuracies": 0.34375, "rewards/chosen": 0.0002665018546395004, "rewards/margins": 0.0014735888689756393, "rewards/rejected": -0.001207086956128478, "step": 5 }, { "epoch": 0.021333333333333333, "grad_norm": 2.98679518699646, "learning_rate": 1.0465116279069768e-06, "logits/chosen": -1.4425337314605713, "logits/rejected": -1.2940964698791504, "logps/chosen": -222.4615020751953, "logps/rejected": -268.4735107421875, "loss": 0.6907, "rewards/accuracies": 0.637499988079071, "rewards/chosen": -0.0007552242605015635, "rewards/margins": 0.004916320089250803, "rewards/rejected": -0.005671544000506401, "step": 10 }, { "epoch": 0.032, "grad_norm": 3.7883105278015137, "learning_rate": 1.6279069767441862e-06, "logits/chosen": -1.313001036643982, "logits/rejected": -1.217395305633545, "logps/chosen": -210.53271484375, "logps/rejected": -285.55682373046875, "loss": 0.6822, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": 0.0025394915137439966, "rewards/margins": 0.02237287536263466, "rewards/rejected": -0.019833387807011604, "step": 15 }, { "epoch": 0.042666666666666665, "grad_norm": 2.714655876159668, "learning_rate": 2.2093023255813954e-06, "logits/chosen": -1.4219006299972534, "logits/rejected": -1.280832052230835, "logps/chosen": -238.50692749023438, "logps/rejected": -272.02459716796875, "loss": 0.6745, "rewards/accuracies": 0.768750011920929, "rewards/chosen": -0.002396273659542203, "rewards/margins": 0.03847304731607437, "rewards/rejected": -0.040869321674108505, "step": 20 }, { "epoch": 0.05333333333333334, "grad_norm": 2.902036428451538, "learning_rate": 2.790697674418605e-06, "logits/chosen": -1.4469196796417236, "logits/rejected": -1.2472175359725952, "logps/chosen": -211.281982421875, "logps/rejected": -252.13522338867188, "loss": 0.6594, "rewards/accuracies": 0.831250011920929, "rewards/chosen": 0.004396043252199888, "rewards/margins": 0.07000456750392914, "rewards/rejected": -0.06560851633548737, "step": 25 }, { "epoch": 0.064, "grad_norm": 2.938281297683716, "learning_rate": 3.372093023255814e-06, "logits/chosen": -1.4945942163467407, "logits/rejected": -1.301879644393921, "logps/chosen": -212.25778198242188, "logps/rejected": -261.4756164550781, "loss": 0.6355, "rewards/accuracies": 0.8125, "rewards/chosen": 0.005297659430652857, "rewards/margins": 0.12356561422348022, "rewards/rejected": -0.1182679682970047, "step": 30 }, { "epoch": 0.07466666666666667, "grad_norm": 2.9229683876037598, "learning_rate": 3.953488372093024e-06, "logits/chosen": -1.4990527629852295, "logits/rejected": -1.313819408416748, "logps/chosen": -206.4686737060547, "logps/rejected": -267.2922058105469, "loss": 0.6139, "rewards/accuracies": 0.8125, "rewards/chosen": 0.006779120769351721, "rewards/margins": 0.17405611276626587, "rewards/rejected": -0.1672770082950592, "step": 35 }, { "epoch": 0.08533333333333333, "grad_norm": 2.707540512084961, "learning_rate": 4.5348837209302326e-06, "logits/chosen": -1.4922987222671509, "logits/rejected": -1.3102424144744873, "logps/chosen": -214.93185424804688, "logps/rejected": -265.9106140136719, "loss": 0.5915, "rewards/accuracies": 0.793749988079071, "rewards/chosen": -0.011891713365912437, "rewards/margins": 0.23512515425682068, "rewards/rejected": -0.24701690673828125, "step": 40 }, { "epoch": 0.096, "grad_norm": 2.74711012840271, "learning_rate": 4.99999336897255e-06, "logits/chosen": -1.4568241834640503, "logits/rejected": -1.3190878629684448, "logps/chosen": -222.8993682861328, "logps/rejected": -264.18621826171875, "loss": 0.5688, "rewards/accuracies": 0.800000011920929, "rewards/chosen": -0.018879670649766922, "rewards/margins": 0.30939120054244995, "rewards/rejected": -0.3282708525657654, "step": 45 }, { "epoch": 0.10666666666666667, "grad_norm": 2.671937942504883, "learning_rate": 4.999761286705285e-06, "logits/chosen": -1.5383602380752563, "logits/rejected": -1.4474451541900635, "logps/chosen": -217.95303344726562, "logps/rejected": -269.7035217285156, "loss": 0.5035, "rewards/accuracies": 0.887499988079071, "rewards/chosen": 0.00281223701313138, "rewards/margins": 0.4914434552192688, "rewards/rejected": -0.48863115906715393, "step": 50 }, { "epoch": 0.11733333333333333, "grad_norm": 2.843602180480957, "learning_rate": 4.999197688241076e-06, "logits/chosen": -1.55707848072052, "logits/rejected": -1.3680062294006348, "logps/chosen": -212.3292236328125, "logps/rejected": -257.45703125, "loss": 0.492, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -0.03159821033477783, "rewards/margins": 0.5501774549484253, "rewards/rejected": -0.5817757248878479, "step": 55 }, { "epoch": 0.128, "grad_norm": 2.6057443618774414, "learning_rate": 4.9983026483238685e-06, "logits/chosen": -1.4305418729782104, "logits/rejected": -1.2694460153579712, "logps/chosen": -216.9158477783203, "logps/rejected": -259.95635986328125, "loss": 0.4569, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -0.14237883687019348, "rewards/margins": 0.7050798535346985, "rewards/rejected": -0.8474586606025696, "step": 60 }, { "epoch": 0.13866666666666666, "grad_norm": 2.425227642059326, "learning_rate": 4.997076285653087e-06, "logits/chosen": -1.608428716659546, "logits/rejected": -1.3585041761398315, "logps/chosen": -262.80120849609375, "logps/rejected": -263.1980895996094, "loss": 0.4232, "rewards/accuracies": 0.831250011920929, "rewards/chosen": -0.00389072741381824, "rewards/margins": 0.8892091512680054, "rewards/rejected": -0.8930999040603638, "step": 65 }, { "epoch": 0.14933333333333335, "grad_norm": 2.3068881034851074, "learning_rate": 4.995518762867897e-06, "logits/chosen": -1.5861982107162476, "logits/rejected": -1.2906171083450317, "logps/chosen": -175.07534790039062, "logps/rejected": -259.4339904785156, "loss": 0.3609, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -0.07811270654201508, "rewards/margins": 1.111189603805542, "rewards/rejected": -1.1893023252487183, "step": 70 }, { "epoch": 0.16, "grad_norm": 1.8398106098175049, "learning_rate": 4.993630286525634e-06, "logits/chosen": -1.5915586948394775, "logits/rejected": -1.345815896987915, "logps/chosen": -199.1811065673828, "logps/rejected": -284.7003173828125, "loss": 0.3512, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -0.17231644690036774, "rewards/margins": 1.2371994256973267, "rewards/rejected": -1.4095157384872437, "step": 75 }, { "epoch": 0.17066666666666666, "grad_norm": 3.092566967010498, "learning_rate": 4.99141110707441e-06, "logits/chosen": -1.5269695520401, "logits/rejected": -1.3563108444213867, "logps/chosen": -229.861328125, "logps/rejected": -276.8891296386719, "loss": 0.421, "rewards/accuracies": 0.8062499761581421, "rewards/chosen": -0.34268826246261597, "rewards/margins": 1.11867356300354, "rewards/rejected": -1.4613617658615112, "step": 80 }, { "epoch": 0.18133333333333335, "grad_norm": 2.967259645462036, "learning_rate": 4.9888615188199e-06, "logits/chosen": -1.6198972463607788, "logits/rejected": -1.4323259592056274, "logps/chosen": -210.46420288085938, "logps/rejected": -266.1463928222656, "loss": 0.3459, "rewards/accuracies": 0.8687499761581421, "rewards/chosen": -0.2865266799926758, "rewards/margins": 1.4747304916381836, "rewards/rejected": -1.7612571716308594, "step": 85 }, { "epoch": 0.192, "grad_norm": 2.3651838302612305, "learning_rate": 4.9859818598863104e-06, "logits/chosen": -1.5707067251205444, "logits/rejected": -1.3580453395843506, "logps/chosen": -205.1324920654297, "logps/rejected": -268.135498046875, "loss": 0.3159, "rewards/accuracies": 0.856249988079071, "rewards/chosen": -0.22517745196819305, "rewards/margins": 1.6200993061065674, "rewards/rejected": -1.8452764749526978, "step": 90 }, { "epoch": 0.20266666666666666, "grad_norm": 2.180738925933838, "learning_rate": 4.982772512171538e-06, "logits/chosen": -1.6069647073745728, "logits/rejected": -1.3733923435211182, "logps/chosen": -221.70205688476562, "logps/rejected": -266.65936279296875, "loss": 0.3344, "rewards/accuracies": 0.862500011920929, "rewards/chosen": -0.3390544652938843, "rewards/margins": 1.6071252822875977, "rewards/rejected": -1.946179747581482, "step": 95 }, { "epoch": 0.21333333333333335, "grad_norm": 2.911017417907715, "learning_rate": 4.979233901296523e-06, "logits/chosen": -1.639183759689331, "logits/rejected": -1.477196455001831, "logps/chosen": -209.12045288085938, "logps/rejected": -278.9235534667969, "loss": 0.3083, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -0.20341868698596954, "rewards/margins": 1.9127600193023682, "rewards/rejected": -2.116178512573242, "step": 100 }, { "epoch": 0.224, "grad_norm": 2.939307689666748, "learning_rate": 4.975366496548804e-06, "logits/chosen": -1.6216596364974976, "logits/rejected": -1.4458906650543213, "logps/chosen": -211.3112335205078, "logps/rejected": -282.20977783203125, "loss": 0.3117, "rewards/accuracies": 0.875, "rewards/chosen": -0.2629513144493103, "rewards/margins": 1.812154769897461, "rewards/rejected": -2.075106143951416, "step": 105 }, { "epoch": 0.23466666666666666, "grad_norm": 2.625629425048828, "learning_rate": 4.971170810820279e-06, "logits/chosen": -1.4949898719787598, "logits/rejected": -1.3904539346694946, "logps/chosen": -223.5279998779297, "logps/rejected": -269.46875, "loss": 0.3132, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.3314862549304962, "rewards/margins": 1.777958631515503, "rewards/rejected": -2.1094448566436768, "step": 110 }, { "epoch": 0.24533333333333332, "grad_norm": 3.023860216140747, "learning_rate": 4.96664740053919e-06, "logits/chosen": -1.642871618270874, "logits/rejected": -1.433465838432312, "logps/chosen": -209.8994140625, "logps/rejected": -304.18511962890625, "loss": 0.255, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.1216890811920166, "rewards/margins": 2.104187488555908, "rewards/rejected": -2.225876569747925, "step": 115 }, { "epoch": 0.256, "grad_norm": 3.2610344886779785, "learning_rate": 4.961796865596328e-06, "logits/chosen": -1.5937464237213135, "logits/rejected": -1.4256340265274048, "logps/chosen": -212.27304077148438, "logps/rejected": -290.52996826171875, "loss": 0.2696, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.09086675941944122, "rewards/margins": 1.9682395458221436, "rewards/rejected": -2.0591063499450684, "step": 120 }, { "epoch": 0.26666666666666666, "grad_norm": 3.3823888301849365, "learning_rate": 4.956619849265473e-06, "logits/chosen": -1.6408603191375732, "logits/rejected": -1.4571561813354492, "logps/chosen": -204.12246704101562, "logps/rejected": -274.4853820800781, "loss": 0.2656, "rewards/accuracies": 0.875, "rewards/chosen": 0.09454663842916489, "rewards/margins": 2.163012742996216, "rewards/rejected": -2.0684661865234375, "step": 125 }, { "epoch": 0.2773333333333333, "grad_norm": 2.7941596508026123, "learning_rate": 4.951117038118091e-06, "logits/chosen": -1.7679383754730225, "logits/rejected": -1.5151549577713013, "logps/chosen": -195.41064453125, "logps/rejected": -282.4126892089844, "loss": 0.2546, "rewards/accuracies": 0.875, "rewards/chosen": 0.09231258183717728, "rewards/margins": 2.3412868976593018, "rewards/rejected": -2.248974323272705, "step": 130 }, { "epoch": 0.288, "grad_norm": 2.4657509326934814, "learning_rate": 4.945289161932273e-06, "logits/chosen": -1.653076171875, "logits/rejected": -1.4966038465499878, "logps/chosen": -220.5636444091797, "logps/rejected": -305.3443298339844, "loss": 0.2716, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.18228843808174133, "rewards/margins": 2.278294324874878, "rewards/rejected": -2.4605822563171387, "step": 135 }, { "epoch": 0.2986666666666667, "grad_norm": 1.8939595222473145, "learning_rate": 4.93913699359596e-06, "logits/chosen": -1.6945682764053345, "logits/rejected": -1.454713225364685, "logps/chosen": -195.47763061523438, "logps/rejected": -295.0526123046875, "loss": 0.243, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.133425772190094, "rewards/margins": 2.4948630332946777, "rewards/rejected": -2.628288984298706, "step": 140 }, { "epoch": 0.30933333333333335, "grad_norm": 2.4227945804595947, "learning_rate": 4.9326613490044364e-06, "logits/chosen": -1.6317930221557617, "logits/rejected": -1.4936482906341553, "logps/chosen": -194.4447479248047, "logps/rejected": -272.43817138671875, "loss": 0.2312, "rewards/accuracies": 0.90625, "rewards/chosen": -0.060553692281246185, "rewards/margins": 2.502532482147217, "rewards/rejected": -2.5630862712860107, "step": 145 }, { "epoch": 0.32, "grad_norm": 2.1593096256256104, "learning_rate": 4.925863086952132e-06, "logits/chosen": -1.7782328128814697, "logits/rejected": -1.4951965808868408, "logps/chosen": -226.15151977539062, "logps/rejected": -306.14990234375, "loss": 0.2448, "rewards/accuracies": 0.90625, "rewards/chosen": -0.27948349714279175, "rewards/margins": 2.5876212120056152, "rewards/rejected": -2.8671042919158936, "step": 150 }, { "epoch": 0.33066666666666666, "grad_norm": 2.2807936668395996, "learning_rate": 4.918743109018727e-06, "logits/chosen": -1.8481355905532837, "logits/rejected": -1.5771901607513428, "logps/chosen": -200.24911499023438, "logps/rejected": -291.2858581542969, "loss": 0.2295, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -0.19684214890003204, "rewards/margins": 2.7656893730163574, "rewards/rejected": -2.962531566619873, "step": 155 }, { "epoch": 0.3413333333333333, "grad_norm": 2.781081199645996, "learning_rate": 4.911302359449585e-06, "logits/chosen": -1.7050254344940186, "logits/rejected": -1.5437817573547363, "logps/chosen": -211.6606903076172, "logps/rejected": -313.52203369140625, "loss": 0.1747, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.4274963438510895, "rewards/margins": 3.1094963550567627, "rewards/rejected": -3.5369930267333984, "step": 160 }, { "epoch": 0.352, "grad_norm": 3.3999805450439453, "learning_rate": 4.9035418250305314e-06, "logits/chosen": -1.7344772815704346, "logits/rejected": -1.4708181619644165, "logps/chosen": -196.78701782226562, "logps/rejected": -279.26397705078125, "loss": 0.2272, "rewards/accuracies": 0.9375, "rewards/chosen": -0.15598328411579132, "rewards/margins": 2.7185912132263184, "rewards/rejected": -2.8745741844177246, "step": 165 }, { "epoch": 0.3626666666666667, "grad_norm": 1.978710412979126, "learning_rate": 4.895462534956981e-06, "logits/chosen": -1.60355544090271, "logits/rejected": -1.51287841796875, "logps/chosen": -216.2396697998047, "logps/rejected": -283.2531433105469, "loss": 0.1758, "rewards/accuracies": 0.9375, "rewards/chosen": -0.42660221457481384, "rewards/margins": 3.12617564201355, "rewards/rejected": -3.5527777671813965, "step": 170 }, { "epoch": 0.37333333333333335, "grad_norm": 2.282623767852783, "learning_rate": 4.887065560697449e-06, "logits/chosen": -1.6703214645385742, "logits/rejected": -1.4852759838104248, "logps/chosen": -228.0096893310547, "logps/rejected": -297.5685729980469, "loss": 0.2108, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -0.459760844707489, "rewards/margins": 3.1451258659362793, "rewards/rejected": -3.604886531829834, "step": 175 }, { "epoch": 0.384, "grad_norm": 5.582287788391113, "learning_rate": 4.878352015851459e-06, "logits/chosen": -1.6780675649642944, "logits/rejected": -1.4558064937591553, "logps/chosen": -244.1021270751953, "logps/rejected": -275.14801025390625, "loss": 0.2492, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.52231365442276, "rewards/margins": 3.0731654167175293, "rewards/rejected": -3.5954787731170654, "step": 180 }, { "epoch": 0.39466666666666667, "grad_norm": 2.2447121143341064, "learning_rate": 4.869323056001852e-06, "logits/chosen": -1.637040376663208, "logits/rejected": -1.53791344165802, "logps/chosen": -196.7782745361328, "logps/rejected": -282.7772216796875, "loss": 0.218, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.40790072083473206, "rewards/margins": 3.118037700653076, "rewards/rejected": -3.5259385108947754, "step": 185 }, { "epoch": 0.4053333333333333, "grad_norm": 2.1624879837036133, "learning_rate": 4.859979878561536e-06, "logits/chosen": -1.8144794702529907, "logits/rejected": -1.6100742816925049, "logps/chosen": -216.9547576904297, "logps/rejected": -315.37200927734375, "loss": 0.2231, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.23768725991249084, "rewards/margins": 3.17393159866333, "rewards/rejected": -3.411618709564209, "step": 190 }, { "epoch": 0.416, "grad_norm": 2.3933207988739014, "learning_rate": 4.850323722614685e-06, "logits/chosen": -1.76116943359375, "logits/rejected": -1.6245193481445312, "logps/chosen": -199.56607055664062, "logps/rejected": -301.9045104980469, "loss": 0.2032, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.5237962603569031, "rewards/margins": 3.485597610473633, "rewards/rejected": -4.00939416885376, "step": 195 }, { "epoch": 0.4266666666666667, "grad_norm": 3.6170833110809326, "learning_rate": 4.840355868752416e-06, "logits/chosen": -1.6510473489761353, "logits/rejected": -1.5840654373168945, "logps/chosen": -268.947265625, "logps/rejected": -308.9850158691406, "loss": 0.2189, "rewards/accuracies": 0.893750011920929, "rewards/chosen": -0.20040014386177063, "rewards/margins": 3.211362838745117, "rewards/rejected": -3.4117634296417236, "step": 200 }, { "epoch": 0.43733333333333335, "grad_norm": 2.984818935394287, "learning_rate": 4.830077638902955e-06, "logits/chosen": -1.811394453048706, "logits/rejected": -1.5767472982406616, "logps/chosen": -205.17953491210938, "logps/rejected": -299.9072265625, "loss": 0.2058, "rewards/accuracies": 0.90625, "rewards/chosen": -0.42685478925704956, "rewards/margins": 3.5523762702941895, "rewards/rejected": -3.9792308807373047, "step": 205 }, { "epoch": 0.448, "grad_norm": 2.1014328002929688, "learning_rate": 4.8194903961563255e-06, "logits/chosen": -1.8685874938964844, "logits/rejected": -1.6196568012237549, "logps/chosen": -221.1055145263672, "logps/rejected": -294.41632080078125, "loss": 0.176, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -0.4206708073616028, "rewards/margins": 3.7292656898498535, "rewards/rejected": -4.149935722351074, "step": 210 }, { "epoch": 0.45866666666666667, "grad_norm": 5.251358985900879, "learning_rate": 4.808595544583578e-06, "logits/chosen": -1.778464674949646, "logits/rejected": -1.6301262378692627, "logps/chosen": -212.44192504882812, "logps/rejected": -294.9122009277344, "loss": 0.1973, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -0.4846112132072449, "rewards/margins": 3.767674684524536, "rewards/rejected": -4.252285957336426, "step": 215 }, { "epoch": 0.4693333333333333, "grad_norm": 2.667045831680298, "learning_rate": 4.797394529050577e-06, "logits/chosen": -1.7264249324798584, "logits/rejected": -1.6010576486587524, "logps/chosen": -236.34164428710938, "logps/rejected": -300.9986572265625, "loss": 0.196, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.28851407766342163, "rewards/margins": 3.629396438598633, "rewards/rejected": -3.917910099029541, "step": 220 }, { "epoch": 0.48, "grad_norm": 1.5569956302642822, "learning_rate": 4.785888835026393e-06, "logits/chosen": -1.781558632850647, "logits/rejected": -1.5860977172851562, "logps/chosen": -217.97152709960938, "logps/rejected": -281.3045654296875, "loss": 0.2418, "rewards/accuracies": 0.8812500238418579, "rewards/chosen": -0.35533851385116577, "rewards/margins": 3.287522077560425, "rewards/rejected": -3.6428608894348145, "step": 225 }, { "epoch": 0.49066666666666664, "grad_norm": 3.2689754962921143, "learning_rate": 4.7740799883862966e-06, "logits/chosen": -1.7362861633300781, "logits/rejected": -1.6026140451431274, "logps/chosen": -214.66616821289062, "logps/rejected": -305.7296447753906, "loss": 0.1496, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -0.2582802176475525, "rewards/margins": 3.910578489303589, "rewards/rejected": -4.168859004974365, "step": 230 }, { "epoch": 0.5013333333333333, "grad_norm": 4.52901029586792, "learning_rate": 4.761969555209394e-06, "logits/chosen": -1.7565912008285522, "logits/rejected": -1.6375453472137451, "logps/chosen": -210.79177856445312, "logps/rejected": -305.73822021484375, "loss": 0.2532, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.5417909026145935, "rewards/margins": 3.8477985858917236, "rewards/rejected": -4.389589309692383, "step": 235 }, { "epoch": 0.512, "grad_norm": 4.18936014175415, "learning_rate": 4.7495591415709435e-06, "logits/chosen": -1.6808313131332397, "logits/rejected": -1.5127551555633545, "logps/chosen": -217.28466796875, "logps/rejected": -306.2400817871094, "loss": 0.2088, "rewards/accuracies": 0.90625, "rewards/chosen": -0.44368869066238403, "rewards/margins": 3.673114061355591, "rewards/rejected": -4.11680269241333, "step": 240 }, { "epoch": 0.5226666666666666, "grad_norm": 3.285203218460083, "learning_rate": 4.736850393329353e-06, "logits/chosen": -1.8288180828094482, "logits/rejected": -1.6284465789794922, "logps/chosen": -221.87826538085938, "logps/rejected": -316.38104248046875, "loss": 0.1934, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -0.36977553367614746, "rewards/margins": 3.9045498371124268, "rewards/rejected": -4.274325370788574, "step": 245 }, { "epoch": 0.5333333333333333, "grad_norm": 4.095683574676514, "learning_rate": 4.7238449959079095e-06, "logits/chosen": -1.8461517095565796, "logits/rejected": -1.6903188228607178, "logps/chosen": -196.01882934570312, "logps/rejected": -302.14788818359375, "loss": 0.2039, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.24502971768379211, "rewards/margins": 3.6217434406280518, "rewards/rejected": -3.8667728900909424, "step": 250 }, { "epoch": 0.544, "grad_norm": 1.374786615371704, "learning_rate": 4.710544674071259e-06, "logits/chosen": -1.7972240447998047, "logits/rejected": -1.580497145652771, "logps/chosen": -213.19131469726562, "logps/rejected": -290.453125, "loss": 0.1502, "rewards/accuracies": 0.9375, "rewards/chosen": -0.20407120883464813, "rewards/margins": 4.157924175262451, "rewards/rejected": -4.361996173858643, "step": 255 }, { "epoch": 0.5546666666666666, "grad_norm": 2.1457080841064453, "learning_rate": 4.696951191696673e-06, "logits/chosen": -1.8307178020477295, "logits/rejected": -1.6306785345077515, "logps/chosen": -203.19503784179688, "logps/rejected": -329.07586669921875, "loss": 0.1253, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.26944035291671753, "rewards/margins": 4.326865196228027, "rewards/rejected": -4.596304893493652, "step": 260 }, { "epoch": 0.5653333333333334, "grad_norm": 2.9863739013671875, "learning_rate": 4.6830663515401196e-06, "logits/chosen": -1.6911954879760742, "logits/rejected": -1.5336120128631592, "logps/chosen": -212.753173828125, "logps/rejected": -283.8844299316406, "loss": 0.1979, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.37001127004623413, "rewards/margins": 3.8743293285369873, "rewards/rejected": -4.244340419769287, "step": 265 }, { "epoch": 0.576, "grad_norm": 2.874260425567627, "learning_rate": 4.6688919949971855e-06, "logits/chosen": -1.7525287866592407, "logits/rejected": -1.6272214651107788, "logps/chosen": -207.9388885498047, "logps/rejected": -303.46429443359375, "loss": 0.1714, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -0.3549277186393738, "rewards/margins": 4.431785583496094, "rewards/rejected": -4.786712646484375, "step": 270 }, { "epoch": 0.5866666666666667, "grad_norm": 2.1960959434509277, "learning_rate": 4.654430001858874e-06, "logits/chosen": -1.7510707378387451, "logits/rejected": -1.570367693901062, "logps/chosen": -238.5811767578125, "logps/rejected": -299.3658752441406, "loss": 0.1824, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.21106000244617462, "rewards/margins": 3.991091251373291, "rewards/rejected": -4.202151298522949, "step": 275 }, { "epoch": 0.5973333333333334, "grad_norm": 2.189987897872925, "learning_rate": 4.6396822900623075e-06, "logits/chosen": -1.7313045263290405, "logits/rejected": -1.5751632452011108, "logps/chosen": -211.08139038085938, "logps/rejected": -303.4233703613281, "loss": 0.1475, "rewards/accuracies": 0.9375, "rewards/chosen": -0.5332785248756409, "rewards/margins": 4.4254255294799805, "rewards/rejected": -4.958703994750977, "step": 280 }, { "epoch": 0.608, "grad_norm": 10.575684547424316, "learning_rate": 4.624650815436368e-06, "logits/chosen": -1.8441358804702759, "logits/rejected": -1.6829391717910767, "logps/chosen": -232.00424194335938, "logps/rejected": -311.0403747558594, "loss": 0.2001, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.5895386338233948, "rewards/margins": 4.146374702453613, "rewards/rejected": -4.735913276672363, "step": 285 }, { "epoch": 0.6186666666666667, "grad_norm": 1.8553959131240845, "learning_rate": 4.609337571442324e-06, "logits/chosen": -1.8459510803222656, "logits/rejected": -1.7000173330307007, "logps/chosen": -191.3725128173828, "logps/rejected": -306.92742919921875, "loss": 0.158, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.5892201662063599, "rewards/margins": 4.554601192474365, "rewards/rejected": -5.143821716308594, "step": 290 }, { "epoch": 0.6293333333333333, "grad_norm": 2.1983144283294678, "learning_rate": 4.5937445889094535e-06, "logits/chosen": -1.6407095193862915, "logits/rejected": -1.5538684129714966, "logps/chosen": -235.87109375, "logps/rejected": -304.3785705566406, "loss": 0.1787, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.501952588558197, "rewards/margins": 4.091989040374756, "rewards/rejected": -4.593941688537598, "step": 295 }, { "epoch": 0.64, "grad_norm": 1.2407987117767334, "learning_rate": 4.577873935765722e-06, "logits/chosen": -1.9018303155899048, "logits/rejected": -1.693389892578125, "logps/chosen": -209.4306640625, "logps/rejected": -332.36517333984375, "loss": 0.0919, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -0.5658477544784546, "rewards/margins": 5.112331390380859, "rewards/rejected": -5.678179740905762, "step": 300 }, { "epoch": 0.6506666666666666, "grad_norm": 1.5631043910980225, "learning_rate": 4.561727716763532e-06, "logits/chosen": -1.7761032581329346, "logits/rejected": -1.6239516735076904, "logps/chosen": -236.88400268554688, "logps/rejected": -315.68780517578125, "loss": 0.1755, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.5217987895011902, "rewards/margins": 4.937346935272217, "rewards/rejected": -5.459145545959473, "step": 305 }, { "epoch": 0.6613333333333333, "grad_norm": 1.763484001159668, "learning_rate": 4.545308073200591e-06, "logits/chosen": -1.6939966678619385, "logits/rejected": -1.5736029148101807, "logps/chosen": -225.3236083984375, "logps/rejected": -334.81121826171875, "loss": 0.1684, "rewards/accuracies": 0.9375, "rewards/chosen": -0.5952404737472534, "rewards/margins": 4.6785101890563965, "rewards/rejected": -5.273751258850098, "step": 310 }, { "epoch": 0.672, "grad_norm": 3.163860321044922, "learning_rate": 4.528617182635943e-06, "logits/chosen": -1.934821367263794, "logits/rejected": -1.7904952764511108, "logps/chosen": -203.1216278076172, "logps/rejected": -331.48663330078125, "loss": 0.1783, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.5681445002555847, "rewards/margins": 4.853375434875488, "rewards/rejected": -5.421519756317139, "step": 315 }, { "epoch": 0.6826666666666666, "grad_norm": 1.8331536054611206, "learning_rate": 4.511657258601171e-06, "logits/chosen": -1.7939611673355103, "logits/rejected": -1.6202869415283203, "logps/chosen": -211.56533813476562, "logps/rejected": -303.11834716796875, "loss": 0.1335, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -0.6103761792182922, "rewards/margins": 5.00363826751709, "rewards/rejected": -5.614014625549316, "step": 320 }, { "epoch": 0.6933333333333334, "grad_norm": 1.3633877038955688, "learning_rate": 4.494430550306848e-06, "logits/chosen": -1.6322972774505615, "logits/rejected": -1.6094653606414795, "logps/chosen": -216.24557495117188, "logps/rejected": -314.92193603515625, "loss": 0.1366, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -0.6063324809074402, "rewards/margins": 4.898473262786865, "rewards/rejected": -5.504805564880371, "step": 325 }, { "epoch": 0.704, "grad_norm": 1.5049735307693481, "learning_rate": 4.476939342344246e-06, "logits/chosen": -1.830894112586975, "logits/rejected": -1.6699556112289429, "logps/chosen": -202.03094482421875, "logps/rejected": -319.1224060058594, "loss": 0.1148, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -0.5489516854286194, "rewards/margins": 5.44413423538208, "rewards/rejected": -5.993086338043213, "step": 330 }, { "epoch": 0.7146666666666667, "grad_norm": 2.9216864109039307, "learning_rate": 4.45918595438236e-06, "logits/chosen": -1.7787328958511353, "logits/rejected": -1.6474401950836182, "logps/chosen": -213.6125030517578, "logps/rejected": -314.5631103515625, "loss": 0.1801, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -0.7099496126174927, "rewards/margins": 4.897684574127197, "rewards/rejected": -5.607633590698242, "step": 335 }, { "epoch": 0.7253333333333334, "grad_norm": 4.806079864501953, "learning_rate": 4.441172740860267e-06, "logits/chosen": -1.8154537677764893, "logits/rejected": -1.6812942028045654, "logps/chosen": -228.5246124267578, "logps/rejected": -320.4506530761719, "loss": 0.1543, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -0.6220763325691223, "rewards/margins": 4.960526943206787, "rewards/rejected": -5.582603454589844, "step": 340 }, { "epoch": 0.736, "grad_norm": 2.260047674179077, "learning_rate": 4.422902090674886e-06, "logits/chosen": -1.6601673364639282, "logits/rejected": -1.5397001504898071, "logps/chosen": -214.0808563232422, "logps/rejected": -315.56097412109375, "loss": 0.1274, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -0.5406688451766968, "rewards/margins": 5.2952775955200195, "rewards/rejected": -5.835946559906006, "step": 345 }, { "epoch": 0.7466666666666667, "grad_norm": 2.84885835647583, "learning_rate": 4.4043764268641705e-06, "logits/chosen": -1.7757313251495361, "logits/rejected": -1.5302140712738037, "logps/chosen": -200.83782958984375, "logps/rejected": -310.81170654296875, "loss": 0.1321, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.5374216437339783, "rewards/margins": 5.517133712768555, "rewards/rejected": -6.054555416107178, "step": 350 }, { "epoch": 0.7573333333333333, "grad_norm": 3.0638837814331055, "learning_rate": 4.385598206285758e-06, "logits/chosen": -1.8033117055892944, "logits/rejected": -1.6644874811172485, "logps/chosen": -227.2172088623047, "logps/rejected": -337.43292236328125, "loss": 0.159, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -0.6855946779251099, "rewards/margins": 5.421025276184082, "rewards/rejected": -6.106619834899902, "step": 355 }, { "epoch": 0.768, "grad_norm": 3.001445770263672, "learning_rate": 4.3665699192911495e-06, "logits/chosen": -1.7176687717437744, "logits/rejected": -1.6000038385391235, "logps/chosen": -218.575927734375, "logps/rejected": -318.5699157714844, "loss": 0.1649, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.8334270715713501, "rewards/margins": 5.054620742797852, "rewards/rejected": -5.888047695159912, "step": 360 }, { "epoch": 0.7786666666666666, "grad_norm": 3.584191083908081, "learning_rate": 4.347294089395438e-06, "logits/chosen": -1.786150574684143, "logits/rejected": -1.6655750274658203, "logps/chosen": -239.9277801513672, "logps/rejected": -339.0256652832031, "loss": 0.132, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -0.7777694463729858, "rewards/margins": 5.329873561859131, "rewards/rejected": -6.107643127441406, "step": 365 }, { "epoch": 0.7893333333333333, "grad_norm": 2.659503698348999, "learning_rate": 4.327773272942647e-06, "logits/chosen": -1.806623101234436, "logits/rejected": -1.6834806203842163, "logps/chosen": -200.3981170654297, "logps/rejected": -310.19976806640625, "loss": 0.1844, "rewards/accuracies": 0.90625, "rewards/chosen": -0.8376480937004089, "rewards/margins": 5.1270952224731445, "rewards/rejected": -5.964743137359619, "step": 370 }, { "epoch": 0.8, "grad_norm": 2.4195191860198975, "learning_rate": 4.308010058766703e-06, "logits/chosen": -1.7857242822647095, "logits/rejected": -1.6443517208099365, "logps/chosen": -219.5111846923828, "logps/rejected": -346.9495544433594, "loss": 0.1878, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -1.1358754634857178, "rewards/margins": 5.381402969360352, "rewards/rejected": -6.51727819442749, "step": 375 }, { "epoch": 0.8106666666666666, "grad_norm": 3.1393349170684814, "learning_rate": 4.2880070678481135e-06, "logits/chosen": -1.8039171695709229, "logits/rejected": -1.5734426975250244, "logps/chosen": -218.7495574951172, "logps/rejected": -320.8000183105469, "loss": 0.156, "rewards/accuracies": 0.9312499761581421, "rewards/chosen": -1.4172883033752441, "rewards/margins": 5.502140045166016, "rewards/rejected": -6.91942834854126, "step": 380 }, { "epoch": 0.8213333333333334, "grad_norm": 6.10773229598999, "learning_rate": 4.267766952966369e-06, "logits/chosen": -1.7366790771484375, "logits/rejected": -1.6303174495697021, "logps/chosen": -213.4806671142578, "logps/rejected": -313.7432861328125, "loss": 0.2032, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": -1.0189698934555054, "rewards/margins": 5.155045509338379, "rewards/rejected": -6.174015045166016, "step": 385 }, { "epoch": 0.832, "grad_norm": 1.2211837768554688, "learning_rate": 4.24729239834814e-06, "logits/chosen": -1.7981315851211548, "logits/rejected": -1.6859023571014404, "logps/chosen": -214.998779296875, "logps/rejected": -304.8877258300781, "loss": 0.1508, "rewards/accuracies": 0.9375, "rewards/chosen": -0.7106130123138428, "rewards/margins": 5.31712007522583, "rewards/rejected": -6.027733325958252, "step": 390 }, { "epoch": 0.8426666666666667, "grad_norm": 5.1816630363464355, "learning_rate": 4.226586119311291e-06, "logits/chosen": -1.8075335025787354, "logits/rejected": -1.7757810354232788, "logps/chosen": -230.5372314453125, "logps/rejected": -332.861572265625, "loss": 0.197, "rewards/accuracies": 0.918749988079071, "rewards/chosen": -0.9396656155586243, "rewards/margins": 4.923933982849121, "rewards/rejected": -5.8635993003845215, "step": 395 }, { "epoch": 0.8533333333333334, "grad_norm": 2.9293625354766846, "learning_rate": 4.205650861904782e-06, "logits/chosen": -1.8099794387817383, "logits/rejected": -1.6509116888046265, "logps/chosen": -201.75900268554688, "logps/rejected": -337.0589294433594, "loss": 0.1469, "rewards/accuracies": 0.9375, "rewards/chosen": -0.8850414156913757, "rewards/margins": 5.205699920654297, "rewards/rejected": -6.090741157531738, "step": 400 }, { "epoch": 0.864, "grad_norm": 1.2313185930252075, "learning_rate": 4.184489402544486e-06, "logits/chosen": -1.8144805431365967, "logits/rejected": -1.6301672458648682, "logps/chosen": -206.3054656982422, "logps/rejected": -325.19049072265625, "loss": 0.135, "rewards/accuracies": 0.9375, "rewards/chosen": -0.9306513071060181, "rewards/margins": 5.58634090423584, "rewards/rejected": -6.516992092132568, "step": 405 }, { "epoch": 0.8746666666666667, "grad_norm": 2.0421454906463623, "learning_rate": 4.1631045476449865e-06, "logits/chosen": -1.8140941858291626, "logits/rejected": -1.686056137084961, "logps/chosen": -221.7446746826172, "logps/rejected": -328.5806884765625, "loss": 0.1628, "rewards/accuracies": 0.90625, "rewards/chosen": -0.9575926065444946, "rewards/margins": 5.376477241516113, "rewards/rejected": -6.334069728851318, "step": 410 }, { "epoch": 0.8853333333333333, "grad_norm": 2.528268337249756, "learning_rate": 4.141499133247391e-06, "logits/chosen": -1.6540710926055908, "logits/rejected": -1.5933629274368286, "logps/chosen": -219.1824188232422, "logps/rejected": -326.68304443359375, "loss": 0.162, "rewards/accuracies": 0.9125000238418579, "rewards/chosen": -0.7679177522659302, "rewards/margins": 5.114426136016846, "rewards/rejected": -5.8823442459106445, "step": 415 }, { "epoch": 0.896, "grad_norm": 1.8557026386260986, "learning_rate": 4.119676024643217e-06, "logits/chosen": -1.6958087682724, "logits/rejected": -1.5895674228668213, "logps/chosen": -214.60275268554688, "logps/rejected": -318.4465026855469, "loss": 0.141, "rewards/accuracies": 0.9375, "rewards/chosen": -0.5810152292251587, "rewards/margins": 5.569883346557617, "rewards/rejected": -6.1508989334106445, "step": 420 }, { "epoch": 0.9066666666666666, "grad_norm": 7.853280544281006, "learning_rate": 4.0976381159944055e-06, "logits/chosen": -1.807705283164978, "logits/rejected": -1.5605567693710327, "logps/chosen": -228.89804077148438, "logps/rejected": -328.572265625, "loss": 0.1435, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.6997815370559692, "rewards/margins": 5.620684623718262, "rewards/rejected": -6.320466995239258, "step": 425 }, { "epoch": 0.9173333333333333, "grad_norm": 4.629083633422852, "learning_rate": 4.07538832994949e-06, "logits/chosen": -1.7461702823638916, "logits/rejected": -1.5441920757293701, "logps/chosen": -213.9942169189453, "logps/rejected": -337.7629089355469, "loss": 0.1637, "rewards/accuracies": 0.9375, "rewards/chosen": -0.9062296748161316, "rewards/margins": 5.746966361999512, "rewards/rejected": -6.653195858001709, "step": 430 }, { "epoch": 0.928, "grad_norm": 3.687288522720337, "learning_rate": 4.0529296172560055e-06, "logits/chosen": -1.8524515628814697, "logits/rejected": -1.7046585083007812, "logps/chosen": -207.7529296875, "logps/rejected": -340.30145263671875, "loss": 0.1495, "rewards/accuracies": 0.9375, "rewards/chosen": -0.9373359680175781, "rewards/margins": 5.981744289398193, "rewards/rejected": -6.9190802574157715, "step": 435 }, { "epoch": 0.9386666666666666, "grad_norm": 2.6075193881988525, "learning_rate": 4.030264956369158e-06, "logits/chosen": -1.7657550573349, "logits/rejected": -1.6465190649032593, "logps/chosen": -209.3252410888672, "logps/rejected": -304.9324035644531, "loss": 0.1542, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.7593217492103577, "rewards/margins": 5.0572614669799805, "rewards/rejected": -5.816583156585693, "step": 440 }, { "epoch": 0.9493333333333334, "grad_norm": 4.559110641479492, "learning_rate": 4.0073973530568274e-06, "logits/chosen": -1.7933986186981201, "logits/rejected": -1.600174903869629, "logps/chosen": -224.45346069335938, "logps/rejected": -342.8002624511719, "loss": 0.1297, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -0.7520440220832825, "rewards/margins": 5.706141471862793, "rewards/rejected": -6.458186149597168, "step": 445 }, { "epoch": 0.96, "grad_norm": 2.7471275329589844, "learning_rate": 3.9843298400009435e-06, "logits/chosen": -1.8151620626449585, "logits/rejected": -1.6723363399505615, "logps/chosen": -218.27420043945312, "logps/rejected": -336.54058837890625, "loss": 0.123, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.027143120765686, "rewards/margins": 5.687465667724609, "rewards/rejected": -6.714608669281006, "step": 450 }, { "epoch": 0.9706666666666667, "grad_norm": 1.708066701889038, "learning_rate": 3.961065476395296e-06, "logits/chosen": -1.8115184307098389, "logits/rejected": -1.6580374240875244, "logps/chosen": -241.6630859375, "logps/rejected": -348.1864318847656, "loss": 0.1565, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -0.9067646861076355, "rewards/margins": 5.730084419250488, "rewards/rejected": -6.636848449707031, "step": 455 }, { "epoch": 0.9813333333333333, "grad_norm": 2.5178802013397217, "learning_rate": 3.937607347539823e-06, "logits/chosen": -1.8042964935302734, "logits/rejected": -1.606554627418518, "logps/chosen": -206.523193359375, "logps/rejected": -315.8009338378906, "loss": 0.1346, "rewards/accuracies": 0.9375, "rewards/chosen": -0.7148440480232239, "rewards/margins": 5.358724594116211, "rewards/rejected": -6.073568820953369, "step": 460 }, { "epoch": 0.992, "grad_norm": 0.6924991011619568, "learning_rate": 3.913958564431447e-06, "logits/chosen": -1.8427581787109375, "logits/rejected": -1.6687357425689697, "logps/chosen": -235.98275756835938, "logps/rejected": -346.68865966796875, "loss": 0.1137, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.1400498151779175, "rewards/margins": 6.211422443389893, "rewards/rejected": -7.351471900939941, "step": 465 }, { "epoch": 1.0021333333333333, "grad_norm": 3.3022406101226807, "learning_rate": 3.89012226335149e-06, "logits/chosen": -1.7215778827667236, "logits/rejected": -1.5378063917160034, "logps/chosen": -236.6403350830078, "logps/rejected": -333.3316345214844, "loss": 0.1219, "rewards/accuracies": 0.9407894611358643, "rewards/chosen": -0.9613617062568665, "rewards/margins": 5.4095001220703125, "rewards/rejected": -6.370861053466797, "step": 470 }, { "epoch": 1.0128, "grad_norm": 1.652114987373352, "learning_rate": 3.86610160544975e-06, "logits/chosen": -1.7459675073623657, "logits/rejected": -1.6606613397598267, "logps/chosen": -211.37924194335938, "logps/rejected": -328.5991516113281, "loss": 0.0908, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -0.9254496693611145, "rewards/margins": 6.109896659851074, "rewards/rejected": -7.035346031188965, "step": 475 }, { "epoch": 1.0234666666666667, "grad_norm": 2.3160781860351562, "learning_rate": 3.841899776325267e-06, "logits/chosen": -1.7546608448028564, "logits/rejected": -1.5244556665420532, "logps/chosen": -245.4587860107422, "logps/rejected": -330.42510986328125, "loss": 0.0982, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -0.7176437377929688, "rewards/margins": 5.7568793296813965, "rewards/rejected": -6.474523067474365, "step": 480 }, { "epoch": 1.0341333333333333, "grad_norm": 2.9602835178375244, "learning_rate": 3.817519985603855e-06, "logits/chosen": -1.8549671173095703, "logits/rejected": -1.6893632411956787, "logps/chosen": -205.00753784179688, "logps/rejected": -327.4825744628906, "loss": 0.1123, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.2238247394561768, "rewards/margins": 6.4054765701293945, "rewards/rejected": -7.62930154800415, "step": 485 }, { "epoch": 1.0448, "grad_norm": 1.5233310461044312, "learning_rate": 3.792965466512437e-06, "logits/chosen": -1.7780135869979858, "logits/rejected": -1.5944887399673462, "logps/chosen": -233.1146240234375, "logps/rejected": -335.52093505859375, "loss": 0.1319, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.054016351699829, "rewards/margins": 6.117007255554199, "rewards/rejected": -7.171023368835449, "step": 490 }, { "epoch": 1.0554666666666668, "grad_norm": 2.667370319366455, "learning_rate": 3.7682394754502687e-06, "logits/chosen": -1.7536907196044922, "logits/rejected": -1.6732431650161743, "logps/chosen": -223.99746704101562, "logps/rejected": -346.42034912109375, "loss": 0.0791, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.2943538427352905, "rewards/margins": 6.558332920074463, "rewards/rejected": -7.852685451507568, "step": 495 }, { "epoch": 1.0661333333333334, "grad_norm": 7.733152389526367, "learning_rate": 3.7433452915570685e-06, "logits/chosen": -1.8311758041381836, "logits/rejected": -1.6713851690292358, "logps/chosen": -215.52981567382812, "logps/rejected": -323.02642822265625, "loss": 0.1374, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -1.3178616762161255, "rewards/margins": 6.0462646484375, "rewards/rejected": -7.364126682281494, "step": 500 }, { "epoch": 1.0768, "grad_norm": 1.6280964612960815, "learning_rate": 3.718286216278143e-06, "logits/chosen": -1.7556006908416748, "logits/rejected": -1.5969337224960327, "logps/chosen": -231.25070190429688, "logps/rejected": -342.3904113769531, "loss": 0.1556, "rewards/accuracies": 0.9375, "rewards/chosen": -1.6533854007720947, "rewards/margins": 6.318906307220459, "rewards/rejected": -7.972291469573975, "step": 505 }, { "epoch": 1.0874666666666666, "grad_norm": 4.397036552429199, "learning_rate": 3.6930655729265557e-06, "logits/chosen": -1.8091108798980713, "logits/rejected": -1.620729684829712, "logps/chosen": -224.5769805908203, "logps/rejected": -329.8936462402344, "loss": 0.1047, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.1303974390029907, "rewards/margins": 6.186360836029053, "rewards/rejected": -7.316758155822754, "step": 510 }, { "epoch": 1.0981333333333334, "grad_norm": 1.592452049255371, "learning_rate": 3.667686706242386e-06, "logits/chosen": -1.6756534576416016, "logits/rejected": -1.5711702108383179, "logps/chosen": -240.41464233398438, "logps/rejected": -355.96331787109375, "loss": 0.0936, "rewards/accuracies": 0.96875, "rewards/chosen": -1.9966108798980713, "rewards/margins": 6.202914237976074, "rewards/rejected": -8.199524879455566, "step": 515 }, { "epoch": 1.1088, "grad_norm": 4.626035690307617, "learning_rate": 3.6421529819491576e-06, "logits/chosen": -1.7973684072494507, "logits/rejected": -1.6707580089569092, "logps/chosen": -238.1591033935547, "logps/rejected": -337.50616455078125, "loss": 0.1082, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -2.04890775680542, "rewards/margins": 6.534150123596191, "rewards/rejected": -8.583057403564453, "step": 520 }, { "epoch": 1.1194666666666666, "grad_norm": 1.8899625539779663, "learning_rate": 3.6164677863074783e-06, "logits/chosen": -1.7659199237823486, "logits/rejected": -1.6688506603240967, "logps/chosen": -217.8213653564453, "logps/rejected": -375.4813537597656, "loss": 0.0715, "rewards/accuracies": 0.96875, "rewards/chosen": -1.9252666234970093, "rewards/margins": 7.536079406738281, "rewards/rejected": -9.461346626281738, "step": 525 }, { "epoch": 1.1301333333333332, "grad_norm": 3.0119969844818115, "learning_rate": 3.590634525665955e-06, "logits/chosen": -1.7619727849960327, "logits/rejected": -1.6695072650909424, "logps/chosen": -217.845703125, "logps/rejected": -355.73760986328125, "loss": 0.1189, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.835972547531128, "rewards/margins": 6.8431878089904785, "rewards/rejected": -8.679161071777344, "step": 530 }, { "epoch": 1.1408, "grad_norm": 3.917595148086548, "learning_rate": 3.564656626009446e-06, "logits/chosen": -1.8848800659179688, "logits/rejected": -1.6976979970932007, "logps/chosen": -215.7821807861328, "logps/rejected": -328.48944091796875, "loss": 0.1398, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.798927903175354, "rewards/margins": 6.75002908706665, "rewards/rejected": -8.548957824707031, "step": 535 }, { "epoch": 1.1514666666666666, "grad_norm": 0.4280731976032257, "learning_rate": 3.5385375325047167e-06, "logits/chosen": -1.8062047958374023, "logits/rejected": -1.6663563251495361, "logps/chosen": -228.35617065429688, "logps/rejected": -342.6147155761719, "loss": 0.0681, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.6262073516845703, "rewards/margins": 6.9966301918029785, "rewards/rejected": -8.62283706665039, "step": 540 }, { "epoch": 1.1621333333333332, "grad_norm": 4.61460542678833, "learning_rate": 3.512280709043534e-06, "logits/chosen": -1.7144701480865479, "logits/rejected": -1.6076030731201172, "logps/chosen": -225.5585174560547, "logps/rejected": -331.7484130859375, "loss": 0.1372, "rewards/accuracies": 0.9375, "rewards/chosen": -1.4640800952911377, "rewards/margins": 6.167746067047119, "rewards/rejected": -7.631825923919678, "step": 545 }, { "epoch": 1.1728, "grad_norm": 1.0511789321899414, "learning_rate": 3.4858896377832966e-06, "logits/chosen": -1.7985868453979492, "logits/rejected": -1.7188758850097656, "logps/chosen": -213.89755249023438, "logps/rejected": -339.4588928222656, "loss": 0.0928, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.4908698797225952, "rewards/margins": 6.6824951171875, "rewards/rejected": -8.173364639282227, "step": 550 }, { "epoch": 1.1834666666666667, "grad_norm": 3.18619441986084, "learning_rate": 3.459367818685231e-06, "logits/chosen": -1.767228364944458, "logits/rejected": -1.650066614151001, "logps/chosen": -237.79940795898438, "logps/rejected": -361.7818908691406, "loss": 0.1198, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.48301100730896, "rewards/margins": 6.895485877990723, "rewards/rejected": -8.378497123718262, "step": 555 }, { "epoch": 1.1941333333333333, "grad_norm": 1.3614966869354248, "learning_rate": 3.432718769050228e-06, "logits/chosen": -1.614767074584961, "logits/rejected": -1.5364725589752197, "logps/chosen": -227.3282470703125, "logps/rejected": -329.19873046875, "loss": 0.1329, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.5325191020965576, "rewards/margins": 6.505125522613525, "rewards/rejected": -8.03764533996582, "step": 560 }, { "epoch": 1.2048, "grad_norm": 4.426132678985596, "learning_rate": 3.405946023052386e-06, "logits/chosen": -1.8132712841033936, "logits/rejected": -1.7653039693832397, "logps/chosen": -215.24392700195312, "logps/rejected": -344.3136901855469, "loss": 0.1388, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.5404421091079712, "rewards/margins": 6.6653618812561035, "rewards/rejected": -8.205804824829102, "step": 565 }, { "epoch": 1.2154666666666667, "grad_norm": 3.402472496032715, "learning_rate": 3.3790531312703055e-06, "logits/chosen": -1.7687180042266846, "logits/rejected": -1.6483733654022217, "logps/chosen": -212.26132202148438, "logps/rejected": -352.9338684082031, "loss": 0.0937, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.4449779987335205, "rewards/margins": 6.969573974609375, "rewards/rejected": -8.414551734924316, "step": 570 }, { "epoch": 1.2261333333333333, "grad_norm": 2.1326146125793457, "learning_rate": 3.35204366021622e-06, "logits/chosen": -1.6894903182983398, "logits/rejected": -1.5975654125213623, "logps/chosen": -215.57028198242188, "logps/rejected": -344.5772399902344, "loss": 0.1223, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.0661876201629639, "rewards/margins": 6.815348148345947, "rewards/rejected": -7.881535530090332, "step": 575 }, { "epoch": 1.2368000000000001, "grad_norm": 1.604225993156433, "learning_rate": 3.3249211918630055e-06, "logits/chosen": -1.7422653436660767, "logits/rejected": -1.6558812856674194, "logps/chosen": -211.7191162109375, "logps/rejected": -320.58526611328125, "loss": 0.1275, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.1895016431808472, "rewards/margins": 6.5496721267700195, "rewards/rejected": -7.739173889160156, "step": 580 }, { "epoch": 1.2474666666666667, "grad_norm": 2.2764594554901123, "learning_rate": 3.2976893231691424e-06, "logits/chosen": -1.7650436162948608, "logits/rejected": -1.6200859546661377, "logps/chosen": -233.5091552734375, "logps/rejected": -338.52587890625, "loss": 0.0806, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -0.9783912897109985, "rewards/margins": 7.381894111633301, "rewards/rejected": -8.360284805297852, "step": 585 }, { "epoch": 1.2581333333333333, "grad_norm": 4.1610636711120605, "learning_rate": 3.270351665601691e-06, "logits/chosen": -1.7916901111602783, "logits/rejected": -1.7062695026397705, "logps/chosen": -219.70193481445312, "logps/rejected": -349.7830810546875, "loss": 0.1074, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.0305535793304443, "rewards/margins": 6.8783745765686035, "rewards/rejected": -7.908926963806152, "step": 590 }, { "epoch": 1.2688, "grad_norm": 1.4349570274353027, "learning_rate": 3.2429118446573417e-06, "logits/chosen": -1.75607430934906, "logits/rejected": -1.6087610721588135, "logps/chosen": -225.0104522705078, "logps/rejected": -354.34979248046875, "loss": 0.0973, "rewards/accuracies": 0.96875, "rewards/chosen": -1.2995376586914062, "rewards/margins": 6.823355197906494, "rewards/rejected": -8.122893333435059, "step": 595 }, { "epoch": 1.2794666666666665, "grad_norm": 5.035015106201172, "learning_rate": 3.215373499381602e-06, "logits/chosen": -1.6800158023834229, "logits/rejected": -1.512091875076294, "logps/chosen": -239.14111328125, "logps/rejected": -337.6920166015625, "loss": 0.1018, "rewards/accuracies": 0.96875, "rewards/chosen": -1.6130025386810303, "rewards/margins": 6.840310096740723, "rewards/rejected": -8.453312873840332, "step": 600 }, { "epoch": 1.2901333333333334, "grad_norm": 1.7663006782531738, "learning_rate": 3.1877402818861954e-06, "logits/chosen": -1.7800567150115967, "logits/rejected": -1.710240364074707, "logps/chosen": -242.6155242919922, "logps/rejected": -339.5917663574219, "loss": 0.092, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.8466907739639282, "rewards/margins": 6.454521179199219, "rewards/rejected": -8.3012113571167, "step": 605 }, { "epoch": 1.3008, "grad_norm": 5.838703632354736, "learning_rate": 3.1600158568647145e-06, "logits/chosen": -1.7841325998306274, "logits/rejected": -1.6554603576660156, "logps/chosen": -235.1628875732422, "logps/rejected": -345.6203918457031, "loss": 0.1131, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.5866191387176514, "rewards/margins": 7.077944755554199, "rewards/rejected": -8.66456413269043, "step": 610 }, { "epoch": 1.3114666666666666, "grad_norm": 2.592312812805176, "learning_rate": 3.1322039011066172e-06, "logits/chosen": -1.7783803939819336, "logits/rejected": -1.6592767238616943, "logps/chosen": -243.6439208984375, "logps/rejected": -343.04833984375, "loss": 0.1224, "rewards/accuracies": 0.96875, "rewards/chosen": -1.3051066398620605, "rewards/margins": 6.644067287445068, "rewards/rejected": -7.949173927307129, "step": 615 }, { "epoch": 1.3221333333333334, "grad_norm": 1.8750396966934204, "learning_rate": 3.1043081030096125e-06, "logits/chosen": -1.6340547800064087, "logits/rejected": -1.5600389242172241, "logps/chosen": -222.5189971923828, "logps/rejected": -371.560302734375, "loss": 0.0874, "rewards/accuracies": 0.96875, "rewards/chosen": -1.6143547296524048, "rewards/margins": 7.084819793701172, "rewards/rejected": -8.699174880981445, "step": 620 }, { "epoch": 1.3328, "grad_norm": 5.306309223175049, "learning_rate": 3.0763321620905072e-06, "logits/chosen": -1.7420177459716797, "logits/rejected": -1.5960488319396973, "logps/chosen": -222.31021118164062, "logps/rejected": -338.64886474609375, "loss": 0.1359, "rewards/accuracies": 0.96875, "rewards/chosen": -1.4612404108047485, "rewards/margins": 7.118841648101807, "rewards/rejected": -8.58008098602295, "step": 625 }, { "epoch": 1.3434666666666666, "grad_norm": 3.0806684494018555, "learning_rate": 3.048279788494583e-06, "logits/chosen": -1.7452303171157837, "logits/rejected": -1.6031157970428467, "logps/chosen": -227.9959716796875, "logps/rejected": -320.9499206542969, "loss": 0.1244, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.5490151643753052, "rewards/margins": 6.89993143081665, "rewards/rejected": -8.448946952819824, "step": 630 }, { "epoch": 1.3541333333333334, "grad_norm": 2.2955572605133057, "learning_rate": 3.0201547025035553e-06, "logits/chosen": -1.7628275156021118, "logits/rejected": -1.6372320652008057, "logps/chosen": -230.3760528564453, "logps/rejected": -343.8782653808594, "loss": 0.1286, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.40870201587677, "rewards/margins": 7.246272087097168, "rewards/rejected": -8.654973983764648, "step": 635 }, { "epoch": 1.3648, "grad_norm": 2.1225922107696533, "learning_rate": 2.9919606340421963e-06, "logits/chosen": -1.7855663299560547, "logits/rejected": -1.5720247030258179, "logps/chosen": -227.51504516601562, "logps/rejected": -342.0578308105469, "loss": 0.1126, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.6880676746368408, "rewards/margins": 6.570213317871094, "rewards/rejected": -8.258280754089355, "step": 640 }, { "epoch": 1.3754666666666666, "grad_norm": 2.0824544429779053, "learning_rate": 2.9637013221836723e-06, "logits/chosen": -1.7979755401611328, "logits/rejected": -1.6156142950057983, "logps/chosen": -235.97024536132812, "logps/rejected": -367.8596496582031, "loss": 0.0867, "rewards/accuracies": 0.96875, "rewards/chosen": -1.5493922233581543, "rewards/margins": 7.716729640960693, "rewards/rejected": -9.266121864318848, "step": 645 }, { "epoch": 1.3861333333333334, "grad_norm": 3.655622959136963, "learning_rate": 2.9353805146536762e-06, "logits/chosen": -1.6617200374603271, "logits/rejected": -1.6281896829605103, "logps/chosen": -220.10995483398438, "logps/rejected": -348.8759765625, "loss": 0.0983, "rewards/accuracies": 0.96875, "rewards/chosen": -1.618729829788208, "rewards/margins": 6.909787654876709, "rewards/rejected": -8.52851676940918, "step": 650 }, { "epoch": 1.3968, "grad_norm": 4.440053939819336, "learning_rate": 2.907001967333401e-06, "logits/chosen": -1.8161550760269165, "logits/rejected": -1.7380571365356445, "logps/chosen": -221.9119110107422, "logps/rejected": -358.5216064453125, "loss": 0.1167, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.8026584386825562, "rewards/margins": 7.0366363525390625, "rewards/rejected": -8.83929443359375, "step": 655 }, { "epoch": 1.4074666666666666, "grad_norm": 1.1048275232315063, "learning_rate": 2.878569443761442e-06, "logits/chosen": -1.7552165985107422, "logits/rejected": -1.6000335216522217, "logps/chosen": -209.29592895507812, "logps/rejected": -341.8609619140625, "loss": 0.0867, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.1091715097427368, "rewards/margins": 7.496683597564697, "rewards/rejected": -8.605854988098145, "step": 660 }, { "epoch": 1.4181333333333335, "grad_norm": 3.342843532562256, "learning_rate": 2.85008671463468e-06, "logits/chosen": -1.7816989421844482, "logits/rejected": -1.6939798593521118, "logps/chosen": -205.90310668945312, "logps/rejected": -349.85711669921875, "loss": 0.126, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -1.2616592645645142, "rewards/margins": 7.516135215759277, "rewards/rejected": -8.777795791625977, "step": 665 }, { "epoch": 1.4288, "grad_norm": 4.1771697998046875, "learning_rate": 2.8215575573082127e-06, "logits/chosen": -1.7190008163452148, "logits/rejected": -1.6227375268936157, "logps/chosen": -230.20742797851562, "logps/rejected": -360.1850280761719, "loss": 0.0662, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.6839618682861328, "rewards/margins": 7.761228084564209, "rewards/rejected": -9.445191383361816, "step": 670 }, { "epoch": 1.4394666666666667, "grad_norm": 2.271709442138672, "learning_rate": 2.792985755294404e-06, "logits/chosen": -1.8138444423675537, "logits/rejected": -1.6258472204208374, "logps/chosen": -222.8135986328125, "logps/rejected": -354.85980224609375, "loss": 0.0697, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.8773260116577148, "rewards/margins": 7.912613868713379, "rewards/rejected": -9.789939880371094, "step": 675 }, { "epoch": 1.4501333333333333, "grad_norm": 1.3745611906051636, "learning_rate": 2.764375097761122e-06, "logits/chosen": -1.6466379165649414, "logits/rejected": -1.5347976684570312, "logps/chosen": -214.34341430664062, "logps/rejected": -339.7757873535156, "loss": 0.0986, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.489703893661499, "rewards/margins": 7.611210823059082, "rewards/rejected": -9.10091495513916, "step": 680 }, { "epoch": 1.4607999999999999, "grad_norm": 3.7492542266845703, "learning_rate": 2.73572937902922e-06, "logits/chosen": -1.737330436706543, "logits/rejected": -1.6359955072402954, "logps/chosen": -250.636474609375, "logps/rejected": -355.47625732421875, "loss": 0.125, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.7210981845855713, "rewards/margins": 6.998046875, "rewards/rejected": -8.719144821166992, "step": 685 }, { "epoch": 1.4714666666666667, "grad_norm": 3.4588992595672607, "learning_rate": 2.7070523980693362e-06, "logits/chosen": -1.556281328201294, "logits/rejected": -1.4768974781036377, "logps/chosen": -238.09829711914062, "logps/rejected": -339.5543518066406, "loss": 0.1122, "rewards/accuracies": 0.96875, "rewards/chosen": -1.520418405532837, "rewards/margins": 7.163352966308594, "rewards/rejected": -8.683772087097168, "step": 690 }, { "epoch": 1.4821333333333333, "grad_norm": 1.5479131937026978, "learning_rate": 2.678347957998081e-06, "logits/chosen": -1.7872568368911743, "logits/rejected": -1.61525559425354, "logps/chosen": -230.7522735595703, "logps/rejected": -361.02545166015625, "loss": 0.0637, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.6853501796722412, "rewards/margins": 8.169617652893066, "rewards/rejected": -9.85496711730957, "step": 695 }, { "epoch": 1.4928, "grad_norm": 1.234351396560669, "learning_rate": 2.649619865573666e-06, "logits/chosen": -1.7372156381607056, "logits/rejected": -1.6385151147842407, "logps/chosen": -227.90835571289062, "logps/rejected": -367.4066467285156, "loss": 0.0767, "rewards/accuracies": 0.96875, "rewards/chosen": -1.9077075719833374, "rewards/margins": 7.612782955169678, "rewards/rejected": -9.520490646362305, "step": 700 }, { "epoch": 1.5034666666666667, "grad_norm": 2.3628101348876953, "learning_rate": 2.620871930691058e-06, "logits/chosen": -1.7833808660507202, "logits/rejected": -1.695267915725708, "logps/chosen": -227.47909545898438, "logps/rejected": -361.0234069824219, "loss": 0.0975, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.7123091220855713, "rewards/margins": 8.293615341186523, "rewards/rejected": -10.005925178527832, "step": 705 }, { "epoch": 1.5141333333333333, "grad_norm": 4.18447732925415, "learning_rate": 2.5921079658767152e-06, "logits/chosen": -1.7901111841201782, "logits/rejected": -1.669463872909546, "logps/chosen": -228.15188598632812, "logps/rejected": -352.96795654296875, "loss": 0.2078, "rewards/accuracies": 0.9437500238418579, "rewards/chosen": -1.9093815088272095, "rewards/margins": 7.494314670562744, "rewards/rejected": -9.403696060180664, "step": 710 }, { "epoch": 1.5248, "grad_norm": 2.716040849685669, "learning_rate": 2.56333178578297e-06, "logits/chosen": -1.76180100440979, "logits/rejected": -1.636069655418396, "logps/chosen": -261.0120544433594, "logps/rejected": -376.29010009765625, "loss": 0.078, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.8959649801254272, "rewards/margins": 7.93648624420166, "rewards/rejected": -9.832451820373535, "step": 715 }, { "epoch": 1.5354666666666668, "grad_norm": 4.260155200958252, "learning_rate": 2.534547206682136e-06, "logits/chosen": -1.8289260864257812, "logits/rejected": -1.6811158657073975, "logps/chosen": -225.0589599609375, "logps/rejected": -378.1023254394531, "loss": 0.0619, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.8060722351074219, "rewards/margins": 8.63095474243164, "rewards/rejected": -10.437026977539062, "step": 720 }, { "epoch": 1.5461333333333334, "grad_norm": 2.0456085205078125, "learning_rate": 2.5057580459603947e-06, "logits/chosen": -1.8401775360107422, "logits/rejected": -1.7199382781982422, "logps/chosen": -222.3778533935547, "logps/rejected": -359.7654724121094, "loss": 0.0655, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.9060947895050049, "rewards/margins": 7.709580421447754, "rewards/rejected": -9.615674018859863, "step": 725 }, { "epoch": 1.5568, "grad_norm": 3.1641626358032227, "learning_rate": 2.4769681216115384e-06, "logits/chosen": -1.7817308902740479, "logits/rejected": -1.650779128074646, "logps/chosen": -219.2471466064453, "logps/rejected": -357.71612548828125, "loss": 0.0896, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.5718821287155151, "rewards/margins": 8.106928825378418, "rewards/rejected": -9.678812026977539, "step": 730 }, { "epoch": 1.5674666666666668, "grad_norm": 3.828000545501709, "learning_rate": 2.448181251730632e-06, "logits/chosen": -1.7795158624649048, "logits/rejected": -1.5841357707977295, "logps/chosen": -220.0762939453125, "logps/rejected": -342.20758056640625, "loss": 0.1752, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.8566948175430298, "rewards/margins": 6.9646759033203125, "rewards/rejected": -8.821371078491211, "step": 735 }, { "epoch": 1.5781333333333334, "grad_norm": 6.140118598937988, "learning_rate": 2.419401254007658e-06, "logits/chosen": -1.6956440210342407, "logits/rejected": -1.5929274559020996, "logps/chosen": -247.3740997314453, "logps/rejected": -374.37677001953125, "loss": 0.1017, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.1039836406707764, "rewards/margins": 7.365347385406494, "rewards/rejected": -9.469330787658691, "step": 740 }, { "epoch": 1.5888, "grad_norm": 1.6513922214508057, "learning_rate": 2.390631945221221e-06, "logits/chosen": -1.8118445873260498, "logits/rejected": -1.636195182800293, "logps/chosen": -215.1251678466797, "logps/rejected": -344.25689697265625, "loss": 0.0714, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.4001673460006714, "rewards/margins": 8.314090728759766, "rewards/rejected": -9.714258193969727, "step": 745 }, { "epoch": 1.5994666666666668, "grad_norm": 5.196778297424316, "learning_rate": 2.361877140732366e-06, "logits/chosen": -1.842757225036621, "logits/rejected": -1.7138402462005615, "logps/chosen": -217.3812255859375, "logps/rejected": -351.89825439453125, "loss": 0.0958, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.4169294834136963, "rewards/margins": 7.893444061279297, "rewards/rejected": -9.310373306274414, "step": 750 }, { "epoch": 1.6101333333333332, "grad_norm": 3.4266786575317383, "learning_rate": 2.333140653978592e-06, "logits/chosen": -1.7201732397079468, "logits/rejected": -1.6220394372940063, "logps/chosen": -218.4654083251953, "logps/rejected": -345.9354248046875, "loss": 0.1068, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.8092172145843506, "rewards/margins": 6.817082405090332, "rewards/rejected": -8.626298904418945, "step": 755 }, { "epoch": 1.6208, "grad_norm": 2.9891090393066406, "learning_rate": 2.3044262959681156e-06, "logits/chosen": -1.7339553833007812, "logits/rejected": -1.6223846673965454, "logps/chosen": -207.73974609375, "logps/rejected": -361.72967529296875, "loss": 0.098, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.3607968091964722, "rewards/margins": 7.7453179359436035, "rewards/rejected": -9.106114387512207, "step": 760 }, { "epoch": 1.6314666666666666, "grad_norm": 5.372440814971924, "learning_rate": 2.2757378747744575e-06, "logits/chosen": -1.8800748586654663, "logits/rejected": -1.6330455541610718, "logps/chosen": -214.36270141601562, "logps/rejected": -335.51654052734375, "loss": 0.0943, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.389737844467163, "rewards/margins": 7.517597198486328, "rewards/rejected": -8.90733528137207, "step": 765 }, { "epoch": 1.6421333333333332, "grad_norm": 4.0247087478637695, "learning_rate": 2.24707919503142e-06, "logits/chosen": -1.8723796606063843, "logits/rejected": -1.7164814472198486, "logps/chosen": -234.10498046875, "logps/rejected": -356.8775634765625, "loss": 0.1229, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.514541506767273, "rewards/margins": 7.490867614746094, "rewards/rejected": -9.00540828704834, "step": 770 }, { "epoch": 1.6528, "grad_norm": 2.84824538230896, "learning_rate": 2.2184540574285213e-06, "logits/chosen": -1.681469202041626, "logits/rejected": -1.5739465951919556, "logps/chosen": -256.971435546875, "logps/rejected": -337.6930236816406, "loss": 0.1087, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.6835079193115234, "rewards/margins": 6.543093204498291, "rewards/rejected": -8.226600646972656, "step": 775 }, { "epoch": 1.6634666666666666, "grad_norm": 4.553780555725098, "learning_rate": 2.1898662582069486e-06, "logits/chosen": -1.7528133392333984, "logits/rejected": -1.6476829051971436, "logps/chosen": -220.4532470703125, "logps/rejected": -357.43438720703125, "loss": 0.1319, "rewards/accuracies": 0.9375, "rewards/chosen": -1.636549949645996, "rewards/margins": 7.442874908447266, "rewards/rejected": -9.079424858093262, "step": 780 }, { "epoch": 1.6741333333333333, "grad_norm": 1.5032289028167725, "learning_rate": 2.1613195886561044e-06, "logits/chosen": -1.7787612676620483, "logits/rejected": -1.6489241123199463, "logps/chosen": -207.38650512695312, "logps/rejected": -330.9730224609375, "loss": 0.1168, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.3346378803253174, "rewards/margins": 7.285429954528809, "rewards/rejected": -8.620067596435547, "step": 785 }, { "epoch": 1.6848, "grad_norm": 6.126774311065674, "learning_rate": 2.132817834610811e-06, "logits/chosen": -1.711124062538147, "logits/rejected": -1.619384527206421, "logps/chosen": -229.5834503173828, "logps/rejected": -359.8412170410156, "loss": 0.1126, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.6115143299102783, "rewards/margins": 6.954829216003418, "rewards/rejected": -8.566343307495117, "step": 790 }, { "epoch": 1.6954666666666667, "grad_norm": 2.0657472610473633, "learning_rate": 2.1043647759492338e-06, "logits/chosen": -1.6726573705673218, "logits/rejected": -1.5993225574493408, "logps/chosen": -223.4843292236328, "logps/rejected": -328.24700927734375, "loss": 0.0887, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -0.9438800811767578, "rewards/margins": 7.049546241760254, "rewards/rejected": -7.993424892425537, "step": 795 }, { "epoch": 1.7061333333333333, "grad_norm": 2.5707919597625732, "learning_rate": 2.075964186091599e-06, "logits/chosen": -1.8048022985458374, "logits/rejected": -1.7015832662582397, "logps/chosen": -216.2055206298828, "logps/rejected": -342.51287841796875, "loss": 0.1203, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.4331550598144531, "rewards/margins": 7.590814113616943, "rewards/rejected": -9.023968696594238, "step": 800 }, { "epoch": 1.7168, "grad_norm": 2.875871419906616, "learning_rate": 2.047619831499765e-06, "logits/chosen": -1.6987117528915405, "logits/rejected": -1.5893299579620361, "logps/chosen": -219.29531860351562, "logps/rejected": -340.4369812011719, "loss": 0.11, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.1939778327941895, "rewards/margins": 6.53455114364624, "rewards/rejected": -7.728528022766113, "step": 805 }, { "epoch": 1.7274666666666667, "grad_norm": 1.0247174501419067, "learning_rate": 2.0193354711777155e-06, "logits/chosen": -1.8061132431030273, "logits/rejected": -1.6443469524383545, "logps/chosen": -244.94760131835938, "logps/rejected": -349.5795593261719, "loss": 0.0906, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.3361376523971558, "rewards/margins": 6.910301208496094, "rewards/rejected": -8.246439933776855, "step": 810 }, { "epoch": 1.7381333333333333, "grad_norm": 0.9948698282241821, "learning_rate": 1.991114856173048e-06, "logits/chosen": -1.8769338130950928, "logits/rejected": -1.70797598361969, "logps/chosen": -210.803466796875, "logps/rejected": -338.79327392578125, "loss": 0.0605, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.4188705682754517, "rewards/margins": 7.125912666320801, "rewards/rejected": -8.544782638549805, "step": 815 }, { "epoch": 1.7488000000000001, "grad_norm": 1.602662444114685, "learning_rate": 1.9629617290795115e-06, "logits/chosen": -1.7712066173553467, "logits/rejected": -1.6187121868133545, "logps/chosen": -215.338623046875, "logps/rejected": -341.73583984375, "loss": 0.0744, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.4782660007476807, "rewards/margins": 7.113155364990234, "rewards/rejected": -8.591421127319336, "step": 820 }, { "epoch": 1.7594666666666665, "grad_norm": 2.919387102127075, "learning_rate": 1.934879823540663e-06, "logits/chosen": -1.8011865615844727, "logits/rejected": -1.6715080738067627, "logps/chosen": -219.78775024414062, "logps/rejected": -354.23187255859375, "loss": 0.1081, "rewards/accuracies": 0.949999988079071, "rewards/chosen": -1.5675954818725586, "rewards/margins": 6.86489725112915, "rewards/rejected": -8.432493209838867, "step": 825 }, { "epoch": 1.7701333333333333, "grad_norm": 2.401257276535034, "learning_rate": 1.9068728637547195e-06, "logits/chosen": -1.76266348361969, "logits/rejected": -1.594229817390442, "logps/chosen": -234.4385223388672, "logps/rejected": -358.3901672363281, "loss": 0.0714, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.7133439779281616, "rewards/margins": 7.900755882263184, "rewards/rejected": -9.614099502563477, "step": 830 }, { "epoch": 1.7808000000000002, "grad_norm": 1.2106441259384155, "learning_rate": 1.878944563980658e-06, "logits/chosen": -1.7576103210449219, "logits/rejected": -1.557202935218811, "logps/chosen": -218.30282592773438, "logps/rejected": -352.62884521484375, "loss": 0.0765, "rewards/accuracies": 0.96875, "rewards/chosen": -1.431818962097168, "rewards/margins": 7.726027488708496, "rewards/rejected": -9.157846450805664, "step": 835 }, { "epoch": 1.7914666666666665, "grad_norm": 3.5941357612609863, "learning_rate": 1.85109862804563e-06, "logits/chosen": -1.754690408706665, "logits/rejected": -1.6672760248184204, "logps/chosen": -225.74581909179688, "logps/rejected": -354.9847717285156, "loss": 0.1013, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.6395050287246704, "rewards/margins": 7.6984686851501465, "rewards/rejected": -9.337973594665527, "step": 840 }, { "epoch": 1.8021333333333334, "grad_norm": 2.2285900115966797, "learning_rate": 1.823338748853766e-06, "logits/chosen": -1.7495044469833374, "logits/rejected": -1.7088682651519775, "logps/chosen": -218.03775024414062, "logps/rejected": -378.6893615722656, "loss": 0.089, "rewards/accuracies": 0.96875, "rewards/chosen": -1.9602687358856201, "rewards/margins": 8.054302215576172, "rewards/rejected": -10.014570236206055, "step": 845 }, { "epoch": 1.8128, "grad_norm": 3.8862228393554688, "learning_rate": 1.7956686078964257e-06, "logits/chosen": -1.7476335763931274, "logits/rejected": -1.646803855895996, "logps/chosen": -248.8168487548828, "logps/rejected": -350.89288330078125, "loss": 0.0824, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.6752967834472656, "rewards/margins": 7.301435947418213, "rewards/rejected": -8.97673225402832, "step": 850 }, { "epoch": 1.8234666666666666, "grad_norm": 3.172503709793091, "learning_rate": 1.7680918747639608e-06, "logits/chosen": -1.8387765884399414, "logits/rejected": -1.7114694118499756, "logps/chosen": -238.186279296875, "logps/rejected": -367.3875732421875, "loss": 0.0701, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.6962471008300781, "rewards/margins": 8.61371898651123, "rewards/rejected": -10.309966087341309, "step": 855 }, { "epoch": 1.8341333333333334, "grad_norm": 4.591922283172607, "learning_rate": 1.740612206659057e-06, "logits/chosen": -1.7826566696166992, "logits/rejected": -1.6185051202774048, "logps/chosen": -219.8805389404297, "logps/rejected": -355.91619873046875, "loss": 0.0744, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.4452508687973022, "rewards/margins": 7.93267297744751, "rewards/rejected": -9.377923965454102, "step": 860 }, { "epoch": 1.8448, "grad_norm": 3.7183308601379395, "learning_rate": 1.7132332479117204e-06, "logits/chosen": -1.8160545825958252, "logits/rejected": -1.6184918880462646, "logps/chosen": -244.14889526367188, "logps/rejected": -360.9537658691406, "loss": 0.0633, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.8190085887908936, "rewards/margins": 7.7830305099487305, "rewards/rejected": -9.602039337158203, "step": 865 }, { "epoch": 1.8554666666666666, "grad_norm": 1.1061958074569702, "learning_rate": 1.6859586294959666e-06, "logits/chosen": -1.7806711196899414, "logits/rejected": -1.6613991260528564, "logps/chosen": -222.4357452392578, "logps/rejected": -371.41986083984375, "loss": 0.0842, "rewards/accuracies": 0.96875, "rewards/chosen": -1.6314767599105835, "rewards/margins": 8.27884292602539, "rewards/rejected": -9.910319328308105, "step": 870 }, { "epoch": 1.8661333333333334, "grad_norm": 3.018303394317627, "learning_rate": 1.6587919685482884e-06, "logits/chosen": -1.7951326370239258, "logits/rejected": -1.6616239547729492, "logps/chosen": -203.40164184570312, "logps/rejected": -358.426025390625, "loss": 0.0797, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.7556968927383423, "rewards/margins": 7.818473815917969, "rewards/rejected": -9.57417106628418, "step": 875 }, { "epoch": 1.8768, "grad_norm": 1.363223671913147, "learning_rate": 1.6317368678879497e-06, "logits/chosen": -1.651288390159607, "logits/rejected": -1.5710906982421875, "logps/chosen": -240.0145721435547, "logps/rejected": -357.17718505859375, "loss": 0.0928, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.6783626079559326, "rewards/margins": 7.36400842666626, "rewards/rejected": -9.042369842529297, "step": 880 }, { "epoch": 1.8874666666666666, "grad_norm": 1.239745020866394, "learning_rate": 1.6047969155391856e-06, "logits/chosen": -1.6190026998519897, "logits/rejected": -1.5436100959777832, "logps/chosen": -246.00247192382812, "logps/rejected": -362.49517822265625, "loss": 0.0564, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.6668789386749268, "rewards/margins": 7.92090368270874, "rewards/rejected": -9.58778190612793, "step": 885 }, { "epoch": 1.8981333333333335, "grad_norm": 1.8271006345748901, "learning_rate": 1.5779756842553617e-06, "logits/chosen": -1.6952009201049805, "logits/rejected": -1.6505413055419922, "logps/chosen": -236.0554656982422, "logps/rejected": -362.10986328125, "loss": 0.0509, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.7946186065673828, "rewards/margins": 7.933407783508301, "rewards/rejected": -9.728026390075684, "step": 890 }, { "epoch": 1.9088, "grad_norm": 1.3855539560317993, "learning_rate": 1.551276731045157e-06, "logits/chosen": -1.7061758041381836, "logits/rejected": -1.6307599544525146, "logps/chosen": -242.96566772460938, "logps/rejected": -367.9177551269531, "loss": 0.0802, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.6677072048187256, "rewards/margins": 7.688159942626953, "rewards/rejected": -9.355867385864258, "step": 895 }, { "epoch": 1.9194666666666667, "grad_norm": 6.346963882446289, "learning_rate": 1.5247035967008369e-06, "logits/chosen": -1.6322176456451416, "logits/rejected": -1.5739306211471558, "logps/chosen": -236.05166625976562, "logps/rejected": -348.62255859375, "loss": 0.1068, "rewards/accuracies": 0.96875, "rewards/chosen": -1.8375927209854126, "rewards/margins": 7.27393102645874, "rewards/rejected": -9.111523628234863, "step": 900 }, { "epoch": 1.9301333333333335, "grad_norm": 2.1471517086029053, "learning_rate": 1.498259805328678e-06, "logits/chosen": -1.801081657409668, "logits/rejected": -1.564719796180725, "logps/chosen": -227.64669799804688, "logps/rejected": -353.7789306640625, "loss": 0.0755, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.8626874685287476, "rewards/margins": 8.565693855285645, "rewards/rejected": -10.428380966186523, "step": 905 }, { "epoch": 1.9407999999999999, "grad_norm": 1.3048449754714966, "learning_rate": 1.471948863881601e-06, "logits/chosen": -1.810206651687622, "logits/rejected": -1.6128194332122803, "logps/chosen": -240.61782836914062, "logps/rejected": -377.509033203125, "loss": 0.0568, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.7367178201675415, "rewards/margins": 8.280123710632324, "rewards/rejected": -10.016841888427734, "step": 910 }, { "epoch": 1.9514666666666667, "grad_norm": 1.4426000118255615, "learning_rate": 1.4457742616940864e-06, "logits/chosen": -1.7471405267715454, "logits/rejected": -1.6417961120605469, "logps/chosen": -225.4718780517578, "logps/rejected": -361.42926025390625, "loss": 0.0985, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -1.6348329782485962, "rewards/margins": 7.608382225036621, "rewards/rejected": -9.243215560913086, "step": 915 }, { "epoch": 1.9621333333333333, "grad_norm": 5.110790729522705, "learning_rate": 1.4197394700194165e-06, "logits/chosen": -1.7448272705078125, "logits/rejected": -1.683083176612854, "logps/chosen": -227.5116729736328, "logps/rejected": -361.52020263671875, "loss": 0.0927, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.834454894065857, "rewards/margins": 7.576612949371338, "rewards/rejected": -9.411067008972168, "step": 920 }, { "epoch": 1.9727999999999999, "grad_norm": 4.627553462982178, "learning_rate": 1.3938479415693244e-06, "logits/chosen": -1.8246326446533203, "logits/rejected": -1.7019975185394287, "logps/chosen": -235.0475616455078, "logps/rejected": -341.9778747558594, "loss": 0.0992, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.870369553565979, "rewards/margins": 7.382562160491943, "rewards/rejected": -9.25293254852295, "step": 925 }, { "epoch": 1.9834666666666667, "grad_norm": 2.9364688396453857, "learning_rate": 1.368103110056096e-06, "logits/chosen": -1.7265510559082031, "logits/rejected": -1.5709378719329834, "logps/chosen": -245.30679321289062, "logps/rejected": -353.5377197265625, "loss": 0.0898, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -2.1614131927490234, "rewards/margins": 7.4779791831970215, "rewards/rejected": -9.639391899108887, "step": 930 }, { "epoch": 1.9941333333333333, "grad_norm": 4.901484489440918, "learning_rate": 1.3425083897371983e-06, "logits/chosen": -1.6642370223999023, "logits/rejected": -1.5926437377929688, "logps/chosen": -242.5086212158203, "logps/rejected": -364.00006103515625, "loss": 0.0977, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.8294131755828857, "rewards/margins": 7.299746513366699, "rewards/rejected": -9.12916088104248, "step": 935 }, { "epoch": 2.0042666666666666, "grad_norm": 3.4245495796203613, "learning_rate": 1.317067174962478e-06, "logits/chosen": -1.6907618045806885, "logits/rejected": -1.559260368347168, "logps/chosen": -224.0897979736328, "logps/rejected": -390.0406188964844, "loss": 0.0619, "rewards/accuracies": 0.9736841917037964, "rewards/chosen": -1.7347691059112549, "rewards/margins": 8.899223327636719, "rewards/rejected": -10.633993148803711, "step": 940 }, { "epoch": 2.0149333333333335, "grad_norm": 1.1779319047927856, "learning_rate": 1.291782839724009e-06, "logits/chosen": -1.7825742959976196, "logits/rejected": -1.6565110683441162, "logps/chosen": -235.5923614501953, "logps/rejected": -369.40374755859375, "loss": 0.0604, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.9541120529174805, "rewards/margins": 8.030762672424316, "rewards/rejected": -9.98487377166748, "step": 945 }, { "epoch": 2.0256, "grad_norm": 0.507275402545929, "learning_rate": 1.2666587372086352e-06, "logits/chosen": -1.6030664443969727, "logits/rejected": -1.5872256755828857, "logps/chosen": -216.4988555908203, "logps/rejected": -369.16253662109375, "loss": 0.0653, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.8965251445770264, "rewards/margins": 8.338624954223633, "rewards/rejected": -10.235151290893555, "step": 950 }, { "epoch": 2.0362666666666667, "grad_norm": 2.7491443157196045, "learning_rate": 1.241698199353275e-06, "logits/chosen": -1.7347681522369385, "logits/rejected": -1.6534042358398438, "logps/chosen": -231.2009735107422, "logps/rejected": -346.7289733886719, "loss": 0.0913, "rewards/accuracies": 0.96875, "rewards/chosen": -2.2630741596221924, "rewards/margins": 7.0988335609436035, "rewards/rejected": -9.361908912658691, "step": 955 }, { "epoch": 2.0469333333333335, "grad_norm": 2.5662288665771484, "learning_rate": 1.2169045364030404e-06, "logits/chosen": -1.7822414636611938, "logits/rejected": -1.6436115503311157, "logps/chosen": -217.4017333984375, "logps/rejected": -367.7906494140625, "loss": 0.0514, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.031571626663208, "rewards/margins": 8.508852005004883, "rewards/rejected": -10.540423393249512, "step": 960 }, { "epoch": 2.0576, "grad_norm": 3.719102382659912, "learning_rate": 1.1922810364722402e-06, "logits/chosen": -1.659507155418396, "logits/rejected": -1.6028478145599365, "logps/chosen": -251.8856658935547, "logps/rejected": -366.27392578125, "loss": 0.1071, "rewards/accuracies": 0.956250011920929, "rewards/chosen": -1.9120813608169556, "rewards/margins": 7.532308101654053, "rewards/rejected": -9.444389343261719, "step": 965 }, { "epoch": 2.0682666666666667, "grad_norm": 3.9604787826538086, "learning_rate": 1.1678309651083055e-06, "logits/chosen": -1.7748504877090454, "logits/rejected": -1.6407390832901, "logps/chosen": -232.59933471679688, "logps/rejected": -380.6536865234375, "loss": 0.0581, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.3573272228240967, "rewards/margins": 9.171064376831055, "rewards/rejected": -11.52839183807373, "step": 970 }, { "epoch": 2.0789333333333335, "grad_norm": 4.073202133178711, "learning_rate": 1.1435575648587224e-06, "logits/chosen": -1.766287088394165, "logits/rejected": -1.6389284133911133, "logps/chosen": -285.7763671875, "logps/rejected": -341.8055114746094, "loss": 0.1355, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -1.9729318618774414, "rewards/margins": 7.9406914710998535, "rewards/rejected": -9.913623809814453, "step": 975 }, { "epoch": 2.0896, "grad_norm": 2.3515031337738037, "learning_rate": 1.1194640548410039e-06, "logits/chosen": -1.6386430263519287, "logits/rejected": -1.5689029693603516, "logps/chosen": -251.07199096679688, "logps/rejected": -351.345458984375, "loss": 0.0837, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.034928321838379, "rewards/margins": 7.436675071716309, "rewards/rejected": -9.471604347229004, "step": 980 }, { "epoch": 2.1002666666666667, "grad_norm": 1.4685736894607544, "learning_rate": 1.095553630315774e-06, "logits/chosen": -1.6688451766967773, "logits/rejected": -1.5495831966400146, "logps/chosen": -240.5880889892578, "logps/rejected": -336.76947021484375, "loss": 0.0744, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.8441340923309326, "rewards/margins": 7.2546892166137695, "rewards/rejected": -9.098822593688965, "step": 985 }, { "epoch": 2.1109333333333336, "grad_norm": 2.042402505874634, "learning_rate": 1.0718294622630188e-06, "logits/chosen": -1.7300097942352295, "logits/rejected": -1.6546669006347656, "logps/chosen": -233.61190795898438, "logps/rejected": -374.1564025878906, "loss": 0.0485, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.8412272930145264, "rewards/margins": 8.404033660888672, "rewards/rejected": -10.245260238647461, "step": 990 }, { "epoch": 2.1216, "grad_norm": 0.9763467311859131, "learning_rate": 1.0482946969615495e-06, "logits/chosen": -1.822035551071167, "logits/rejected": -1.673659086227417, "logps/chosen": -214.61703491210938, "logps/rejected": -368.96636962890625, "loss": 0.0653, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.0393638610839844, "rewards/margins": 8.412542343139648, "rewards/rejected": -10.451906204223633, "step": 995 }, { "epoch": 2.1322666666666668, "grad_norm": 0.9107277989387512, "learning_rate": 1.0249524555717461e-06, "logits/chosen": -1.7914142608642578, "logits/rejected": -1.6238925457000732, "logps/chosen": -228.9333038330078, "logps/rejected": -378.4297790527344, "loss": 0.0541, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.8040258884429932, "rewards/margins": 8.715888977050781, "rewards/rejected": -10.519915580749512, "step": 1000 }, { "epoch": 2.142933333333333, "grad_norm": 0.46706506609916687, "learning_rate": 1.0018058337216327e-06, "logits/chosen": -1.6658966541290283, "logits/rejected": -1.629685640335083, "logps/chosen": -225.38327026367188, "logps/rejected": -384.2342224121094, "loss": 0.0488, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.277534008026123, "rewards/margins": 8.842560768127441, "rewards/rejected": -11.120096206665039, "step": 1005 }, { "epoch": 2.1536, "grad_norm": 0.696917712688446, "learning_rate": 9.788579010963387e-07, "logits/chosen": -1.8070281744003296, "logits/rejected": -1.6128984689712524, "logps/chosen": -234.9412078857422, "logps/rejected": -374.0914306640625, "loss": 0.0619, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.9849834442138672, "rewards/margins": 8.614242553710938, "rewards/rejected": -10.599225997924805, "step": 1010 }, { "epoch": 2.164266666666667, "grad_norm": 3.232445478439331, "learning_rate": 9.56111701030998e-07, "logits/chosen": -1.7992016077041626, "logits/rejected": -1.6329628229141235, "logps/chosen": -223.1928253173828, "logps/rejected": -355.9103698730469, "loss": 0.0846, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.365701198577881, "rewards/margins": 7.757696628570557, "rewards/rejected": -10.123396873474121, "step": 1015 }, { "epoch": 2.174933333333333, "grad_norm": 2.396444797515869, "learning_rate": 9.335702501071459e-07, "logits/chosen": -1.786289930343628, "logits/rejected": -1.632266640663147, "logps/chosen": -243.3270263671875, "logps/rejected": -354.66802978515625, "loss": 0.0683, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -1.8624069690704346, "rewards/margins": 8.355226516723633, "rewards/rejected": -10.217633247375488, "step": 1020 }, { "epoch": 2.1856, "grad_norm": 4.311215400695801, "learning_rate": 9.112365377526636e-07, "logits/chosen": -1.7827332019805908, "logits/rejected": -1.6588077545166016, "logps/chosen": -219.1776123046875, "logps/rejected": -376.1236572265625, "loss": 0.0534, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.2685389518737793, "rewards/margins": 8.695510864257812, "rewards/rejected": -10.964048385620117, "step": 1025 }, { "epoch": 2.196266666666667, "grad_norm": 1.4199345111846924, "learning_rate": 8.891135258453195e-07, "logits/chosen": -1.6969398260116577, "logits/rejected": -1.6353518962860107, "logps/chosen": -217.14682006835938, "logps/rejected": -352.71136474609375, "loss": 0.0646, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.6732425689697266, "rewards/margins": 8.477716445922852, "rewards/rejected": -10.150958061218262, "step": 1030 }, { "epoch": 2.206933333333333, "grad_norm": 1.0060111284255981, "learning_rate": 8.672041483199709e-07, "logits/chosen": -1.7296987771987915, "logits/rejected": -1.6749894618988037, "logps/chosen": -212.9332275390625, "logps/rejected": -356.7399597167969, "loss": 0.0892, "rewards/accuracies": 0.96875, "rewards/chosen": -1.9000186920166016, "rewards/margins": 8.221338272094727, "rewards/rejected": -10.121357917785645, "step": 1035 }, { "epoch": 2.2176, "grad_norm": 1.1529102325439453, "learning_rate": 8.455113107794652e-07, "logits/chosen": -1.7899577617645264, "logits/rejected": -1.684852957725525, "logps/chosen": -208.1083984375, "logps/rejected": -348.6238708496094, "loss": 0.0486, "rewards/accuracies": 1.0, "rewards/chosen": -2.258780002593994, "rewards/margins": 8.805041313171387, "rewards/rejected": -11.063821792602539, "step": 1040 }, { "epoch": 2.228266666666667, "grad_norm": 2.7815699577331543, "learning_rate": 8.240378901093035e-07, "logits/chosen": -1.6827160120010376, "logits/rejected": -1.5520236492156982, "logps/chosen": -235.8473358154297, "logps/rejected": -384.362060546875, "loss": 0.075, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.263546943664551, "rewards/margins": 8.731700897216797, "rewards/rejected": -10.995248794555664, "step": 1045 }, { "epoch": 2.238933333333333, "grad_norm": 3.8146376609802246, "learning_rate": 8.027867340961113e-07, "logits/chosen": -1.6788311004638672, "logits/rejected": -1.5772473812103271, "logps/chosen": -233.0138397216797, "logps/rejected": -352.76666259765625, "loss": 0.0815, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.020775318145752, "rewards/margins": 8.058589935302734, "rewards/rejected": -10.079363822937012, "step": 1050 }, { "epoch": 2.2496, "grad_norm": 4.936167240142822, "learning_rate": 7.817606610499639e-07, "logits/chosen": -1.7382968664169312, "logits/rejected": -1.6289379596710205, "logps/chosen": -243.76571655273438, "logps/rejected": -367.06884765625, "loss": 0.0747, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.0522563457489014, "rewards/margins": 8.087520599365234, "rewards/rejected": -10.139778137207031, "step": 1055 }, { "epoch": 2.2602666666666664, "grad_norm": 1.672697901725769, "learning_rate": 7.609624594306278e-07, "logits/chosen": -1.8423585891723633, "logits/rejected": -1.6809555292129517, "logps/chosen": -247.50497436523438, "logps/rejected": -400.91009521484375, "loss": 0.0697, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.359173536300659, "rewards/margins": 8.937664985656738, "rewards/rejected": -11.29683780670166, "step": 1060 }, { "epoch": 2.2709333333333332, "grad_norm": 1.9851620197296143, "learning_rate": 7.403948874777556e-07, "logits/chosen": -1.6603851318359375, "logits/rejected": -1.6031215190887451, "logps/chosen": -215.9097442626953, "logps/rejected": -357.3408508300781, "loss": 0.0697, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.0506255626678467, "rewards/margins": 8.248406410217285, "rewards/rejected": -10.299031257629395, "step": 1065 }, { "epoch": 2.2816, "grad_norm": 0.7464717030525208, "learning_rate": 7.20060672845091e-07, "logits/chosen": -1.6729190349578857, "logits/rejected": -1.677329421043396, "logps/chosen": -238.68331909179688, "logps/rejected": -372.63372802734375, "loss": 0.0629, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.739391803741455, "rewards/margins": 8.386183738708496, "rewards/rejected": -11.125576972961426, "step": 1070 }, { "epoch": 2.2922666666666665, "grad_norm": 2.3664517402648926, "learning_rate": 6.99962512238733e-07, "logits/chosen": -1.6504055261611938, "logits/rejected": -1.5495134592056274, "logps/chosen": -220.272216796875, "logps/rejected": -367.61602783203125, "loss": 0.0527, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.9944871664047241, "rewards/margins": 9.167341232299805, "rewards/rejected": -11.161828994750977, "step": 1075 }, { "epoch": 2.3029333333333333, "grad_norm": 1.6238492727279663, "learning_rate": 6.801030710594949e-07, "logits/chosen": -1.6580880880355835, "logits/rejected": -1.5914041996002197, "logps/chosen": -217.9814910888672, "logps/rejected": -347.48876953125, "loss": 0.0803, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.2583582401275635, "rewards/margins": 8.29969596862793, "rewards/rejected": -10.558053970336914, "step": 1080 }, { "epoch": 2.3136, "grad_norm": 2.321399688720703, "learning_rate": 6.604849830494264e-07, "logits/chosen": -1.8174232244491577, "logits/rejected": -1.685241460800171, "logps/chosen": -238.77615356445312, "logps/rejected": -348.0065002441406, "loss": 0.0858, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.586581230163574, "rewards/margins": 7.984179496765137, "rewards/rejected": -10.570761680603027, "step": 1085 }, { "epoch": 2.3242666666666665, "grad_norm": 0.6217744946479797, "learning_rate": 6.411108499425253e-07, "logits/chosen": -1.7700458765029907, "logits/rejected": -1.6312090158462524, "logps/chosen": -242.84988403320312, "logps/rejected": -364.59869384765625, "loss": 0.0587, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.2468154430389404, "rewards/margins": 8.542348861694336, "rewards/rejected": -10.789164543151855, "step": 1090 }, { "epoch": 2.3349333333333333, "grad_norm": 1.4033160209655762, "learning_rate": 6.219832411197013e-07, "logits/chosen": -1.6557588577270508, "logits/rejected": -1.5285712480545044, "logps/chosen": -232.84140014648438, "logps/rejected": -368.0865783691406, "loss": 0.0784, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -1.9303604364395142, "rewards/margins": 8.661425590515137, "rewards/rejected": -10.591785430908203, "step": 1095 }, { "epoch": 2.3456, "grad_norm": 1.5255765914916992, "learning_rate": 6.031046932680229e-07, "logits/chosen": -1.7217681407928467, "logits/rejected": -1.5818337202072144, "logps/chosen": -236.8732147216797, "logps/rejected": -360.173583984375, "loss": 0.0681, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.275178909301758, "rewards/margins": 7.931247711181641, "rewards/rejected": -10.206426620483398, "step": 1100 }, { "epoch": 2.3562666666666665, "grad_norm": 0.8869664669036865, "learning_rate": 5.844777100443091e-07, "logits/chosen": -1.769890546798706, "logits/rejected": -1.6314458847045898, "logps/chosen": -225.7227325439453, "logps/rejected": -370.30084228515625, "loss": 0.0573, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.3325419425964355, "rewards/margins": 9.121394157409668, "rewards/rejected": -11.453936576843262, "step": 1105 }, { "epoch": 2.3669333333333333, "grad_norm": 1.9305953979492188, "learning_rate": 5.66104761743092e-07, "logits/chosen": -1.8521592617034912, "logits/rejected": -1.786136269569397, "logps/chosen": -237.04428100585938, "logps/rejected": -393.07232666015625, "loss": 0.0569, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.924665689468384, "rewards/margins": 9.040776252746582, "rewards/rejected": -11.965442657470703, "step": 1110 }, { "epoch": 2.3776, "grad_norm": 1.5030986070632935, "learning_rate": 5.479882849690102e-07, "logits/chosen": -1.7002397775650024, "logits/rejected": -1.6100225448608398, "logps/chosen": -249.65884399414062, "logps/rejected": -375.28472900390625, "loss": 0.055, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.4748988151550293, "rewards/margins": 8.668333053588867, "rewards/rejected": -11.143232345581055, "step": 1115 }, { "epoch": 2.3882666666666665, "grad_norm": 3.2749836444854736, "learning_rate": 5.301306823136687e-07, "logits/chosen": -1.6064075231552124, "logits/rejected": -1.5443446636199951, "logps/chosen": -222.10498046875, "logps/rejected": -354.99224853515625, "loss": 0.1063, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -2.446816921234131, "rewards/margins": 7.639315605163574, "rewards/rejected": -10.086133003234863, "step": 1120 }, { "epoch": 2.3989333333333334, "grad_norm": 6.177357196807861, "learning_rate": 5.125343220370088e-07, "logits/chosen": -1.7167507410049438, "logits/rejected": -1.5875396728515625, "logps/chosen": -251.5364532470703, "logps/rejected": -350.0692138671875, "loss": 0.1133, "rewards/accuracies": 0.96875, "rewards/chosen": -2.6598141193389893, "rewards/margins": 7.746614933013916, "rewards/rejected": -10.406428337097168, "step": 1125 }, { "epoch": 2.4096, "grad_norm": 2.799936056137085, "learning_rate": 4.952015377532329e-07, "logits/chosen": -1.8022699356079102, "logits/rejected": -1.648284912109375, "logps/chosen": -219.56723022460938, "logps/rejected": -375.37835693359375, "loss": 0.0589, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.597977638244629, "rewards/margins": 9.261964797973633, "rewards/rejected": -11.859942436218262, "step": 1130 }, { "epoch": 2.4202666666666666, "grad_norm": 0.8679878115653992, "learning_rate": 4.781346281213204e-07, "logits/chosen": -1.758948564529419, "logits/rejected": -1.6984516382217407, "logps/chosen": -216.8728790283203, "logps/rejected": -372.5053405761719, "loss": 0.0526, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.3770511150360107, "rewards/margins": 9.284753799438477, "rewards/rejected": -11.66180419921875, "step": 1135 }, { "epoch": 2.4309333333333334, "grad_norm": 3.049687147140503, "learning_rate": 4.613358565401818e-07, "logits/chosen": -1.703428864479065, "logits/rejected": -1.647216796875, "logps/chosen": -251.83456420898438, "logps/rejected": -382.45513916015625, "loss": 0.0735, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.692182779312134, "rewards/margins": 7.887045860290527, "rewards/rejected": -10.579229354858398, "step": 1140 }, { "epoch": 2.4416, "grad_norm": 1.0848246812820435, "learning_rate": 4.448074508484906e-07, "logits/chosen": -1.7292941808700562, "logits/rejected": -1.5465002059936523, "logps/chosen": -243.04476928710938, "logps/rejected": -368.3805236816406, "loss": 0.0597, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.3163161277770996, "rewards/margins": 8.721205711364746, "rewards/rejected": -11.037521362304688, "step": 1145 }, { "epoch": 2.4522666666666666, "grad_norm": 1.5578293800354004, "learning_rate": 4.2855160302922777e-07, "logits/chosen": -1.7324931621551514, "logits/rejected": -1.696868896484375, "logps/chosen": -221.2848358154297, "logps/rejected": -381.62359619140625, "loss": 0.0938, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.416508197784424, "rewards/margins": 9.063284873962402, "rewards/rejected": -11.479792594909668, "step": 1150 }, { "epoch": 2.4629333333333334, "grad_norm": 2.7087442874908447, "learning_rate": 4.125704689189819e-07, "logits/chosen": -1.6965996026992798, "logits/rejected": -1.5242478847503662, "logps/chosen": -222.7657928466797, "logps/rejected": -364.8160095214844, "loss": 0.0673, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.0459980964660645, "rewards/margins": 8.586442947387695, "rewards/rejected": -10.632440567016602, "step": 1155 }, { "epoch": 2.4736000000000002, "grad_norm": 4.279755592346191, "learning_rate": 3.9686616792204677e-07, "logits/chosen": -1.6694886684417725, "logits/rejected": -1.6094839572906494, "logps/chosen": -223.513916015625, "logps/rejected": -360.60211181640625, "loss": 0.0563, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.202442169189453, "rewards/margins": 8.466798782348633, "rewards/rejected": -10.669241905212402, "step": 1160 }, { "epoch": 2.4842666666666666, "grad_norm": 3.6986382007598877, "learning_rate": 3.8144078272934676e-07, "logits/chosen": -1.7204971313476562, "logits/rejected": -1.6476894617080688, "logps/chosen": -246.6821746826172, "logps/rejected": -372.75634765625, "loss": 0.0671, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.262946128845215, "rewards/margins": 9.07579231262207, "rewards/rejected": -11.338738441467285, "step": 1165 }, { "epoch": 2.4949333333333334, "grad_norm": 0.5171156525611877, "learning_rate": 3.6629635904222943e-07, "logits/chosen": -1.7819716930389404, "logits/rejected": -1.6632438898086548, "logps/chosen": -226.53421020507812, "logps/rejected": -402.94598388671875, "loss": 0.048, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.1981759071350098, "rewards/margins": 9.268115043640137, "rewards/rejected": -11.466291427612305, "step": 1170 }, { "epoch": 2.5056000000000003, "grad_norm": 1.4313619136810303, "learning_rate": 3.514349053011698e-07, "logits/chosen": -1.7268043756484985, "logits/rejected": -1.5567543506622314, "logps/chosen": -242.3755645751953, "logps/rejected": -378.0379333496094, "loss": 0.0647, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.5210442543029785, "rewards/margins": 8.706852912902832, "rewards/rejected": -11.227896690368652, "step": 1175 }, { "epoch": 2.5162666666666667, "grad_norm": 4.2524638175964355, "learning_rate": 3.3685839241941133e-07, "logits/chosen": -1.6792681217193604, "logits/rejected": -1.6164538860321045, "logps/chosen": -227.9314727783203, "logps/rejected": -347.4278259277344, "loss": 0.0808, "rewards/accuracies": 0.96875, "rewards/chosen": -2.1522090435028076, "rewards/margins": 8.178491592407227, "rewards/rejected": -10.33070182800293, "step": 1180 }, { "epoch": 2.5269333333333335, "grad_norm": 4.731504917144775, "learning_rate": 3.2256875352158435e-07, "logits/chosen": -1.698481798171997, "logits/rejected": -1.6122955083847046, "logps/chosen": -238.3931427001953, "logps/rejected": -390.96356201171875, "loss": 0.0562, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.2728371620178223, "rewards/margins": 9.056289672851562, "rewards/rejected": -11.329126358032227, "step": 1185 }, { "epoch": 2.5376, "grad_norm": 2.2026448249816895, "learning_rate": 3.085678836873399e-07, "logits/chosen": -1.700873613357544, "logits/rejected": -1.6878290176391602, "logps/chosen": -203.16343688964844, "logps/rejected": -371.3619079589844, "loss": 0.0637, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.1309585571289062, "rewards/margins": 8.881011009216309, "rewards/rejected": -11.011968612670898, "step": 1190 }, { "epoch": 2.5482666666666667, "grad_norm": 2.0885634422302246, "learning_rate": 2.9485763970002106e-07, "logits/chosen": -1.6919275522232056, "logits/rejected": -1.6454477310180664, "logps/chosen": -229.4306640625, "logps/rejected": -385.67364501953125, "loss": 0.0581, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -3.0748720169067383, "rewards/margins": 9.040914535522461, "rewards/rejected": -12.1157865524292, "step": 1195 }, { "epoch": 2.558933333333333, "grad_norm": 1.6398587226867676, "learning_rate": 2.8143983980042174e-07, "logits/chosen": -1.6853199005126953, "logits/rejected": -1.563265085220337, "logps/chosen": -220.6924591064453, "logps/rejected": -361.0828552246094, "loss": 0.0533, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.5653061866760254, "rewards/margins": 8.984647750854492, "rewards/rejected": -11.549954414367676, "step": 1200 }, { "epoch": 2.5696, "grad_norm": 4.250986099243164, "learning_rate": 2.683162634456515e-07, "logits/chosen": -1.7149680852890015, "logits/rejected": -1.6375681161880493, "logps/chosen": -243.39767456054688, "logps/rejected": -375.77197265625, "loss": 0.122, "rewards/accuracies": 0.96875, "rewards/chosen": -2.3229522705078125, "rewards/margins": 8.259476661682129, "rewards/rejected": -10.582427978515625, "step": 1205 }, { "epoch": 2.5802666666666667, "grad_norm": 1.7875478267669678, "learning_rate": 2.5548865107314606e-07, "logits/chosen": -1.6222026348114014, "logits/rejected": -1.5698870420455933, "logps/chosen": -252.62234497070312, "logps/rejected": -371.35968017578125, "loss": 0.0474, "rewards/accuracies": 1.0, "rewards/chosen": -1.9672263860702515, "rewards/margins": 8.441937446594238, "rewards/rejected": -10.409163475036621, "step": 1210 }, { "epoch": 2.590933333333333, "grad_norm": 2.756495475769043, "learning_rate": 2.429587038698511e-07, "logits/chosen": -1.7179148197174072, "logits/rejected": -1.6547319889068604, "logps/chosen": -215.96231079101562, "logps/rejected": -383.422607421875, "loss": 0.0606, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.6292073726654053, "rewards/margins": 9.533937454223633, "rewards/rejected": -12.1631441116333, "step": 1215 }, { "epoch": 2.6016, "grad_norm": 2.755079984664917, "learning_rate": 2.307280835466144e-07, "logits/chosen": -1.7297523021697998, "logits/rejected": -1.5602505207061768, "logps/chosen": -226.3815155029297, "logps/rejected": -379.71246337890625, "loss": 0.0489, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.6925208568573, "rewards/margins": 8.994084358215332, "rewards/rejected": -11.686605453491211, "step": 1220 }, { "epoch": 2.6122666666666667, "grad_norm": 2.1932475566864014, "learning_rate": 2.1879841211780856e-07, "logits/chosen": -1.6469560861587524, "logits/rejected": -1.5588574409484863, "logps/chosen": -237.40298461914062, "logps/rejected": -373.273681640625, "loss": 0.067, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.41096568107605, "rewards/margins": 8.6922607421875, "rewards/rejected": -11.103225708007812, "step": 1225 }, { "epoch": 2.622933333333333, "grad_norm": 2.3217146396636963, "learning_rate": 2.0717127168622202e-07, "logits/chosen": -1.7379156351089478, "logits/rejected": -1.5808634757995605, "logps/chosen": -245.385498046875, "logps/rejected": -379.40460205078125, "loss": 0.0561, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.5605995655059814, "rewards/margins": 8.971899032592773, "rewards/rejected": -11.532498359680176, "step": 1230 }, { "epoch": 2.6336, "grad_norm": 3.9729578495025635, "learning_rate": 1.958482042332424e-07, "logits/chosen": -1.7232568264007568, "logits/rejected": -1.5390961170196533, "logps/chosen": -262.4111328125, "logps/rejected": -380.0039978027344, "loss": 0.0991, "rewards/accuracies": 0.96875, "rewards/chosen": -2.5589394569396973, "rewards/margins": 8.578977584838867, "rewards/rejected": -11.137918472290039, "step": 1235 }, { "epoch": 2.6442666666666668, "grad_norm": 3.108950138092041, "learning_rate": 1.8483071141435937e-07, "logits/chosen": -1.6421682834625244, "logits/rejected": -1.5635117292404175, "logps/chosen": -269.7666015625, "logps/rejected": -369.1126403808594, "loss": 0.0626, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.4174113273620605, "rewards/margins": 8.311561584472656, "rewards/rejected": -10.728974342346191, "step": 1240 }, { "epoch": 2.654933333333333, "grad_norm": 2.487255811691284, "learning_rate": 1.741202543600176e-07, "logits/chosen": -1.7232882976531982, "logits/rejected": -1.6298853158950806, "logps/chosen": -225.5282440185547, "logps/rejected": -367.0820007324219, "loss": 0.0481, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.8948845863342285, "rewards/margins": 8.565133094787598, "rewards/rejected": -11.460018157958984, "step": 1245 }, { "epoch": 2.6656, "grad_norm": 3.5590980052948, "learning_rate": 1.6371825348184155e-07, "logits/chosen": -1.6784608364105225, "logits/rejected": -1.5763369798660278, "logps/chosen": -233.04873657226562, "logps/rejected": -362.5445861816406, "loss": 0.0588, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.41444730758667, "rewards/margins": 8.470220565795898, "rewards/rejected": -10.884668350219727, "step": 1250 }, { "epoch": 2.676266666666667, "grad_norm": 2.1251602172851562, "learning_rate": 1.5362608828426246e-07, "logits/chosen": -1.7499914169311523, "logits/rejected": -1.6538130044937134, "logps/chosen": -247.6387939453125, "logps/rejected": -385.8589782714844, "loss": 0.0527, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.267554759979248, "rewards/margins": 9.327815055847168, "rewards/rejected": -11.595370292663574, "step": 1255 }, { "epoch": 2.686933333333333, "grad_norm": 3.002373456954956, "learning_rate": 1.438450971815686e-07, "logits/chosen": -1.7843786478042603, "logits/rejected": -1.6649062633514404, "logps/chosen": -237.3309326171875, "logps/rejected": -372.9043884277344, "loss": 0.0639, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.4575018882751465, "rewards/margins": 9.09184741973877, "rewards/rejected": -11.549348831176758, "step": 1260 }, { "epoch": 2.6976, "grad_norm": 3.6394524574279785, "learning_rate": 1.3437657732040783e-07, "logits/chosen": -1.7269392013549805, "logits/rejected": -1.5753428936004639, "logps/chosen": -228.05859375, "logps/rejected": -347.77587890625, "loss": 0.0659, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.337057590484619, "rewards/margins": 8.416463851928711, "rewards/rejected": -10.753522872924805, "step": 1265 }, { "epoch": 2.708266666666667, "grad_norm": 2.0372815132141113, "learning_rate": 1.2522178440776022e-07, "logits/chosen": -1.7152751684188843, "logits/rejected": -1.6060600280761719, "logps/chosen": -223.6414337158203, "logps/rejected": -367.9722595214844, "loss": 0.0512, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.0669002532958984, "rewards/margins": 8.808296203613281, "rewards/rejected": -10.87519645690918, "step": 1270 }, { "epoch": 2.718933333333333, "grad_norm": 3.3202083110809326, "learning_rate": 1.1638193254440655e-07, "logits/chosen": -1.7251523733139038, "logits/rejected": -1.6463289260864258, "logps/chosen": -242.5394287109375, "logps/rejected": -384.67828369140625, "loss": 0.0626, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.65033221244812, "rewards/margins": 9.154813766479492, "rewards/rejected": -11.805147171020508, "step": 1275 }, { "epoch": 2.7296, "grad_norm": 3.912275791168213, "learning_rate": 1.0785819406391562e-07, "logits/chosen": -1.629055380821228, "logits/rejected": -1.6328773498535156, "logps/chosen": -222.7128448486328, "logps/rejected": -362.33697509765625, "loss": 0.0828, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.1510047912597656, "rewards/margins": 8.43962287902832, "rewards/rejected": -10.59062671661377, "step": 1280 }, { "epoch": 2.740266666666667, "grad_norm": 1.5474649667739868, "learning_rate": 9.965169937717095e-08, "logits/chosen": -1.731776475906372, "logits/rejected": -1.6290785074234009, "logps/chosen": -220.5186004638672, "logps/rejected": -371.43017578125, "loss": 0.0489, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.538206100463867, "rewards/margins": 8.95645809173584, "rewards/rejected": -11.494664192199707, "step": 1285 }, { "epoch": 2.7509333333333332, "grad_norm": 4.089476585388184, "learning_rate": 9.176353682245676e-08, "logits/chosen": -1.7597144842147827, "logits/rejected": -1.7338988780975342, "logps/chosen": -222.9378204345703, "logps/rejected": -379.48046875, "loss": 0.0821, "rewards/accuracies": 0.96875, "rewards/chosen": -2.5998435020446777, "rewards/margins": 8.580057144165039, "rewards/rejected": -11.179900169372559, "step": 1290 }, { "epoch": 2.7616, "grad_norm": 2.3440282344818115, "learning_rate": 8.41947525211223e-08, "logits/chosen": -1.6561254262924194, "logits/rejected": -1.5427234172821045, "logps/chosen": -236.984130859375, "logps/rejected": -358.4666442871094, "loss": 0.0705, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.1874051094055176, "rewards/margins": 8.178415298461914, "rewards/rejected": -10.36582088470459, "step": 1295 }, { "epoch": 2.772266666666667, "grad_norm": 0.8540581464767456, "learning_rate": 7.694635023884789e-08, "logits/chosen": -1.6888628005981445, "logits/rejected": -1.6380894184112549, "logps/chosen": -244.80453491210938, "logps/rejected": -398.01416015625, "loss": 0.0572, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.498230457305908, "rewards/margins": 9.02320671081543, "rewards/rejected": -11.521438598632812, "step": 1300 }, { "epoch": 2.7829333333333333, "grad_norm": 2.1657631397247314, "learning_rate": 7.00192912525241e-08, "logits/chosen": -1.7026500701904297, "logits/rejected": -1.6592811346054077, "logps/chosen": -236.1211395263672, "logps/rejected": -372.11700439453125, "loss": 0.0853, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.4645047187805176, "rewards/margins": 8.029794692993164, "rewards/rejected": -10.494298934936523, "step": 1305 }, { "epoch": 2.7936, "grad_norm": 2.2320377826690674, "learning_rate": 6.341449422277047e-08, "logits/chosen": -1.6398401260375977, "logits/rejected": -1.586236834526062, "logps/chosen": -264.7616882324219, "logps/rejected": -387.8482971191406, "loss": 0.0761, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.7749290466308594, "rewards/margins": 8.451501846313477, "rewards/rejected": -11.226430892944336, "step": 1310 }, { "epoch": 2.804266666666667, "grad_norm": 2.414036989212036, "learning_rate": 5.7132835072101486e-08, "logits/chosen": -1.7663352489471436, "logits/rejected": -1.7004578113555908, "logps/chosen": -232.11264038085938, "logps/rejected": -388.02520751953125, "loss": 0.0685, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.7615132331848145, "rewards/margins": 9.383472442626953, "rewards/rejected": -12.144986152648926, "step": 1315 }, { "epoch": 2.8149333333333333, "grad_norm": 3.1423819065093994, "learning_rate": 5.117514686876379e-08, "logits/chosen": -1.639988660812378, "logits/rejected": -1.6136987209320068, "logps/chosen": -226.6444854736328, "logps/rejected": -338.81243896484375, "loss": 0.0958, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -1.9987974166870117, "rewards/margins": 7.690194606781006, "rewards/rejected": -9.688993453979492, "step": 1320 }, { "epoch": 2.8256, "grad_norm": 3.009913206100464, "learning_rate": 4.5542219716255274e-08, "logits/chosen": -1.726658582687378, "logits/rejected": -1.6036049127578735, "logps/chosen": -248.6483154296875, "logps/rejected": -389.01104736328125, "loss": 0.0751, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.6450247764587402, "rewards/margins": 9.053154945373535, "rewards/rejected": -11.698179244995117, "step": 1325 }, { "epoch": 2.836266666666667, "grad_norm": 1.047540545463562, "learning_rate": 4.023480064854174e-08, "logits/chosen": -1.7443851232528687, "logits/rejected": -1.6466442346572876, "logps/chosen": -232.3708038330078, "logps/rejected": -355.1757507324219, "loss": 0.0386, "rewards/accuracies": 1.0, "rewards/chosen": -2.3548583984375, "rewards/margins": 8.295844078063965, "rewards/rejected": -10.650702476501465, "step": 1330 }, { "epoch": 2.8469333333333333, "grad_norm": 2.1108577251434326, "learning_rate": 3.525359353098584e-08, "logits/chosen": -1.711523413658142, "logits/rejected": -1.564855694770813, "logps/chosen": -234.50955200195312, "logps/rejected": -401.87237548828125, "loss": 0.085, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.5529706478118896, "rewards/margins": 9.332552909851074, "rewards/rejected": -11.885522842407227, "step": 1335 }, { "epoch": 2.8576, "grad_norm": 1.4761799573898315, "learning_rate": 3.059925896700094e-08, "logits/chosen": -1.7107757329940796, "logits/rejected": -1.5677356719970703, "logps/chosen": -256.050048828125, "logps/rejected": -387.1049499511719, "loss": 0.0517, "rewards/accuracies": 1.0, "rewards/chosen": -2.124401330947876, "rewards/margins": 9.434419631958008, "rewards/rejected": -11.558820724487305, "step": 1340 }, { "epoch": 2.8682666666666665, "grad_norm": 2.649878978729248, "learning_rate": 2.6272414210443964e-08, "logits/chosen": -1.6712446212768555, "logits/rejected": -1.666788101196289, "logps/chosen": -246.9146270751953, "logps/rejected": -387.96697998046875, "loss": 0.107, "rewards/accuracies": 0.9624999761581421, "rewards/chosen": -2.460017681121826, "rewards/margins": 8.867338180541992, "rewards/rejected": -11.32735538482666, "step": 1345 }, { "epoch": 2.8789333333333333, "grad_norm": 0.556817889213562, "learning_rate": 2.2273633083753632e-08, "logits/chosen": -1.7067201137542725, "logits/rejected": -1.613102674484253, "logps/chosen": -216.364501953125, "logps/rejected": -377.43341064453125, "loss": 0.0453, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.5222439765930176, "rewards/margins": 9.433998107910156, "rewards/rejected": -11.956242561340332, "step": 1350 }, { "epoch": 2.8895999999999997, "grad_norm": 2.3840444087982178, "learning_rate": 1.8603445901851647e-08, "logits/chosen": -1.6174293756484985, "logits/rejected": -1.5605263710021973, "logps/chosen": -234.5871124267578, "logps/rejected": -393.0818176269531, "loss": 0.053, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.399482250213623, "rewards/margins": 8.842782020568848, "rewards/rejected": -11.242264747619629, "step": 1355 }, { "epoch": 2.9002666666666665, "grad_norm": 1.3605575561523438, "learning_rate": 1.5262339401813375e-08, "logits/chosen": -1.721879005432129, "logits/rejected": -1.6476589441299438, "logps/chosen": -225.7846221923828, "logps/rejected": -390.84075927734375, "loss": 0.0683, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.652186632156372, "rewards/margins": 9.073946952819824, "rewards/rejected": -11.726133346557617, "step": 1360 }, { "epoch": 2.9109333333333334, "grad_norm": 7.382095813751221, "learning_rate": 1.2250756678315601e-08, "logits/chosen": -1.7093260288238525, "logits/rejected": -1.6412489414215088, "logps/chosen": -242.4256591796875, "logps/rejected": -390.5251159667969, "loss": 0.0874, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.4845237731933594, "rewards/margins": 8.939850807189941, "rewards/rejected": -11.424372673034668, "step": 1365 }, { "epoch": 2.9215999999999998, "grad_norm": 3.5759291648864746, "learning_rate": 9.569097124875193e-09, "logits/chosen": -1.6812433004379272, "logits/rejected": -1.6351463794708252, "logps/chosen": -224.98519897460938, "logps/rejected": -373.64129638671875, "loss": 0.0635, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.5327999591827393, "rewards/margins": 8.734055519104004, "rewards/rejected": -11.266853332519531, "step": 1370 }, { "epoch": 2.9322666666666666, "grad_norm": 3.4827802181243896, "learning_rate": 7.217716380881479e-09, "logits/chosen": -1.7382152080535889, "logits/rejected": -1.634007215499878, "logps/chosen": -261.80853271484375, "logps/rejected": -366.9189758300781, "loss": 0.0837, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -2.2536139488220215, "rewards/margins": 8.559202194213867, "rewards/rejected": -10.812816619873047, "step": 1375 }, { "epoch": 2.9429333333333334, "grad_norm": 4.551513195037842, "learning_rate": 5.196926284430359e-09, "logits/chosen": -1.6665912866592407, "logits/rejected": -1.5187146663665771, "logps/chosen": -244.35867309570312, "logps/rejected": -395.45672607421875, "loss": 0.051, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.5061800479888916, "rewards/margins": 8.809569358825684, "rewards/rejected": -11.31574821472168, "step": 1380 }, { "epoch": 2.9536, "grad_norm": 2.610435724258423, "learning_rate": 3.5069948309712775e-09, "logits/chosen": -1.6568130254745483, "logits/rejected": -1.6110321283340454, "logps/chosen": -270.39569091796875, "logps/rejected": -387.37115478515625, "loss": 0.0588, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.6436781883239746, "rewards/margins": 8.328022956848145, "rewards/rejected": -10.971700668334961, "step": 1385 }, { "epoch": 2.9642666666666666, "grad_norm": 1.42137610912323, "learning_rate": 2.1481461377634294e-09, "logits/chosen": -1.6595335006713867, "logits/rejected": -1.6308250427246094, "logps/chosen": -232.0425262451172, "logps/rejected": -376.16619873046875, "loss": 0.0508, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.516741991043091, "rewards/margins": 8.540366172790527, "rewards/rejected": -11.057107925415039, "step": 1390 }, { "epoch": 2.9749333333333334, "grad_norm": 0.8431697487831116, "learning_rate": 1.120560414154259e-09, "logits/chosen": -1.7065174579620361, "logits/rejected": -1.6119663715362549, "logps/chosen": -232.8285675048828, "logps/rejected": -367.2099914550781, "loss": 0.0596, "rewards/accuracies": 0.9937499761581421, "rewards/chosen": -2.5289416313171387, "rewards/margins": 8.318876266479492, "rewards/rejected": -10.847818374633789, "step": 1395 }, { "epoch": 2.9856, "grad_norm": 3.3298075199127197, "learning_rate": 4.2437393768079983e-10, "logits/chosen": -1.663039207458496, "logits/rejected": -1.571150779724121, "logps/chosen": -250.9148406982422, "logps/rejected": -356.40277099609375, "loss": 0.0692, "rewards/accuracies": 0.981249988079071, "rewards/chosen": -2.4644951820373535, "rewards/margins": 8.116125106811523, "rewards/rejected": -10.580621719360352, "step": 1400 }, { "epoch": 2.9962666666666666, "grad_norm": 2.234276056289673, "learning_rate": 5.967903599635171e-11, "logits/chosen": -1.6961066722869873, "logits/rejected": -1.5509659051895142, "logps/chosen": -247.11538696289062, "logps/rejected": -370.01708984375, "loss": 0.0646, "rewards/accuracies": 0.987500011920929, "rewards/chosen": -2.5652105808258057, "rewards/margins": 8.701112747192383, "rewards/rejected": -11.266324043273926, "step": 1405 } ], "logging_steps": 5, "max_steps": 1407, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }