| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1407, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 3.066758394241333, |
| "learning_rate": 4.651162790697675e-07, |
| "logits/chosen": -1.3197795152664185, |
| "logits/rejected": -1.2224810123443604, |
| "logps/chosen": -204.17431640625, |
| "logps/rejected": -269.80377197265625, |
| "loss": 0.6924, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": 0.0002665018546395004, |
| "rewards/margins": 0.0014735888689756393, |
| "rewards/rejected": -0.001207086956128478, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 2.98679518699646, |
| "learning_rate": 1.0465116279069768e-06, |
| "logits/chosen": -1.4425337314605713, |
| "logits/rejected": -1.2940964698791504, |
| "logps/chosen": -222.4615020751953, |
| "logps/rejected": -268.4735107421875, |
| "loss": 0.6907, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.0007552242605015635, |
| "rewards/margins": 0.004916320089250803, |
| "rewards/rejected": -0.005671544000506401, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 3.7883105278015137, |
| "learning_rate": 1.6279069767441862e-06, |
| "logits/chosen": -1.313001036643982, |
| "logits/rejected": -1.217395305633545, |
| "logps/chosen": -210.53271484375, |
| "logps/rejected": -285.55682373046875, |
| "loss": 0.6822, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 0.0025394915137439966, |
| "rewards/margins": 0.02237287536263466, |
| "rewards/rejected": -0.019833387807011604, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 2.714655876159668, |
| "learning_rate": 2.2093023255813954e-06, |
| "logits/chosen": -1.4219006299972534, |
| "logits/rejected": -1.280832052230835, |
| "logps/chosen": -238.50692749023438, |
| "logps/rejected": -272.02459716796875, |
| "loss": 0.6745, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.002396273659542203, |
| "rewards/margins": 0.03847304731607437, |
| "rewards/rejected": -0.040869321674108505, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 2.902036428451538, |
| "learning_rate": 2.790697674418605e-06, |
| "logits/chosen": -1.4469196796417236, |
| "logits/rejected": -1.2472175359725952, |
| "logps/chosen": -211.281982421875, |
| "logps/rejected": -252.13522338867188, |
| "loss": 0.6594, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.004396043252199888, |
| "rewards/margins": 0.07000456750392914, |
| "rewards/rejected": -0.06560851633548737, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 2.938281297683716, |
| "learning_rate": 3.372093023255814e-06, |
| "logits/chosen": -1.4945942163467407, |
| "logits/rejected": -1.301879644393921, |
| "logps/chosen": -212.25778198242188, |
| "logps/rejected": -261.4756164550781, |
| "loss": 0.6355, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.005297659430652857, |
| "rewards/margins": 0.12356561422348022, |
| "rewards/rejected": -0.1182679682970047, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 2.9229683876037598, |
| "learning_rate": 3.953488372093024e-06, |
| "logits/chosen": -1.4990527629852295, |
| "logits/rejected": -1.313819408416748, |
| "logps/chosen": -206.4686737060547, |
| "logps/rejected": -267.2922058105469, |
| "loss": 0.6139, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.006779120769351721, |
| "rewards/margins": 0.17405611276626587, |
| "rewards/rejected": -0.1672770082950592, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 2.707540512084961, |
| "learning_rate": 4.5348837209302326e-06, |
| "logits/chosen": -1.4922987222671509, |
| "logits/rejected": -1.3102424144744873, |
| "logps/chosen": -214.93185424804688, |
| "logps/rejected": -265.9106140136719, |
| "loss": 0.5915, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.011891713365912437, |
| "rewards/margins": 0.23512515425682068, |
| "rewards/rejected": -0.24701690673828125, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 2.74711012840271, |
| "learning_rate": 4.99999336897255e-06, |
| "logits/chosen": -1.4568241834640503, |
| "logits/rejected": -1.3190878629684448, |
| "logps/chosen": -222.8993682861328, |
| "logps/rejected": -264.18621826171875, |
| "loss": 0.5688, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.018879670649766922, |
| "rewards/margins": 0.30939120054244995, |
| "rewards/rejected": -0.3282708525657654, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 2.671937942504883, |
| "learning_rate": 4.999761286705285e-06, |
| "logits/chosen": -1.5383602380752563, |
| "logits/rejected": -1.4474451541900635, |
| "logps/chosen": -217.95303344726562, |
| "logps/rejected": -269.7035217285156, |
| "loss": 0.5035, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.00281223701313138, |
| "rewards/margins": 0.4914434552192688, |
| "rewards/rejected": -0.48863115906715393, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 2.843602180480957, |
| "learning_rate": 4.999197688241076e-06, |
| "logits/chosen": -1.55707848072052, |
| "logits/rejected": -1.3680062294006348, |
| "logps/chosen": -212.3292236328125, |
| "logps/rejected": -257.45703125, |
| "loss": 0.492, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.03159821033477783, |
| "rewards/margins": 0.5501774549484253, |
| "rewards/rejected": -0.5817757248878479, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 2.6057443618774414, |
| "learning_rate": 4.9983026483238685e-06, |
| "logits/chosen": -1.4305418729782104, |
| "logits/rejected": -1.2694460153579712, |
| "logps/chosen": -216.9158477783203, |
| "logps/rejected": -259.95635986328125, |
| "loss": 0.4569, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.14237883687019348, |
| "rewards/margins": 0.7050798535346985, |
| "rewards/rejected": -0.8474586606025696, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 2.425227642059326, |
| "learning_rate": 4.997076285653087e-06, |
| "logits/chosen": -1.608428716659546, |
| "logits/rejected": -1.3585041761398315, |
| "logps/chosen": -262.80120849609375, |
| "logps/rejected": -263.1980895996094, |
| "loss": 0.4232, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.00389072741381824, |
| "rewards/margins": 0.8892091512680054, |
| "rewards/rejected": -0.8930999040603638, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 2.3068881034851074, |
| "learning_rate": 4.995518762867897e-06, |
| "logits/chosen": -1.5861982107162476, |
| "logits/rejected": -1.2906171083450317, |
| "logps/chosen": -175.07534790039062, |
| "logps/rejected": -259.4339904785156, |
| "loss": 0.3609, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.07811270654201508, |
| "rewards/margins": 1.111189603805542, |
| "rewards/rejected": -1.1893023252487183, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 1.8398106098175049, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -1.5915586948394775, |
| "logits/rejected": -1.345815896987915, |
| "logps/chosen": -199.1811065673828, |
| "logps/rejected": -284.7003173828125, |
| "loss": 0.3512, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.17231644690036774, |
| "rewards/margins": 1.2371994256973267, |
| "rewards/rejected": -1.4095157384872437, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 3.092566967010498, |
| "learning_rate": 4.99141110707441e-06, |
| "logits/chosen": -1.5269695520401, |
| "logits/rejected": -1.3563108444213867, |
| "logps/chosen": -229.861328125, |
| "logps/rejected": -276.8891296386719, |
| "loss": 0.421, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.34268826246261597, |
| "rewards/margins": 1.11867356300354, |
| "rewards/rejected": -1.4613617658615112, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 2.967259645462036, |
| "learning_rate": 4.9888615188199e-06, |
| "logits/chosen": -1.6198972463607788, |
| "logits/rejected": -1.4323259592056274, |
| "logps/chosen": -210.46420288085938, |
| "logps/rejected": -266.1463928222656, |
| "loss": 0.3459, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.2865266799926758, |
| "rewards/margins": 1.4747304916381836, |
| "rewards/rejected": -1.7612571716308594, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 2.3651838302612305, |
| "learning_rate": 4.9859818598863104e-06, |
| "logits/chosen": -1.5707067251205444, |
| "logits/rejected": -1.3580453395843506, |
| "logps/chosen": -205.1324920654297, |
| "logps/rejected": -268.135498046875, |
| "loss": 0.3159, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.22517745196819305, |
| "rewards/margins": 1.6200993061065674, |
| "rewards/rejected": -1.8452764749526978, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 2.180738925933838, |
| "learning_rate": 4.982772512171538e-06, |
| "logits/chosen": -1.6069647073745728, |
| "logits/rejected": -1.3733923435211182, |
| "logps/chosen": -221.70205688476562, |
| "logps/rejected": -266.65936279296875, |
| "loss": 0.3344, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.3390544652938843, |
| "rewards/margins": 1.6071252822875977, |
| "rewards/rejected": -1.946179747581482, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 2.911017417907715, |
| "learning_rate": 4.979233901296523e-06, |
| "logits/chosen": -1.639183759689331, |
| "logits/rejected": -1.477196455001831, |
| "logps/chosen": -209.12045288085938, |
| "logps/rejected": -278.9235534667969, |
| "loss": 0.3083, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.20341868698596954, |
| "rewards/margins": 1.9127600193023682, |
| "rewards/rejected": -2.116178512573242, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 2.939307689666748, |
| "learning_rate": 4.975366496548804e-06, |
| "logits/chosen": -1.6216596364974976, |
| "logits/rejected": -1.4458906650543213, |
| "logps/chosen": -211.3112335205078, |
| "logps/rejected": -282.20977783203125, |
| "loss": 0.3117, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.2629513144493103, |
| "rewards/margins": 1.812154769897461, |
| "rewards/rejected": -2.075106143951416, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 2.625629425048828, |
| "learning_rate": 4.971170810820279e-06, |
| "logits/chosen": -1.4949898719787598, |
| "logits/rejected": -1.3904539346694946, |
| "logps/chosen": -223.5279998779297, |
| "logps/rejected": -269.46875, |
| "loss": 0.3132, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.3314862549304962, |
| "rewards/margins": 1.777958631515503, |
| "rewards/rejected": -2.1094448566436768, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 3.023860216140747, |
| "learning_rate": 4.96664740053919e-06, |
| "logits/chosen": -1.642871618270874, |
| "logits/rejected": -1.433465838432312, |
| "logps/chosen": -209.8994140625, |
| "logps/rejected": -304.18511962890625, |
| "loss": 0.255, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.1216890811920166, |
| "rewards/margins": 2.104187488555908, |
| "rewards/rejected": -2.225876569747925, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 3.2610344886779785, |
| "learning_rate": 4.961796865596328e-06, |
| "logits/chosen": -1.5937464237213135, |
| "logits/rejected": -1.4256340265274048, |
| "logps/chosen": -212.27304077148438, |
| "logps/rejected": -290.52996826171875, |
| "loss": 0.2696, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.09086675941944122, |
| "rewards/margins": 1.9682395458221436, |
| "rewards/rejected": -2.0591063499450684, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 3.3823888301849365, |
| "learning_rate": 4.956619849265473e-06, |
| "logits/chosen": -1.6408603191375732, |
| "logits/rejected": -1.4571561813354492, |
| "logps/chosen": -204.12246704101562, |
| "logps/rejected": -274.4853820800781, |
| "loss": 0.2656, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.09454663842916489, |
| "rewards/margins": 2.163012742996216, |
| "rewards/rejected": -2.0684661865234375, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 2.7941596508026123, |
| "learning_rate": 4.951117038118091e-06, |
| "logits/chosen": -1.7679383754730225, |
| "logits/rejected": -1.5151549577713013, |
| "logps/chosen": -195.41064453125, |
| "logps/rejected": -282.4126892089844, |
| "loss": 0.2546, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.09231258183717728, |
| "rewards/margins": 2.3412868976593018, |
| "rewards/rejected": -2.248974323272705, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 2.4657509326934814, |
| "learning_rate": 4.945289161932273e-06, |
| "logits/chosen": -1.653076171875, |
| "logits/rejected": -1.4966038465499878, |
| "logps/chosen": -220.5636444091797, |
| "logps/rejected": -305.3443298339844, |
| "loss": 0.2716, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.18228843808174133, |
| "rewards/margins": 2.278294324874878, |
| "rewards/rejected": -2.4605822563171387, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 1.8939595222473145, |
| "learning_rate": 4.93913699359596e-06, |
| "logits/chosen": -1.6945682764053345, |
| "logits/rejected": -1.454713225364685, |
| "logps/chosen": -195.47763061523438, |
| "logps/rejected": -295.0526123046875, |
| "loss": 0.243, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.133425772190094, |
| "rewards/margins": 2.4948630332946777, |
| "rewards/rejected": -2.628288984298706, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 2.4227945804595947, |
| "learning_rate": 4.9326613490044364e-06, |
| "logits/chosen": -1.6317930221557617, |
| "logits/rejected": -1.4936482906341553, |
| "logps/chosen": -194.4447479248047, |
| "logps/rejected": -272.43817138671875, |
| "loss": 0.2312, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.060553692281246185, |
| "rewards/margins": 2.502532482147217, |
| "rewards/rejected": -2.5630862712860107, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 2.1593096256256104, |
| "learning_rate": 4.925863086952132e-06, |
| "logits/chosen": -1.7782328128814697, |
| "logits/rejected": -1.4951965808868408, |
| "logps/chosen": -226.15151977539062, |
| "logps/rejected": -306.14990234375, |
| "loss": 0.2448, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.27948349714279175, |
| "rewards/margins": 2.5876212120056152, |
| "rewards/rejected": -2.8671042919158936, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 2.2807936668395996, |
| "learning_rate": 4.918743109018727e-06, |
| "logits/chosen": -1.8481355905532837, |
| "logits/rejected": -1.5771901607513428, |
| "logps/chosen": -200.24911499023438, |
| "logps/rejected": -291.2858581542969, |
| "loss": 0.2295, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.19684214890003204, |
| "rewards/margins": 2.7656893730163574, |
| "rewards/rejected": -2.962531566619873, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 2.781081199645996, |
| "learning_rate": 4.911302359449585e-06, |
| "logits/chosen": -1.7050254344940186, |
| "logits/rejected": -1.5437817573547363, |
| "logps/chosen": -211.6606903076172, |
| "logps/rejected": -313.52203369140625, |
| "loss": 0.1747, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.4274963438510895, |
| "rewards/margins": 3.1094963550567627, |
| "rewards/rejected": -3.5369930267333984, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 3.3999805450439453, |
| "learning_rate": 4.9035418250305314e-06, |
| "logits/chosen": -1.7344772815704346, |
| "logits/rejected": -1.4708181619644165, |
| "logps/chosen": -196.78701782226562, |
| "logps/rejected": -279.26397705078125, |
| "loss": 0.2272, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.15598328411579132, |
| "rewards/margins": 2.7185912132263184, |
| "rewards/rejected": -2.8745741844177246, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 1.978710412979126, |
| "learning_rate": 4.895462534956981e-06, |
| "logits/chosen": -1.60355544090271, |
| "logits/rejected": -1.51287841796875, |
| "logps/chosen": -216.2396697998047, |
| "logps/rejected": -283.2531433105469, |
| "loss": 0.1758, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.42660221457481384, |
| "rewards/margins": 3.12617564201355, |
| "rewards/rejected": -3.5527777671813965, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 2.282623767852783, |
| "learning_rate": 4.887065560697449e-06, |
| "logits/chosen": -1.6703214645385742, |
| "logits/rejected": -1.4852759838104248, |
| "logps/chosen": -228.0096893310547, |
| "logps/rejected": -297.5685729980469, |
| "loss": 0.2108, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.459760844707489, |
| "rewards/margins": 3.1451258659362793, |
| "rewards/rejected": -3.604886531829834, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 5.582287788391113, |
| "learning_rate": 4.878352015851459e-06, |
| "logits/chosen": -1.6780675649642944, |
| "logits/rejected": -1.4558064937591553, |
| "logps/chosen": -244.1021270751953, |
| "logps/rejected": -275.14801025390625, |
| "loss": 0.2492, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.52231365442276, |
| "rewards/margins": 3.0731654167175293, |
| "rewards/rejected": -3.5954787731170654, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 2.2447121143341064, |
| "learning_rate": 4.869323056001852e-06, |
| "logits/chosen": -1.637040376663208, |
| "logits/rejected": -1.53791344165802, |
| "logps/chosen": -196.7782745361328, |
| "logps/rejected": -282.7772216796875, |
| "loss": 0.218, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.40790072083473206, |
| "rewards/margins": 3.118037700653076, |
| "rewards/rejected": -3.5259385108947754, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 2.1624879837036133, |
| "learning_rate": 4.859979878561536e-06, |
| "logits/chosen": -1.8144794702529907, |
| "logits/rejected": -1.6100742816925049, |
| "logps/chosen": -216.9547576904297, |
| "logps/rejected": -315.37200927734375, |
| "loss": 0.2231, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.23768725991249084, |
| "rewards/margins": 3.17393159866333, |
| "rewards/rejected": -3.411618709564209, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 2.3933207988739014, |
| "learning_rate": 4.850323722614685e-06, |
| "logits/chosen": -1.76116943359375, |
| "logits/rejected": -1.6245193481445312, |
| "logps/chosen": -199.56607055664062, |
| "logps/rejected": -301.9045104980469, |
| "loss": 0.2032, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.5237962603569031, |
| "rewards/margins": 3.485597610473633, |
| "rewards/rejected": -4.00939416885376, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 3.6170833110809326, |
| "learning_rate": 4.840355868752416e-06, |
| "logits/chosen": -1.6510473489761353, |
| "logits/rejected": -1.5840654373168945, |
| "logps/chosen": -268.947265625, |
| "logps/rejected": -308.9850158691406, |
| "loss": 0.2189, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.20040014386177063, |
| "rewards/margins": 3.211362838745117, |
| "rewards/rejected": -3.4117634296417236, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 2.984818935394287, |
| "learning_rate": 4.830077638902955e-06, |
| "logits/chosen": -1.811394453048706, |
| "logits/rejected": -1.5767472982406616, |
| "logps/chosen": -205.17953491210938, |
| "logps/rejected": -299.9072265625, |
| "loss": 0.2058, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.42685478925704956, |
| "rewards/margins": 3.5523762702941895, |
| "rewards/rejected": -3.9792308807373047, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 2.1014328002929688, |
| "learning_rate": 4.8194903961563255e-06, |
| "logits/chosen": -1.8685874938964844, |
| "logits/rejected": -1.6196568012237549, |
| "logps/chosen": -221.1055145263672, |
| "logps/rejected": -294.41632080078125, |
| "loss": 0.176, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.4206708073616028, |
| "rewards/margins": 3.7292656898498535, |
| "rewards/rejected": -4.149935722351074, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 5.251358985900879, |
| "learning_rate": 4.808595544583578e-06, |
| "logits/chosen": -1.778464674949646, |
| "logits/rejected": -1.6301262378692627, |
| "logps/chosen": -212.44192504882812, |
| "logps/rejected": -294.9122009277344, |
| "loss": 0.1973, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.4846112132072449, |
| "rewards/margins": 3.767674684524536, |
| "rewards/rejected": -4.252285957336426, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 2.667045831680298, |
| "learning_rate": 4.797394529050577e-06, |
| "logits/chosen": -1.7264249324798584, |
| "logits/rejected": -1.6010576486587524, |
| "logps/chosen": -236.34164428710938, |
| "logps/rejected": -300.9986572265625, |
| "loss": 0.196, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.28851407766342163, |
| "rewards/margins": 3.629396438598633, |
| "rewards/rejected": -3.917910099029541, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 1.5569956302642822, |
| "learning_rate": 4.785888835026393e-06, |
| "logits/chosen": -1.781558632850647, |
| "logits/rejected": -1.5860977172851562, |
| "logps/chosen": -217.97152709960938, |
| "logps/rejected": -281.3045654296875, |
| "loss": 0.2418, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.35533851385116577, |
| "rewards/margins": 3.287522077560425, |
| "rewards/rejected": -3.6428608894348145, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 3.2689754962921143, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -1.7362861633300781, |
| "logits/rejected": -1.6026140451431274, |
| "logps/chosen": -214.66616821289062, |
| "logps/rejected": -305.7296447753906, |
| "loss": 0.1496, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.2582802176475525, |
| "rewards/margins": 3.910578489303589, |
| "rewards/rejected": -4.168859004974365, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 4.52901029586792, |
| "learning_rate": 4.761969555209394e-06, |
| "logits/chosen": -1.7565912008285522, |
| "logits/rejected": -1.6375453472137451, |
| "logps/chosen": -210.79177856445312, |
| "logps/rejected": -305.73822021484375, |
| "loss": 0.2532, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.5417909026145935, |
| "rewards/margins": 3.8477985858917236, |
| "rewards/rejected": -4.389589309692383, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 4.18936014175415, |
| "learning_rate": 4.7495591415709435e-06, |
| "logits/chosen": -1.6808313131332397, |
| "logits/rejected": -1.5127551555633545, |
| "logps/chosen": -217.28466796875, |
| "logps/rejected": -306.2400817871094, |
| "loss": 0.2088, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.44368869066238403, |
| "rewards/margins": 3.673114061355591, |
| "rewards/rejected": -4.11680269241333, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 3.285203218460083, |
| "learning_rate": 4.736850393329353e-06, |
| "logits/chosen": -1.8288180828094482, |
| "logits/rejected": -1.6284465789794922, |
| "logps/chosen": -221.87826538085938, |
| "logps/rejected": -316.38104248046875, |
| "loss": 0.1934, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.36977553367614746, |
| "rewards/margins": 3.9045498371124268, |
| "rewards/rejected": -4.274325370788574, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 4.095683574676514, |
| "learning_rate": 4.7238449959079095e-06, |
| "logits/chosen": -1.8461517095565796, |
| "logits/rejected": -1.6903188228607178, |
| "logps/chosen": -196.01882934570312, |
| "logps/rejected": -302.14788818359375, |
| "loss": 0.2039, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.24502971768379211, |
| "rewards/margins": 3.6217434406280518, |
| "rewards/rejected": -3.8667728900909424, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 1.374786615371704, |
| "learning_rate": 4.710544674071259e-06, |
| "logits/chosen": -1.7972240447998047, |
| "logits/rejected": -1.580497145652771, |
| "logps/chosen": -213.19131469726562, |
| "logps/rejected": -290.453125, |
| "loss": 0.1502, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.20407120883464813, |
| "rewards/margins": 4.157924175262451, |
| "rewards/rejected": -4.361996173858643, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 2.1457080841064453, |
| "learning_rate": 4.696951191696673e-06, |
| "logits/chosen": -1.8307178020477295, |
| "logits/rejected": -1.6306785345077515, |
| "logps/chosen": -203.19503784179688, |
| "logps/rejected": -329.07586669921875, |
| "loss": 0.1253, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.26944035291671753, |
| "rewards/margins": 4.326865196228027, |
| "rewards/rejected": -4.596304893493652, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 2.9863739013671875, |
| "learning_rate": 4.6830663515401196e-06, |
| "logits/chosen": -1.6911954879760742, |
| "logits/rejected": -1.5336120128631592, |
| "logps/chosen": -212.753173828125, |
| "logps/rejected": -283.8844299316406, |
| "loss": 0.1979, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.37001127004623413, |
| "rewards/margins": 3.8743293285369873, |
| "rewards/rejected": -4.244340419769287, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 2.874260425567627, |
| "learning_rate": 4.6688919949971855e-06, |
| "logits/chosen": -1.7525287866592407, |
| "logits/rejected": -1.6272214651107788, |
| "logps/chosen": -207.9388885498047, |
| "logps/rejected": -303.46429443359375, |
| "loss": 0.1714, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.3549277186393738, |
| "rewards/margins": 4.431785583496094, |
| "rewards/rejected": -4.786712646484375, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 2.1960959434509277, |
| "learning_rate": 4.654430001858874e-06, |
| "logits/chosen": -1.7510707378387451, |
| "logits/rejected": -1.570367693901062, |
| "logps/chosen": -238.5811767578125, |
| "logps/rejected": -299.3658752441406, |
| "loss": 0.1824, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.21106000244617462, |
| "rewards/margins": 3.991091251373291, |
| "rewards/rejected": -4.202151298522949, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 2.189987897872925, |
| "learning_rate": 4.6396822900623075e-06, |
| "logits/chosen": -1.7313045263290405, |
| "logits/rejected": -1.5751632452011108, |
| "logps/chosen": -211.08139038085938, |
| "logps/rejected": -303.4233703613281, |
| "loss": 0.1475, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5332785248756409, |
| "rewards/margins": 4.4254255294799805, |
| "rewards/rejected": -4.958703994750977, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 10.575684547424316, |
| "learning_rate": 4.624650815436368e-06, |
| "logits/chosen": -1.8441358804702759, |
| "logits/rejected": -1.6829391717910767, |
| "logps/chosen": -232.00424194335938, |
| "logps/rejected": -311.0403747558594, |
| "loss": 0.2001, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5895386338233948, |
| "rewards/margins": 4.146374702453613, |
| "rewards/rejected": -4.735913276672363, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 1.8553959131240845, |
| "learning_rate": 4.609337571442324e-06, |
| "logits/chosen": -1.8459510803222656, |
| "logits/rejected": -1.7000173330307007, |
| "logps/chosen": -191.3725128173828, |
| "logps/rejected": -306.92742919921875, |
| "loss": 0.158, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5892201662063599, |
| "rewards/margins": 4.554601192474365, |
| "rewards/rejected": -5.143821716308594, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 2.1983144283294678, |
| "learning_rate": 4.5937445889094535e-06, |
| "logits/chosen": -1.6407095193862915, |
| "logits/rejected": -1.5538684129714966, |
| "logps/chosen": -235.87109375, |
| "logps/rejected": -304.3785705566406, |
| "loss": 0.1787, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.501952588558197, |
| "rewards/margins": 4.091989040374756, |
| "rewards/rejected": -4.593941688537598, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 1.2407987117767334, |
| "learning_rate": 4.577873935765722e-06, |
| "logits/chosen": -1.9018303155899048, |
| "logits/rejected": -1.693389892578125, |
| "logps/chosen": -209.4306640625, |
| "logps/rejected": -332.36517333984375, |
| "loss": 0.0919, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.5658477544784546, |
| "rewards/margins": 5.112331390380859, |
| "rewards/rejected": -5.678179740905762, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 1.5631043910980225, |
| "learning_rate": 4.561727716763532e-06, |
| "logits/chosen": -1.7761032581329346, |
| "logits/rejected": -1.6239516735076904, |
| "logps/chosen": -236.88400268554688, |
| "logps/rejected": -315.68780517578125, |
| "loss": 0.1755, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5217987895011902, |
| "rewards/margins": 4.937346935272217, |
| "rewards/rejected": -5.459145545959473, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 1.763484001159668, |
| "learning_rate": 4.545308073200591e-06, |
| "logits/chosen": -1.6939966678619385, |
| "logits/rejected": -1.5736029148101807, |
| "logps/chosen": -225.3236083984375, |
| "logps/rejected": -334.81121826171875, |
| "loss": 0.1684, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5952404737472534, |
| "rewards/margins": 4.6785101890563965, |
| "rewards/rejected": -5.273751258850098, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 3.163860321044922, |
| "learning_rate": 4.528617182635943e-06, |
| "logits/chosen": -1.934821367263794, |
| "logits/rejected": -1.7904952764511108, |
| "logps/chosen": -203.1216278076172, |
| "logps/rejected": -331.48663330078125, |
| "loss": 0.1783, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5681445002555847, |
| "rewards/margins": 4.853375434875488, |
| "rewards/rejected": -5.421519756317139, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 1.8331536054611206, |
| "learning_rate": 4.511657258601171e-06, |
| "logits/chosen": -1.7939611673355103, |
| "logits/rejected": -1.6202869415283203, |
| "logps/chosen": -211.56533813476562, |
| "logps/rejected": -303.11834716796875, |
| "loss": 0.1335, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.6103761792182922, |
| "rewards/margins": 5.00363826751709, |
| "rewards/rejected": -5.614014625549316, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 1.3633877038955688, |
| "learning_rate": 4.494430550306848e-06, |
| "logits/chosen": -1.6322972774505615, |
| "logits/rejected": -1.6094653606414795, |
| "logps/chosen": -216.24557495117188, |
| "logps/rejected": -314.92193603515625, |
| "loss": 0.1366, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.6063324809074402, |
| "rewards/margins": 4.898473262786865, |
| "rewards/rejected": -5.504805564880371, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 1.5049735307693481, |
| "learning_rate": 4.476939342344246e-06, |
| "logits/chosen": -1.830894112586975, |
| "logits/rejected": -1.6699556112289429, |
| "logps/chosen": -202.03094482421875, |
| "logps/rejected": -319.1224060058594, |
| "loss": 0.1148, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.5489516854286194, |
| "rewards/margins": 5.44413423538208, |
| "rewards/rejected": -5.993086338043213, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 2.9216864109039307, |
| "learning_rate": 4.45918595438236e-06, |
| "logits/chosen": -1.7787328958511353, |
| "logits/rejected": -1.6474401950836182, |
| "logps/chosen": -213.6125030517578, |
| "logps/rejected": -314.5631103515625, |
| "loss": 0.1801, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.7099496126174927, |
| "rewards/margins": 4.897684574127197, |
| "rewards/rejected": -5.607633590698242, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 4.806079864501953, |
| "learning_rate": 4.441172740860267e-06, |
| "logits/chosen": -1.8154537677764893, |
| "logits/rejected": -1.6812942028045654, |
| "logps/chosen": -228.5246124267578, |
| "logps/rejected": -320.4506530761719, |
| "loss": 0.1543, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.6220763325691223, |
| "rewards/margins": 4.960526943206787, |
| "rewards/rejected": -5.582603454589844, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 2.260047674179077, |
| "learning_rate": 4.422902090674886e-06, |
| "logits/chosen": -1.6601673364639282, |
| "logits/rejected": -1.5397001504898071, |
| "logps/chosen": -214.0808563232422, |
| "logps/rejected": -315.56097412109375, |
| "loss": 0.1274, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.5406688451766968, |
| "rewards/margins": 5.2952775955200195, |
| "rewards/rejected": -5.835946559906006, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 2.84885835647583, |
| "learning_rate": 4.4043764268641705e-06, |
| "logits/chosen": -1.7757313251495361, |
| "logits/rejected": -1.5302140712738037, |
| "logps/chosen": -200.83782958984375, |
| "logps/rejected": -310.81170654296875, |
| "loss": 0.1321, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.5374216437339783, |
| "rewards/margins": 5.517133712768555, |
| "rewards/rejected": -6.054555416107178, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 3.0638837814331055, |
| "learning_rate": 4.385598206285758e-06, |
| "logits/chosen": -1.8033117055892944, |
| "logits/rejected": -1.6644874811172485, |
| "logps/chosen": -227.2172088623047, |
| "logps/rejected": -337.43292236328125, |
| "loss": 0.159, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.6855946779251099, |
| "rewards/margins": 5.421025276184082, |
| "rewards/rejected": -6.106619834899902, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 3.001445770263672, |
| "learning_rate": 4.3665699192911495e-06, |
| "logits/chosen": -1.7176687717437744, |
| "logits/rejected": -1.6000038385391235, |
| "logps/chosen": -218.575927734375, |
| "logps/rejected": -318.5699157714844, |
| "loss": 0.1649, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8334270715713501, |
| "rewards/margins": 5.054620742797852, |
| "rewards/rejected": -5.888047695159912, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 3.584191083908081, |
| "learning_rate": 4.347294089395438e-06, |
| "logits/chosen": -1.786150574684143, |
| "logits/rejected": -1.6655750274658203, |
| "logps/chosen": -239.9277801513672, |
| "logps/rejected": -339.0256652832031, |
| "loss": 0.132, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.7777694463729858, |
| "rewards/margins": 5.329873561859131, |
| "rewards/rejected": -6.107643127441406, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 2.659503698348999, |
| "learning_rate": 4.327773272942647e-06, |
| "logits/chosen": -1.806623101234436, |
| "logits/rejected": -1.6834806203842163, |
| "logps/chosen": -200.3981170654297, |
| "logps/rejected": -310.19976806640625, |
| "loss": 0.1844, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.8376480937004089, |
| "rewards/margins": 5.1270952224731445, |
| "rewards/rejected": -5.964743137359619, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 2.4195191860198975, |
| "learning_rate": 4.308010058766703e-06, |
| "logits/chosen": -1.7857242822647095, |
| "logits/rejected": -1.6443517208099365, |
| "logps/chosen": -219.5111846923828, |
| "logps/rejected": -346.9495544433594, |
| "loss": 0.1878, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.1358754634857178, |
| "rewards/margins": 5.381402969360352, |
| "rewards/rejected": -6.51727819442749, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 3.1393349170684814, |
| "learning_rate": 4.2880070678481135e-06, |
| "logits/chosen": -1.8039171695709229, |
| "logits/rejected": -1.5734426975250244, |
| "logps/chosen": -218.7495574951172, |
| "logps/rejected": -320.8000183105469, |
| "loss": 0.156, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.4172883033752441, |
| "rewards/margins": 5.502140045166016, |
| "rewards/rejected": -6.91942834854126, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 6.10773229598999, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.7366790771484375, |
| "logits/rejected": -1.6303174495697021, |
| "logps/chosen": -213.4806671142578, |
| "logps/rejected": -313.7432861328125, |
| "loss": 0.2032, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.0189698934555054, |
| "rewards/margins": 5.155045509338379, |
| "rewards/rejected": -6.174015045166016, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 1.2211837768554688, |
| "learning_rate": 4.24729239834814e-06, |
| "logits/chosen": -1.7981315851211548, |
| "logits/rejected": -1.6859023571014404, |
| "logps/chosen": -214.998779296875, |
| "logps/rejected": -304.8877258300781, |
| "loss": 0.1508, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7106130123138428, |
| "rewards/margins": 5.31712007522583, |
| "rewards/rejected": -6.027733325958252, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 5.1816630363464355, |
| "learning_rate": 4.226586119311291e-06, |
| "logits/chosen": -1.8075335025787354, |
| "logits/rejected": -1.7757810354232788, |
| "logps/chosen": -230.5372314453125, |
| "logps/rejected": -332.861572265625, |
| "loss": 0.197, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.9396656155586243, |
| "rewards/margins": 4.923933982849121, |
| "rewards/rejected": -5.8635993003845215, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 2.9293625354766846, |
| "learning_rate": 4.205650861904782e-06, |
| "logits/chosen": -1.8099794387817383, |
| "logits/rejected": -1.6509116888046265, |
| "logps/chosen": -201.75900268554688, |
| "logps/rejected": -337.0589294433594, |
| "loss": 0.1469, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8850414156913757, |
| "rewards/margins": 5.205699920654297, |
| "rewards/rejected": -6.090741157531738, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 1.2313185930252075, |
| "learning_rate": 4.184489402544486e-06, |
| "logits/chosen": -1.8144805431365967, |
| "logits/rejected": -1.6301672458648682, |
| "logps/chosen": -206.3054656982422, |
| "logps/rejected": -325.19049072265625, |
| "loss": 0.135, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9306513071060181, |
| "rewards/margins": 5.58634090423584, |
| "rewards/rejected": -6.516992092132568, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 2.0421454906463623, |
| "learning_rate": 4.1631045476449865e-06, |
| "logits/chosen": -1.8140941858291626, |
| "logits/rejected": -1.686056137084961, |
| "logps/chosen": -221.7446746826172, |
| "logps/rejected": -328.5806884765625, |
| "loss": 0.1628, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.9575926065444946, |
| "rewards/margins": 5.376477241516113, |
| "rewards/rejected": -6.334069728851318, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 2.528268337249756, |
| "learning_rate": 4.141499133247391e-06, |
| "logits/chosen": -1.6540710926055908, |
| "logits/rejected": -1.5933629274368286, |
| "logps/chosen": -219.1824188232422, |
| "logps/rejected": -326.68304443359375, |
| "loss": 0.162, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7679177522659302, |
| "rewards/margins": 5.114426136016846, |
| "rewards/rejected": -5.8823442459106445, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 1.8557026386260986, |
| "learning_rate": 4.119676024643217e-06, |
| "logits/chosen": -1.6958087682724, |
| "logits/rejected": -1.5895674228668213, |
| "logps/chosen": -214.60275268554688, |
| "logps/rejected": -318.4465026855469, |
| "loss": 0.141, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5810152292251587, |
| "rewards/margins": 5.569883346557617, |
| "rewards/rejected": -6.1508989334106445, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 7.853280544281006, |
| "learning_rate": 4.0976381159944055e-06, |
| "logits/chosen": -1.807705283164978, |
| "logits/rejected": -1.5605567693710327, |
| "logps/chosen": -228.89804077148438, |
| "logps/rejected": -328.572265625, |
| "loss": 0.1435, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.6997815370559692, |
| "rewards/margins": 5.620684623718262, |
| "rewards/rejected": -6.320466995239258, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 4.629083633422852, |
| "learning_rate": 4.07538832994949e-06, |
| "logits/chosen": -1.7461702823638916, |
| "logits/rejected": -1.5441920757293701, |
| "logps/chosen": -213.9942169189453, |
| "logps/rejected": -337.7629089355469, |
| "loss": 0.1637, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9062296748161316, |
| "rewards/margins": 5.746966361999512, |
| "rewards/rejected": -6.653195858001709, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 3.687288522720337, |
| "learning_rate": 4.0529296172560055e-06, |
| "logits/chosen": -1.8524515628814697, |
| "logits/rejected": -1.7046585083007812, |
| "logps/chosen": -207.7529296875, |
| "logps/rejected": -340.30145263671875, |
| "loss": 0.1495, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.9373359680175781, |
| "rewards/margins": 5.981744289398193, |
| "rewards/rejected": -6.9190802574157715, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 2.6075193881988525, |
| "learning_rate": 4.030264956369158e-06, |
| "logits/chosen": -1.7657550573349, |
| "logits/rejected": -1.6465190649032593, |
| "logps/chosen": -209.3252410888672, |
| "logps/rejected": -304.9324035644531, |
| "loss": 0.1542, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.7593217492103577, |
| "rewards/margins": 5.0572614669799805, |
| "rewards/rejected": -5.816583156585693, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 4.559110641479492, |
| "learning_rate": 4.0073973530568274e-06, |
| "logits/chosen": -1.7933986186981201, |
| "logits/rejected": -1.600174903869629, |
| "logps/chosen": -224.45346069335938, |
| "logps/rejected": -342.8002624511719, |
| "loss": 0.1297, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.7520440220832825, |
| "rewards/margins": 5.706141471862793, |
| "rewards/rejected": -6.458186149597168, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 2.7471275329589844, |
| "learning_rate": 3.9843298400009435e-06, |
| "logits/chosen": -1.8151620626449585, |
| "logits/rejected": -1.6723363399505615, |
| "logps/chosen": -218.27420043945312, |
| "logps/rejected": -336.54058837890625, |
| "loss": 0.123, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.027143120765686, |
| "rewards/margins": 5.687465667724609, |
| "rewards/rejected": -6.714608669281006, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 1.708066701889038, |
| "learning_rate": 3.961065476395296e-06, |
| "logits/chosen": -1.8115184307098389, |
| "logits/rejected": -1.6580374240875244, |
| "logps/chosen": -241.6630859375, |
| "logps/rejected": -348.1864318847656, |
| "loss": 0.1565, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.9067646861076355, |
| "rewards/margins": 5.730084419250488, |
| "rewards/rejected": -6.636848449707031, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 2.5178802013397217, |
| "learning_rate": 3.937607347539823e-06, |
| "logits/chosen": -1.8042964935302734, |
| "logits/rejected": -1.606554627418518, |
| "logps/chosen": -206.523193359375, |
| "logps/rejected": -315.8009338378906, |
| "loss": 0.1346, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7148440480232239, |
| "rewards/margins": 5.358724594116211, |
| "rewards/rejected": -6.073568820953369, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 0.6924991011619568, |
| "learning_rate": 3.913958564431447e-06, |
| "logits/chosen": -1.8427581787109375, |
| "logits/rejected": -1.6687357425689697, |
| "logps/chosen": -235.98275756835938, |
| "logps/rejected": -346.68865966796875, |
| "loss": 0.1137, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.1400498151779175, |
| "rewards/margins": 6.211422443389893, |
| "rewards/rejected": -7.351471900939941, |
| "step": 465 |
| }, |
| { |
| "epoch": 1.0021333333333333, |
| "grad_norm": 3.3022406101226807, |
| "learning_rate": 3.89012226335149e-06, |
| "logits/chosen": -1.7215778827667236, |
| "logits/rejected": -1.5378063917160034, |
| "logps/chosen": -236.6403350830078, |
| "logps/rejected": -333.3316345214844, |
| "loss": 0.1219, |
| "rewards/accuracies": 0.9407894611358643, |
| "rewards/chosen": -0.9613617062568665, |
| "rewards/margins": 5.4095001220703125, |
| "rewards/rejected": -6.370861053466797, |
| "step": 470 |
| }, |
| { |
| "epoch": 1.0128, |
| "grad_norm": 1.652114987373352, |
| "learning_rate": 3.86610160544975e-06, |
| "logits/chosen": -1.7459675073623657, |
| "logits/rejected": -1.6606613397598267, |
| "logps/chosen": -211.37924194335938, |
| "logps/rejected": -328.5991516113281, |
| "loss": 0.0908, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9254496693611145, |
| "rewards/margins": 6.109896659851074, |
| "rewards/rejected": -7.035346031188965, |
| "step": 475 |
| }, |
| { |
| "epoch": 1.0234666666666667, |
| "grad_norm": 2.3160781860351562, |
| "learning_rate": 3.841899776325267e-06, |
| "logits/chosen": -1.7546608448028564, |
| "logits/rejected": -1.5244556665420532, |
| "logps/chosen": -245.4587860107422, |
| "logps/rejected": -330.42510986328125, |
| "loss": 0.0982, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.7176437377929688, |
| "rewards/margins": 5.7568793296813965, |
| "rewards/rejected": -6.474523067474365, |
| "step": 480 |
| }, |
| { |
| "epoch": 1.0341333333333333, |
| "grad_norm": 2.9602835178375244, |
| "learning_rate": 3.817519985603855e-06, |
| "logits/chosen": -1.8549671173095703, |
| "logits/rejected": -1.6893632411956787, |
| "logps/chosen": -205.00753784179688, |
| "logps/rejected": -327.4825744628906, |
| "loss": 0.1123, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.2238247394561768, |
| "rewards/margins": 6.4054765701293945, |
| "rewards/rejected": -7.62930154800415, |
| "step": 485 |
| }, |
| { |
| "epoch": 1.0448, |
| "grad_norm": 1.5233310461044312, |
| "learning_rate": 3.792965466512437e-06, |
| "logits/chosen": -1.7780135869979858, |
| "logits/rejected": -1.5944887399673462, |
| "logps/chosen": -233.1146240234375, |
| "logps/rejected": -335.52093505859375, |
| "loss": 0.1319, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.054016351699829, |
| "rewards/margins": 6.117007255554199, |
| "rewards/rejected": -7.171023368835449, |
| "step": 490 |
| }, |
| { |
| "epoch": 1.0554666666666668, |
| "grad_norm": 2.667370319366455, |
| "learning_rate": 3.7682394754502687e-06, |
| "logits/chosen": -1.7536907196044922, |
| "logits/rejected": -1.6732431650161743, |
| "logps/chosen": -223.99746704101562, |
| "logps/rejected": -346.42034912109375, |
| "loss": 0.0791, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.2943538427352905, |
| "rewards/margins": 6.558332920074463, |
| "rewards/rejected": -7.852685451507568, |
| "step": 495 |
| }, |
| { |
| "epoch": 1.0661333333333334, |
| "grad_norm": 7.733152389526367, |
| "learning_rate": 3.7433452915570685e-06, |
| "logits/chosen": -1.8311758041381836, |
| "logits/rejected": -1.6713851690292358, |
| "logps/chosen": -215.52981567382812, |
| "logps/rejected": -323.02642822265625, |
| "loss": 0.1374, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.3178616762161255, |
| "rewards/margins": 6.0462646484375, |
| "rewards/rejected": -7.364126682281494, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0768, |
| "grad_norm": 1.6280964612960815, |
| "learning_rate": 3.718286216278143e-06, |
| "logits/chosen": -1.7556006908416748, |
| "logits/rejected": -1.5969337224960327, |
| "logps/chosen": -231.25070190429688, |
| "logps/rejected": -342.3904113769531, |
| "loss": 0.1556, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.6533854007720947, |
| "rewards/margins": 6.318906307220459, |
| "rewards/rejected": -7.972291469573975, |
| "step": 505 |
| }, |
| { |
| "epoch": 1.0874666666666666, |
| "grad_norm": 4.397036552429199, |
| "learning_rate": 3.6930655729265557e-06, |
| "logits/chosen": -1.8091108798980713, |
| "logits/rejected": -1.620729684829712, |
| "logps/chosen": -224.5769805908203, |
| "logps/rejected": -329.8936462402344, |
| "loss": 0.1047, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1303974390029907, |
| "rewards/margins": 6.186360836029053, |
| "rewards/rejected": -7.316758155822754, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.0981333333333334, |
| "grad_norm": 1.592452049255371, |
| "learning_rate": 3.667686706242386e-06, |
| "logits/chosen": -1.6756534576416016, |
| "logits/rejected": -1.5711702108383179, |
| "logps/chosen": -240.41464233398438, |
| "logps/rejected": -355.96331787109375, |
| "loss": 0.0936, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9966108798980713, |
| "rewards/margins": 6.202914237976074, |
| "rewards/rejected": -8.199524879455566, |
| "step": 515 |
| }, |
| { |
| "epoch": 1.1088, |
| "grad_norm": 4.626035690307617, |
| "learning_rate": 3.6421529819491576e-06, |
| "logits/chosen": -1.7973684072494507, |
| "logits/rejected": -1.6707580089569092, |
| "logps/chosen": -238.1591033935547, |
| "logps/rejected": -337.50616455078125, |
| "loss": 0.1082, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.04890775680542, |
| "rewards/margins": 6.534150123596191, |
| "rewards/rejected": -8.583057403564453, |
| "step": 520 |
| }, |
| { |
| "epoch": 1.1194666666666666, |
| "grad_norm": 1.8899625539779663, |
| "learning_rate": 3.6164677863074783e-06, |
| "logits/chosen": -1.7659199237823486, |
| "logits/rejected": -1.6688506603240967, |
| "logps/chosen": -217.8213653564453, |
| "logps/rejected": -375.4813537597656, |
| "loss": 0.0715, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9252666234970093, |
| "rewards/margins": 7.536079406738281, |
| "rewards/rejected": -9.461346626281738, |
| "step": 525 |
| }, |
| { |
| "epoch": 1.1301333333333332, |
| "grad_norm": 3.0119969844818115, |
| "learning_rate": 3.590634525665955e-06, |
| "logits/chosen": -1.7619727849960327, |
| "logits/rejected": -1.6695072650909424, |
| "logps/chosen": -217.845703125, |
| "logps/rejected": -355.73760986328125, |
| "loss": 0.1189, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.835972547531128, |
| "rewards/margins": 6.8431878089904785, |
| "rewards/rejected": -8.679161071777344, |
| "step": 530 |
| }, |
| { |
| "epoch": 1.1408, |
| "grad_norm": 3.917595148086548, |
| "learning_rate": 3.564656626009446e-06, |
| "logits/chosen": -1.8848800659179688, |
| "logits/rejected": -1.6976979970932007, |
| "logps/chosen": -215.7821807861328, |
| "logps/rejected": -328.48944091796875, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.798927903175354, |
| "rewards/margins": 6.75002908706665, |
| "rewards/rejected": -8.548957824707031, |
| "step": 535 |
| }, |
| { |
| "epoch": 1.1514666666666666, |
| "grad_norm": 0.4280731976032257, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.8062047958374023, |
| "logits/rejected": -1.6663563251495361, |
| "logps/chosen": -228.35617065429688, |
| "logps/rejected": -342.6147155761719, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6262073516845703, |
| "rewards/margins": 6.9966301918029785, |
| "rewards/rejected": -8.62283706665039, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.1621333333333332, |
| "grad_norm": 4.61460542678833, |
| "learning_rate": 3.512280709043534e-06, |
| "logits/chosen": -1.7144701480865479, |
| "logits/rejected": -1.6076030731201172, |
| "logps/chosen": -225.5585174560547, |
| "logps/rejected": -331.7484130859375, |
| "loss": 0.1372, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.4640800952911377, |
| "rewards/margins": 6.167746067047119, |
| "rewards/rejected": -7.631825923919678, |
| "step": 545 |
| }, |
| { |
| "epoch": 1.1728, |
| "grad_norm": 1.0511789321899414, |
| "learning_rate": 3.4858896377832966e-06, |
| "logits/chosen": -1.7985868453979492, |
| "logits/rejected": -1.7188758850097656, |
| "logps/chosen": -213.89755249023438, |
| "logps/rejected": -339.4588928222656, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4908698797225952, |
| "rewards/margins": 6.6824951171875, |
| "rewards/rejected": -8.173364639282227, |
| "step": 550 |
| }, |
| { |
| "epoch": 1.1834666666666667, |
| "grad_norm": 3.18619441986084, |
| "learning_rate": 3.459367818685231e-06, |
| "logits/chosen": -1.767228364944458, |
| "logits/rejected": -1.650066614151001, |
| "logps/chosen": -237.79940795898438, |
| "logps/rejected": -361.7818908691406, |
| "loss": 0.1198, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.48301100730896, |
| "rewards/margins": 6.895485877990723, |
| "rewards/rejected": -8.378497123718262, |
| "step": 555 |
| }, |
| { |
| "epoch": 1.1941333333333333, |
| "grad_norm": 1.3614966869354248, |
| "learning_rate": 3.432718769050228e-06, |
| "logits/chosen": -1.614767074584961, |
| "logits/rejected": -1.5364725589752197, |
| "logps/chosen": -227.3282470703125, |
| "logps/rejected": -329.19873046875, |
| "loss": 0.1329, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.5325191020965576, |
| "rewards/margins": 6.505125522613525, |
| "rewards/rejected": -8.03764533996582, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.2048, |
| "grad_norm": 4.426132678985596, |
| "learning_rate": 3.405946023052386e-06, |
| "logits/chosen": -1.8132712841033936, |
| "logits/rejected": -1.7653039693832397, |
| "logps/chosen": -215.24392700195312, |
| "logps/rejected": -344.3136901855469, |
| "loss": 0.1388, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.5404421091079712, |
| "rewards/margins": 6.6653618812561035, |
| "rewards/rejected": -8.205804824829102, |
| "step": 565 |
| }, |
| { |
| "epoch": 1.2154666666666667, |
| "grad_norm": 3.402472496032715, |
| "learning_rate": 3.3790531312703055e-06, |
| "logits/chosen": -1.7687180042266846, |
| "logits/rejected": -1.6483733654022217, |
| "logps/chosen": -212.26132202148438, |
| "logps/rejected": -352.9338684082031, |
| "loss": 0.0937, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.4449779987335205, |
| "rewards/margins": 6.969573974609375, |
| "rewards/rejected": -8.414551734924316, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.2261333333333333, |
| "grad_norm": 2.1326146125793457, |
| "learning_rate": 3.35204366021622e-06, |
| "logits/chosen": -1.6894903182983398, |
| "logits/rejected": -1.5975654125213623, |
| "logps/chosen": -215.57028198242188, |
| "logps/rejected": -344.5772399902344, |
| "loss": 0.1223, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.0661876201629639, |
| "rewards/margins": 6.815348148345947, |
| "rewards/rejected": -7.881535530090332, |
| "step": 575 |
| }, |
| { |
| "epoch": 1.2368000000000001, |
| "grad_norm": 1.604225993156433, |
| "learning_rate": 3.3249211918630055e-06, |
| "logits/chosen": -1.7422653436660767, |
| "logits/rejected": -1.6558812856674194, |
| "logps/chosen": -211.7191162109375, |
| "logps/rejected": -320.58526611328125, |
| "loss": 0.1275, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.1895016431808472, |
| "rewards/margins": 6.5496721267700195, |
| "rewards/rejected": -7.739173889160156, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.2474666666666667, |
| "grad_norm": 2.2764594554901123, |
| "learning_rate": 3.2976893231691424e-06, |
| "logits/chosen": -1.7650436162948608, |
| "logits/rejected": -1.6200859546661377, |
| "logps/chosen": -233.5091552734375, |
| "logps/rejected": -338.52587890625, |
| "loss": 0.0806, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9783912897109985, |
| "rewards/margins": 7.381894111633301, |
| "rewards/rejected": -8.360284805297852, |
| "step": 585 |
| }, |
| { |
| "epoch": 1.2581333333333333, |
| "grad_norm": 4.1610636711120605, |
| "learning_rate": 3.270351665601691e-06, |
| "logits/chosen": -1.7916901111602783, |
| "logits/rejected": -1.7062695026397705, |
| "logps/chosen": -219.70193481445312, |
| "logps/rejected": -349.7830810546875, |
| "loss": 0.1074, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0305535793304443, |
| "rewards/margins": 6.8783745765686035, |
| "rewards/rejected": -7.908926963806152, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.2688, |
| "grad_norm": 1.4349570274353027, |
| "learning_rate": 3.2429118446573417e-06, |
| "logits/chosen": -1.75607430934906, |
| "logits/rejected": -1.6087610721588135, |
| "logps/chosen": -225.0104522705078, |
| "logps/rejected": -354.34979248046875, |
| "loss": 0.0973, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2995376586914062, |
| "rewards/margins": 6.823355197906494, |
| "rewards/rejected": -8.122893333435059, |
| "step": 595 |
| }, |
| { |
| "epoch": 1.2794666666666665, |
| "grad_norm": 5.035015106201172, |
| "learning_rate": 3.215373499381602e-06, |
| "logits/chosen": -1.6800158023834229, |
| "logits/rejected": -1.512091875076294, |
| "logps/chosen": -239.14111328125, |
| "logps/rejected": -337.6920166015625, |
| "loss": 0.1018, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6130025386810303, |
| "rewards/margins": 6.840310096740723, |
| "rewards/rejected": -8.453312873840332, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.2901333333333334, |
| "grad_norm": 1.7663006782531738, |
| "learning_rate": 3.1877402818861954e-06, |
| "logits/chosen": -1.7800567150115967, |
| "logits/rejected": -1.710240364074707, |
| "logps/chosen": -242.6155242919922, |
| "logps/rejected": -339.5917663574219, |
| "loss": 0.092, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8466907739639282, |
| "rewards/margins": 6.454521179199219, |
| "rewards/rejected": -8.3012113571167, |
| "step": 605 |
| }, |
| { |
| "epoch": 1.3008, |
| "grad_norm": 5.838703632354736, |
| "learning_rate": 3.1600158568647145e-06, |
| "logits/chosen": -1.7841325998306274, |
| "logits/rejected": -1.6554603576660156, |
| "logps/chosen": -235.1628875732422, |
| "logps/rejected": -345.6203918457031, |
| "loss": 0.1131, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.5866191387176514, |
| "rewards/margins": 7.077944755554199, |
| "rewards/rejected": -8.66456413269043, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.3114666666666666, |
| "grad_norm": 2.592312812805176, |
| "learning_rate": 3.1322039011066172e-06, |
| "logits/chosen": -1.7783803939819336, |
| "logits/rejected": -1.6592767238616943, |
| "logps/chosen": -243.6439208984375, |
| "logps/rejected": -343.04833984375, |
| "loss": 0.1224, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3051066398620605, |
| "rewards/margins": 6.644067287445068, |
| "rewards/rejected": -7.949173927307129, |
| "step": 615 |
| }, |
| { |
| "epoch": 1.3221333333333334, |
| "grad_norm": 1.8750396966934204, |
| "learning_rate": 3.1043081030096125e-06, |
| "logits/chosen": -1.6340547800064087, |
| "logits/rejected": -1.5600389242172241, |
| "logps/chosen": -222.5189971923828, |
| "logps/rejected": -371.560302734375, |
| "loss": 0.0874, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6143547296524048, |
| "rewards/margins": 7.084819793701172, |
| "rewards/rejected": -8.699174880981445, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.3328, |
| "grad_norm": 5.306309223175049, |
| "learning_rate": 3.0763321620905072e-06, |
| "logits/chosen": -1.7420177459716797, |
| "logits/rejected": -1.5960488319396973, |
| "logps/chosen": -222.31021118164062, |
| "logps/rejected": -338.64886474609375, |
| "loss": 0.1359, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.4612404108047485, |
| "rewards/margins": 7.118841648101807, |
| "rewards/rejected": -8.58008098602295, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.3434666666666666, |
| "grad_norm": 3.0806684494018555, |
| "learning_rate": 3.048279788494583e-06, |
| "logits/chosen": -1.7452303171157837, |
| "logits/rejected": -1.6031157970428467, |
| "logps/chosen": -227.9959716796875, |
| "logps/rejected": -320.9499206542969, |
| "loss": 0.1244, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.5490151643753052, |
| "rewards/margins": 6.89993143081665, |
| "rewards/rejected": -8.448946952819824, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.3541333333333334, |
| "grad_norm": 2.2955572605133057, |
| "learning_rate": 3.0201547025035553e-06, |
| "logits/chosen": -1.7628275156021118, |
| "logits/rejected": -1.6372320652008057, |
| "logps/chosen": -230.3760528564453, |
| "logps/rejected": -343.8782653808594, |
| "loss": 0.1286, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.40870201587677, |
| "rewards/margins": 7.246272087097168, |
| "rewards/rejected": -8.654973983764648, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.3648, |
| "grad_norm": 2.1225922107696533, |
| "learning_rate": 2.9919606340421963e-06, |
| "logits/chosen": -1.7855663299560547, |
| "logits/rejected": -1.5720247030258179, |
| "logps/chosen": -227.51504516601562, |
| "logps/rejected": -342.0578308105469, |
| "loss": 0.1126, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.6880676746368408, |
| "rewards/margins": 6.570213317871094, |
| "rewards/rejected": -8.258280754089355, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.3754666666666666, |
| "grad_norm": 2.0824544429779053, |
| "learning_rate": 2.9637013221836723e-06, |
| "logits/chosen": -1.7979755401611328, |
| "logits/rejected": -1.6156142950057983, |
| "logps/chosen": -235.97024536132812, |
| "logps/rejected": -367.8596496582031, |
| "loss": 0.0867, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.5493922233581543, |
| "rewards/margins": 7.716729640960693, |
| "rewards/rejected": -9.266121864318848, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.3861333333333334, |
| "grad_norm": 3.655622959136963, |
| "learning_rate": 2.9353805146536762e-06, |
| "logits/chosen": -1.6617200374603271, |
| "logits/rejected": -1.6281896829605103, |
| "logps/chosen": -220.10995483398438, |
| "logps/rejected": -348.8759765625, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.618729829788208, |
| "rewards/margins": 6.909787654876709, |
| "rewards/rejected": -8.52851676940918, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.3968, |
| "grad_norm": 4.440053939819336, |
| "learning_rate": 2.907001967333401e-06, |
| "logits/chosen": -1.8161550760269165, |
| "logits/rejected": -1.7380571365356445, |
| "logps/chosen": -221.9119110107422, |
| "logps/rejected": -358.5216064453125, |
| "loss": 0.1167, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8026584386825562, |
| "rewards/margins": 7.0366363525390625, |
| "rewards/rejected": -8.83929443359375, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.4074666666666666, |
| "grad_norm": 1.1048275232315063, |
| "learning_rate": 2.878569443761442e-06, |
| "logits/chosen": -1.7552165985107422, |
| "logits/rejected": -1.6000335216522217, |
| "logps/chosen": -209.29592895507812, |
| "logps/rejected": -341.8609619140625, |
| "loss": 0.0867, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1091715097427368, |
| "rewards/margins": 7.496683597564697, |
| "rewards/rejected": -8.605854988098145, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.4181333333333335, |
| "grad_norm": 3.342843532562256, |
| "learning_rate": 2.85008671463468e-06, |
| "logits/chosen": -1.7816989421844482, |
| "logits/rejected": -1.6939798593521118, |
| "logps/chosen": -205.90310668945312, |
| "logps/rejected": -349.85711669921875, |
| "loss": 0.126, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.2616592645645142, |
| "rewards/margins": 7.516135215759277, |
| "rewards/rejected": -8.777795791625977, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.4288, |
| "grad_norm": 4.1771697998046875, |
| "learning_rate": 2.8215575573082127e-06, |
| "logits/chosen": -1.7190008163452148, |
| "logits/rejected": -1.6227375268936157, |
| "logps/chosen": -230.20742797851562, |
| "logps/rejected": -360.1850280761719, |
| "loss": 0.0662, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6839618682861328, |
| "rewards/margins": 7.761228084564209, |
| "rewards/rejected": -9.445191383361816, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.4394666666666667, |
| "grad_norm": 2.271709442138672, |
| "learning_rate": 2.792985755294404e-06, |
| "logits/chosen": -1.8138444423675537, |
| "logits/rejected": -1.6258472204208374, |
| "logps/chosen": -222.8135986328125, |
| "logps/rejected": -354.85980224609375, |
| "loss": 0.0697, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8773260116577148, |
| "rewards/margins": 7.912613868713379, |
| "rewards/rejected": -9.789939880371094, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.4501333333333333, |
| "grad_norm": 1.3745611906051636, |
| "learning_rate": 2.764375097761122e-06, |
| "logits/chosen": -1.6466379165649414, |
| "logits/rejected": -1.5347976684570312, |
| "logps/chosen": -214.34341430664062, |
| "logps/rejected": -339.7757873535156, |
| "loss": 0.0986, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.489703893661499, |
| "rewards/margins": 7.611210823059082, |
| "rewards/rejected": -9.10091495513916, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.4607999999999999, |
| "grad_norm": 3.7492542266845703, |
| "learning_rate": 2.73572937902922e-06, |
| "logits/chosen": -1.737330436706543, |
| "logits/rejected": -1.6359955072402954, |
| "logps/chosen": -250.636474609375, |
| "logps/rejected": -355.47625732421875, |
| "loss": 0.125, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7210981845855713, |
| "rewards/margins": 6.998046875, |
| "rewards/rejected": -8.719144821166992, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.4714666666666667, |
| "grad_norm": 3.4588992595672607, |
| "learning_rate": 2.7070523980693362e-06, |
| "logits/chosen": -1.556281328201294, |
| "logits/rejected": -1.4768974781036377, |
| "logps/chosen": -238.09829711914062, |
| "logps/rejected": -339.5543518066406, |
| "loss": 0.1122, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.520418405532837, |
| "rewards/margins": 7.163352966308594, |
| "rewards/rejected": -8.683772087097168, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.4821333333333333, |
| "grad_norm": 1.5479131937026978, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.7872568368911743, |
| "logits/rejected": -1.61525559425354, |
| "logps/chosen": -230.7522735595703, |
| "logps/rejected": -361.02545166015625, |
| "loss": 0.0637, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6853501796722412, |
| "rewards/margins": 8.169617652893066, |
| "rewards/rejected": -9.85496711730957, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.4928, |
| "grad_norm": 1.234351396560669, |
| "learning_rate": 2.649619865573666e-06, |
| "logits/chosen": -1.7372156381607056, |
| "logits/rejected": -1.6385151147842407, |
| "logps/chosen": -227.90835571289062, |
| "logps/rejected": -367.4066467285156, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9077075719833374, |
| "rewards/margins": 7.612782955169678, |
| "rewards/rejected": -9.520490646362305, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.5034666666666667, |
| "grad_norm": 2.3628101348876953, |
| "learning_rate": 2.620871930691058e-06, |
| "logits/chosen": -1.7833808660507202, |
| "logits/rejected": -1.695267915725708, |
| "logps/chosen": -227.47909545898438, |
| "logps/rejected": -361.0234069824219, |
| "loss": 0.0975, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7123091220855713, |
| "rewards/margins": 8.293615341186523, |
| "rewards/rejected": -10.005925178527832, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.5141333333333333, |
| "grad_norm": 4.18447732925415, |
| "learning_rate": 2.5921079658767152e-06, |
| "logits/chosen": -1.7901111841201782, |
| "logits/rejected": -1.669463872909546, |
| "logps/chosen": -228.15188598632812, |
| "logps/rejected": -352.96795654296875, |
| "loss": 0.2078, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.9093815088272095, |
| "rewards/margins": 7.494314670562744, |
| "rewards/rejected": -9.403696060180664, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.5248, |
| "grad_norm": 2.716040849685669, |
| "learning_rate": 2.56333178578297e-06, |
| "logits/chosen": -1.76180100440979, |
| "logits/rejected": -1.636069655418396, |
| "logps/chosen": -261.0120544433594, |
| "logps/rejected": -376.29010009765625, |
| "loss": 0.078, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8959649801254272, |
| "rewards/margins": 7.93648624420166, |
| "rewards/rejected": -9.832451820373535, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.5354666666666668, |
| "grad_norm": 4.260155200958252, |
| "learning_rate": 2.534547206682136e-06, |
| "logits/chosen": -1.8289260864257812, |
| "logits/rejected": -1.6811158657073975, |
| "logps/chosen": -225.0589599609375, |
| "logps/rejected": -378.1023254394531, |
| "loss": 0.0619, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.8060722351074219, |
| "rewards/margins": 8.63095474243164, |
| "rewards/rejected": -10.437026977539062, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.5461333333333334, |
| "grad_norm": 2.0456085205078125, |
| "learning_rate": 2.5057580459603947e-06, |
| "logits/chosen": -1.8401775360107422, |
| "logits/rejected": -1.7199382781982422, |
| "logps/chosen": -222.3778533935547, |
| "logps/rejected": -359.7654724121094, |
| "loss": 0.0655, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9060947895050049, |
| "rewards/margins": 7.709580421447754, |
| "rewards/rejected": -9.615674018859863, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.5568, |
| "grad_norm": 3.1641626358032227, |
| "learning_rate": 2.4769681216115384e-06, |
| "logits/chosen": -1.7817308902740479, |
| "logits/rejected": -1.650779128074646, |
| "logps/chosen": -219.2471466064453, |
| "logps/rejected": -357.71612548828125, |
| "loss": 0.0896, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5718821287155151, |
| "rewards/margins": 8.106928825378418, |
| "rewards/rejected": -9.678812026977539, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.5674666666666668, |
| "grad_norm": 3.828000545501709, |
| "learning_rate": 2.448181251730632e-06, |
| "logits/chosen": -1.7795158624649048, |
| "logits/rejected": -1.5841357707977295, |
| "logps/chosen": -220.0762939453125, |
| "logps/rejected": -342.20758056640625, |
| "loss": 0.1752, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.8566948175430298, |
| "rewards/margins": 6.9646759033203125, |
| "rewards/rejected": -8.821371078491211, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.5781333333333334, |
| "grad_norm": 6.140118598937988, |
| "learning_rate": 2.419401254007658e-06, |
| "logits/chosen": -1.6956440210342407, |
| "logits/rejected": -1.5929274559020996, |
| "logps/chosen": -247.3740997314453, |
| "logps/rejected": -374.37677001953125, |
| "loss": 0.1017, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.1039836406707764, |
| "rewards/margins": 7.365347385406494, |
| "rewards/rejected": -9.469330787658691, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.5888, |
| "grad_norm": 1.6513922214508057, |
| "learning_rate": 2.390631945221221e-06, |
| "logits/chosen": -1.8118445873260498, |
| "logits/rejected": -1.636195182800293, |
| "logps/chosen": -215.1251678466797, |
| "logps/rejected": -344.25689697265625, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.4001673460006714, |
| "rewards/margins": 8.314090728759766, |
| "rewards/rejected": -9.714258193969727, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.5994666666666668, |
| "grad_norm": 5.196778297424316, |
| "learning_rate": 2.361877140732366e-06, |
| "logits/chosen": -1.842757225036621, |
| "logits/rejected": -1.7138402462005615, |
| "logps/chosen": -217.3812255859375, |
| "logps/rejected": -351.89825439453125, |
| "loss": 0.0958, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4169294834136963, |
| "rewards/margins": 7.893444061279297, |
| "rewards/rejected": -9.310373306274414, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.6101333333333332, |
| "grad_norm": 3.4266786575317383, |
| "learning_rate": 2.333140653978592e-06, |
| "logits/chosen": -1.7201732397079468, |
| "logits/rejected": -1.6220394372940063, |
| "logps/chosen": -218.4654083251953, |
| "logps/rejected": -345.9354248046875, |
| "loss": 0.1068, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8092172145843506, |
| "rewards/margins": 6.817082405090332, |
| "rewards/rejected": -8.626298904418945, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.6208, |
| "grad_norm": 2.9891090393066406, |
| "learning_rate": 2.3044262959681156e-06, |
| "logits/chosen": -1.7339553833007812, |
| "logits/rejected": -1.6223846673965454, |
| "logps/chosen": -207.73974609375, |
| "logps/rejected": -361.72967529296875, |
| "loss": 0.098, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.3607968091964722, |
| "rewards/margins": 7.7453179359436035, |
| "rewards/rejected": -9.106114387512207, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.6314666666666666, |
| "grad_norm": 5.372440814971924, |
| "learning_rate": 2.2757378747744575e-06, |
| "logits/chosen": -1.8800748586654663, |
| "logits/rejected": -1.6330455541610718, |
| "logps/chosen": -214.36270141601562, |
| "logps/rejected": -335.51654052734375, |
| "loss": 0.0943, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.389737844467163, |
| "rewards/margins": 7.517597198486328, |
| "rewards/rejected": -8.90733528137207, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.6421333333333332, |
| "grad_norm": 4.0247087478637695, |
| "learning_rate": 2.24707919503142e-06, |
| "logits/chosen": -1.8723796606063843, |
| "logits/rejected": -1.7164814472198486, |
| "logps/chosen": -234.10498046875, |
| "logps/rejected": -356.8775634765625, |
| "loss": 0.1229, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.514541506767273, |
| "rewards/margins": 7.490867614746094, |
| "rewards/rejected": -9.00540828704834, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.6528, |
| "grad_norm": 2.84824538230896, |
| "learning_rate": 2.2184540574285213e-06, |
| "logits/chosen": -1.681469202041626, |
| "logits/rejected": -1.5739465951919556, |
| "logps/chosen": -256.971435546875, |
| "logps/rejected": -337.6930236816406, |
| "loss": 0.1087, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6835079193115234, |
| "rewards/margins": 6.543093204498291, |
| "rewards/rejected": -8.226600646972656, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.6634666666666666, |
| "grad_norm": 4.553780555725098, |
| "learning_rate": 2.1898662582069486e-06, |
| "logits/chosen": -1.7528133392333984, |
| "logits/rejected": -1.6476829051971436, |
| "logps/chosen": -220.4532470703125, |
| "logps/rejected": -357.43438720703125, |
| "loss": 0.1319, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.636549949645996, |
| "rewards/margins": 7.442874908447266, |
| "rewards/rejected": -9.079424858093262, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.6741333333333333, |
| "grad_norm": 1.5032289028167725, |
| "learning_rate": 2.1613195886561044e-06, |
| "logits/chosen": -1.7787612676620483, |
| "logits/rejected": -1.6489241123199463, |
| "logps/chosen": -207.38650512695312, |
| "logps/rejected": -330.9730224609375, |
| "loss": 0.1168, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3346378803253174, |
| "rewards/margins": 7.285429954528809, |
| "rewards/rejected": -8.620067596435547, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.6848, |
| "grad_norm": 6.126774311065674, |
| "learning_rate": 2.132817834610811e-06, |
| "logits/chosen": -1.711124062538147, |
| "logits/rejected": -1.619384527206421, |
| "logps/chosen": -229.5834503173828, |
| "logps/rejected": -359.8412170410156, |
| "loss": 0.1126, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.6115143299102783, |
| "rewards/margins": 6.954829216003418, |
| "rewards/rejected": -8.566343307495117, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.6954666666666667, |
| "grad_norm": 2.0657472610473633, |
| "learning_rate": 2.1043647759492338e-06, |
| "logits/chosen": -1.6726573705673218, |
| "logits/rejected": -1.5993225574493408, |
| "logps/chosen": -223.4843292236328, |
| "logps/rejected": -328.24700927734375, |
| "loss": 0.0887, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.9438800811767578, |
| "rewards/margins": 7.049546241760254, |
| "rewards/rejected": -7.993424892425537, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.7061333333333333, |
| "grad_norm": 2.5707919597625732, |
| "learning_rate": 2.075964186091599e-06, |
| "logits/chosen": -1.8048022985458374, |
| "logits/rejected": -1.7015832662582397, |
| "logps/chosen": -216.2055206298828, |
| "logps/rejected": -342.51287841796875, |
| "loss": 0.1203, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.4331550598144531, |
| "rewards/margins": 7.590814113616943, |
| "rewards/rejected": -9.023968696594238, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.7168, |
| "grad_norm": 2.875871419906616, |
| "learning_rate": 2.047619831499765e-06, |
| "logits/chosen": -1.6987117528915405, |
| "logits/rejected": -1.5893299579620361, |
| "logps/chosen": -219.29531860351562, |
| "logps/rejected": -340.4369812011719, |
| "loss": 0.11, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.1939778327941895, |
| "rewards/margins": 6.53455114364624, |
| "rewards/rejected": -7.728528022766113, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.7274666666666667, |
| "grad_norm": 1.0247174501419067, |
| "learning_rate": 2.0193354711777155e-06, |
| "logits/chosen": -1.8061132431030273, |
| "logits/rejected": -1.6443469524383545, |
| "logps/chosen": -244.94760131835938, |
| "logps/rejected": -349.5795593261719, |
| "loss": 0.0906, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.3361376523971558, |
| "rewards/margins": 6.910301208496094, |
| "rewards/rejected": -8.246439933776855, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.7381333333333333, |
| "grad_norm": 0.9948698282241821, |
| "learning_rate": 1.991114856173048e-06, |
| "logits/chosen": -1.8769338130950928, |
| "logits/rejected": -1.70797598361969, |
| "logps/chosen": -210.803466796875, |
| "logps/rejected": -338.79327392578125, |
| "loss": 0.0605, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.4188705682754517, |
| "rewards/margins": 7.125912666320801, |
| "rewards/rejected": -8.544782638549805, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.7488000000000001, |
| "grad_norm": 1.602662444114685, |
| "learning_rate": 1.9629617290795115e-06, |
| "logits/chosen": -1.7712066173553467, |
| "logits/rejected": -1.6187121868133545, |
| "logps/chosen": -215.338623046875, |
| "logps/rejected": -341.73583984375, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.4782660007476807, |
| "rewards/margins": 7.113155364990234, |
| "rewards/rejected": -8.591421127319336, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.7594666666666665, |
| "grad_norm": 2.919387102127075, |
| "learning_rate": 1.934879823540663e-06, |
| "logits/chosen": -1.8011865615844727, |
| "logits/rejected": -1.6715080738067627, |
| "logps/chosen": -219.78775024414062, |
| "logps/rejected": -354.23187255859375, |
| "loss": 0.1081, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.5675954818725586, |
| "rewards/margins": 6.86489725112915, |
| "rewards/rejected": -8.432493209838867, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.7701333333333333, |
| "grad_norm": 2.401257276535034, |
| "learning_rate": 1.9068728637547195e-06, |
| "logits/chosen": -1.76266348361969, |
| "logits/rejected": -1.594229817390442, |
| "logps/chosen": -234.4385223388672, |
| "logps/rejected": -358.3901672363281, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7133439779281616, |
| "rewards/margins": 7.900755882263184, |
| "rewards/rejected": -9.614099502563477, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.7808000000000002, |
| "grad_norm": 1.2106441259384155, |
| "learning_rate": 1.878944563980658e-06, |
| "logits/chosen": -1.7576103210449219, |
| "logits/rejected": -1.557202935218811, |
| "logps/chosen": -218.30282592773438, |
| "logps/rejected": -352.62884521484375, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.431818962097168, |
| "rewards/margins": 7.726027488708496, |
| "rewards/rejected": -9.157846450805664, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.7914666666666665, |
| "grad_norm": 3.5941357612609863, |
| "learning_rate": 1.85109862804563e-06, |
| "logits/chosen": -1.754690408706665, |
| "logits/rejected": -1.6672760248184204, |
| "logps/chosen": -225.74581909179688, |
| "logps/rejected": -354.9847717285156, |
| "loss": 0.1013, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6395050287246704, |
| "rewards/margins": 7.6984686851501465, |
| "rewards/rejected": -9.337973594665527, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.8021333333333334, |
| "grad_norm": 2.2285900115966797, |
| "learning_rate": 1.823338748853766e-06, |
| "logits/chosen": -1.7495044469833374, |
| "logits/rejected": -1.7088682651519775, |
| "logps/chosen": -218.03775024414062, |
| "logps/rejected": -378.6893615722656, |
| "loss": 0.089, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9602687358856201, |
| "rewards/margins": 8.054302215576172, |
| "rewards/rejected": -10.014570236206055, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.8128, |
| "grad_norm": 3.8862228393554688, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.7476335763931274, |
| "logits/rejected": -1.646803855895996, |
| "logps/chosen": -248.8168487548828, |
| "logps/rejected": -350.89288330078125, |
| "loss": 0.0824, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6752967834472656, |
| "rewards/margins": 7.301435947418213, |
| "rewards/rejected": -8.97673225402832, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.8234666666666666, |
| "grad_norm": 3.172503709793091, |
| "learning_rate": 1.7680918747639608e-06, |
| "logits/chosen": -1.8387765884399414, |
| "logits/rejected": -1.7114694118499756, |
| "logps/chosen": -238.186279296875, |
| "logps/rejected": -367.3875732421875, |
| "loss": 0.0701, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6962471008300781, |
| "rewards/margins": 8.61371898651123, |
| "rewards/rejected": -10.309966087341309, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.8341333333333334, |
| "grad_norm": 4.591922283172607, |
| "learning_rate": 1.740612206659057e-06, |
| "logits/chosen": -1.7826566696166992, |
| "logits/rejected": -1.6185051202774048, |
| "logps/chosen": -219.8805389404297, |
| "logps/rejected": -355.91619873046875, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.4452508687973022, |
| "rewards/margins": 7.93267297744751, |
| "rewards/rejected": -9.377923965454102, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.8448, |
| "grad_norm": 3.7183308601379395, |
| "learning_rate": 1.7132332479117204e-06, |
| "logits/chosen": -1.8160545825958252, |
| "logits/rejected": -1.6184918880462646, |
| "logps/chosen": -244.14889526367188, |
| "logps/rejected": -360.9537658691406, |
| "loss": 0.0633, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.8190085887908936, |
| "rewards/margins": 7.7830305099487305, |
| "rewards/rejected": -9.602039337158203, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.8554666666666666, |
| "grad_norm": 1.1061958074569702, |
| "learning_rate": 1.6859586294959666e-06, |
| "logits/chosen": -1.7806711196899414, |
| "logits/rejected": -1.6613991260528564, |
| "logps/chosen": -222.4357452392578, |
| "logps/rejected": -371.41986083984375, |
| "loss": 0.0842, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6314767599105835, |
| "rewards/margins": 8.27884292602539, |
| "rewards/rejected": -9.910319328308105, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.8661333333333334, |
| "grad_norm": 3.018303394317627, |
| "learning_rate": 1.6587919685482884e-06, |
| "logits/chosen": -1.7951326370239258, |
| "logits/rejected": -1.6616239547729492, |
| "logps/chosen": -203.40164184570312, |
| "logps/rejected": -358.426025390625, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7556968927383423, |
| "rewards/margins": 7.818473815917969, |
| "rewards/rejected": -9.57417106628418, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.8768, |
| "grad_norm": 1.363223671913147, |
| "learning_rate": 1.6317368678879497e-06, |
| "logits/chosen": -1.651288390159607, |
| "logits/rejected": -1.5710906982421875, |
| "logps/chosen": -240.0145721435547, |
| "logps/rejected": -357.17718505859375, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6783626079559326, |
| "rewards/margins": 7.36400842666626, |
| "rewards/rejected": -9.042369842529297, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.8874666666666666, |
| "grad_norm": 1.239745020866394, |
| "learning_rate": 1.6047969155391856e-06, |
| "logits/chosen": -1.6190026998519897, |
| "logits/rejected": -1.5436100959777832, |
| "logps/chosen": -246.00247192382812, |
| "logps/rejected": -362.49517822265625, |
| "loss": 0.0564, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.6668789386749268, |
| "rewards/margins": 7.92090368270874, |
| "rewards/rejected": -9.58778190612793, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.8981333333333335, |
| "grad_norm": 1.8271006345748901, |
| "learning_rate": 1.5779756842553617e-06, |
| "logits/chosen": -1.6952009201049805, |
| "logits/rejected": -1.6505413055419922, |
| "logps/chosen": -236.0554656982422, |
| "logps/rejected": -362.10986328125, |
| "loss": 0.0509, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.7946186065673828, |
| "rewards/margins": 7.933407783508301, |
| "rewards/rejected": -9.728026390075684, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.9088, |
| "grad_norm": 1.3855539560317993, |
| "learning_rate": 1.551276731045157e-06, |
| "logits/chosen": -1.7061758041381836, |
| "logits/rejected": -1.6307599544525146, |
| "logps/chosen": -242.96566772460938, |
| "logps/rejected": -367.9177551269531, |
| "loss": 0.0802, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6677072048187256, |
| "rewards/margins": 7.688159942626953, |
| "rewards/rejected": -9.355867385864258, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.9194666666666667, |
| "grad_norm": 6.346963882446289, |
| "learning_rate": 1.5247035967008369e-06, |
| "logits/chosen": -1.6322176456451416, |
| "logits/rejected": -1.5739306211471558, |
| "logps/chosen": -236.05166625976562, |
| "logps/rejected": -348.62255859375, |
| "loss": 0.1068, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.8375927209854126, |
| "rewards/margins": 7.27393102645874, |
| "rewards/rejected": -9.111523628234863, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9301333333333335, |
| "grad_norm": 2.1471517086029053, |
| "learning_rate": 1.498259805328678e-06, |
| "logits/chosen": -1.801081657409668, |
| "logits/rejected": -1.564719796180725, |
| "logps/chosen": -227.64669799804688, |
| "logps/rejected": -353.7789306640625, |
| "loss": 0.0755, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.8626874685287476, |
| "rewards/margins": 8.565693855285645, |
| "rewards/rejected": -10.428380966186523, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.9407999999999999, |
| "grad_norm": 1.3048449754714966, |
| "learning_rate": 1.471948863881601e-06, |
| "logits/chosen": -1.810206651687622, |
| "logits/rejected": -1.6128194332122803, |
| "logps/chosen": -240.61782836914062, |
| "logps/rejected": -377.509033203125, |
| "loss": 0.0568, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7367178201675415, |
| "rewards/margins": 8.280123710632324, |
| "rewards/rejected": -10.016841888427734, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.9514666666666667, |
| "grad_norm": 1.4426000118255615, |
| "learning_rate": 1.4457742616940864e-06, |
| "logits/chosen": -1.7471405267715454, |
| "logits/rejected": -1.6417961120605469, |
| "logps/chosen": -225.4718780517578, |
| "logps/rejected": -361.42926025390625, |
| "loss": 0.0985, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6348329782485962, |
| "rewards/margins": 7.608382225036621, |
| "rewards/rejected": -9.243215560913086, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.9621333333333333, |
| "grad_norm": 5.110790729522705, |
| "learning_rate": 1.4197394700194165e-06, |
| "logits/chosen": -1.7448272705078125, |
| "logits/rejected": -1.683083176612854, |
| "logps/chosen": -227.5116729736328, |
| "logps/rejected": -361.52020263671875, |
| "loss": 0.0927, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.834454894065857, |
| "rewards/margins": 7.576612949371338, |
| "rewards/rejected": -9.411067008972168, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.9727999999999999, |
| "grad_norm": 4.627553462982178, |
| "learning_rate": 1.3938479415693244e-06, |
| "logits/chosen": -1.8246326446533203, |
| "logits/rejected": -1.7019975185394287, |
| "logps/chosen": -235.0475616455078, |
| "logps/rejected": -341.9778747558594, |
| "loss": 0.0992, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.870369553565979, |
| "rewards/margins": 7.382562160491943, |
| "rewards/rejected": -9.25293254852295, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.9834666666666667, |
| "grad_norm": 2.9364688396453857, |
| "learning_rate": 1.368103110056096e-06, |
| "logits/chosen": -1.7265510559082031, |
| "logits/rejected": -1.5709378719329834, |
| "logps/chosen": -245.30679321289062, |
| "logps/rejected": -353.5377197265625, |
| "loss": 0.0898, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1614131927490234, |
| "rewards/margins": 7.4779791831970215, |
| "rewards/rejected": -9.639391899108887, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.9941333333333333, |
| "grad_norm": 4.901484489440918, |
| "learning_rate": 1.3425083897371983e-06, |
| "logits/chosen": -1.6642370223999023, |
| "logits/rejected": -1.5926437377929688, |
| "logps/chosen": -242.5086212158203, |
| "logps/rejected": -364.00006103515625, |
| "loss": 0.0977, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.8294131755828857, |
| "rewards/margins": 7.299746513366699, |
| "rewards/rejected": -9.12916088104248, |
| "step": 935 |
| }, |
| { |
| "epoch": 2.0042666666666666, |
| "grad_norm": 3.4245495796203613, |
| "learning_rate": 1.317067174962478e-06, |
| "logits/chosen": -1.6907618045806885, |
| "logits/rejected": -1.559260368347168, |
| "logps/chosen": -224.0897979736328, |
| "logps/rejected": -390.0406188964844, |
| "loss": 0.0619, |
| "rewards/accuracies": 0.9736841917037964, |
| "rewards/chosen": -1.7347691059112549, |
| "rewards/margins": 8.899223327636719, |
| "rewards/rejected": -10.633993148803711, |
| "step": 940 |
| }, |
| { |
| "epoch": 2.0149333333333335, |
| "grad_norm": 1.1779319047927856, |
| "learning_rate": 1.291782839724009e-06, |
| "logits/chosen": -1.7825742959976196, |
| "logits/rejected": -1.6565110683441162, |
| "logps/chosen": -235.5923614501953, |
| "logps/rejected": -369.40374755859375, |
| "loss": 0.0604, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9541120529174805, |
| "rewards/margins": 8.030762672424316, |
| "rewards/rejected": -9.98487377166748, |
| "step": 945 |
| }, |
| { |
| "epoch": 2.0256, |
| "grad_norm": 0.507275402545929, |
| "learning_rate": 1.2666587372086352e-06, |
| "logits/chosen": -1.6030664443969727, |
| "logits/rejected": -1.5872256755828857, |
| "logps/chosen": -216.4988555908203, |
| "logps/rejected": -369.16253662109375, |
| "loss": 0.0653, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.8965251445770264, |
| "rewards/margins": 8.338624954223633, |
| "rewards/rejected": -10.235151290893555, |
| "step": 950 |
| }, |
| { |
| "epoch": 2.0362666666666667, |
| "grad_norm": 2.7491443157196045, |
| "learning_rate": 1.241698199353275e-06, |
| "logits/chosen": -1.7347681522369385, |
| "logits/rejected": -1.6534042358398438, |
| "logps/chosen": -231.2009735107422, |
| "logps/rejected": -346.7289733886719, |
| "loss": 0.0913, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2630741596221924, |
| "rewards/margins": 7.0988335609436035, |
| "rewards/rejected": -9.361908912658691, |
| "step": 955 |
| }, |
| { |
| "epoch": 2.0469333333333335, |
| "grad_norm": 2.5662288665771484, |
| "learning_rate": 1.2169045364030404e-06, |
| "logits/chosen": -1.7822414636611938, |
| "logits/rejected": -1.6436115503311157, |
| "logps/chosen": -217.4017333984375, |
| "logps/rejected": -367.7906494140625, |
| "loss": 0.0514, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.031571626663208, |
| "rewards/margins": 8.508852005004883, |
| "rewards/rejected": -10.540423393249512, |
| "step": 960 |
| }, |
| { |
| "epoch": 2.0576, |
| "grad_norm": 3.719102382659912, |
| "learning_rate": 1.1922810364722402e-06, |
| "logits/chosen": -1.659507155418396, |
| "logits/rejected": -1.6028478145599365, |
| "logps/chosen": -251.8856658935547, |
| "logps/rejected": -366.27392578125, |
| "loss": 0.1071, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.9120813608169556, |
| "rewards/margins": 7.532308101654053, |
| "rewards/rejected": -9.444389343261719, |
| "step": 965 |
| }, |
| { |
| "epoch": 2.0682666666666667, |
| "grad_norm": 3.9604787826538086, |
| "learning_rate": 1.1678309651083055e-06, |
| "logits/chosen": -1.7748504877090454, |
| "logits/rejected": -1.6407390832901, |
| "logps/chosen": -232.59933471679688, |
| "logps/rejected": -380.6536865234375, |
| "loss": 0.0581, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3573272228240967, |
| "rewards/margins": 9.171064376831055, |
| "rewards/rejected": -11.52839183807373, |
| "step": 970 |
| }, |
| { |
| "epoch": 2.0789333333333335, |
| "grad_norm": 4.073202133178711, |
| "learning_rate": 1.1435575648587224e-06, |
| "logits/chosen": -1.766287088394165, |
| "logits/rejected": -1.6389284133911133, |
| "logps/chosen": -285.7763671875, |
| "logps/rejected": -341.8055114746094, |
| "loss": 0.1355, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9729318618774414, |
| "rewards/margins": 7.9406914710998535, |
| "rewards/rejected": -9.913623809814453, |
| "step": 975 |
| }, |
| { |
| "epoch": 2.0896, |
| "grad_norm": 2.3515031337738037, |
| "learning_rate": 1.1194640548410039e-06, |
| "logits/chosen": -1.6386430263519287, |
| "logits/rejected": -1.5689029693603516, |
| "logps/chosen": -251.07199096679688, |
| "logps/rejected": -351.345458984375, |
| "loss": 0.0837, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.034928321838379, |
| "rewards/margins": 7.436675071716309, |
| "rewards/rejected": -9.471604347229004, |
| "step": 980 |
| }, |
| { |
| "epoch": 2.1002666666666667, |
| "grad_norm": 1.4685736894607544, |
| "learning_rate": 1.095553630315774e-06, |
| "logits/chosen": -1.6688451766967773, |
| "logits/rejected": -1.5495831966400146, |
| "logps/chosen": -240.5880889892578, |
| "logps/rejected": -336.76947021484375, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.8441340923309326, |
| "rewards/margins": 7.2546892166137695, |
| "rewards/rejected": -9.098822593688965, |
| "step": 985 |
| }, |
| { |
| "epoch": 2.1109333333333336, |
| "grad_norm": 2.042402505874634, |
| "learning_rate": 1.0718294622630188e-06, |
| "logits/chosen": -1.7300097942352295, |
| "logits/rejected": -1.6546669006347656, |
| "logps/chosen": -233.61190795898438, |
| "logps/rejected": -374.1564025878906, |
| "loss": 0.0485, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.8412272930145264, |
| "rewards/margins": 8.404033660888672, |
| "rewards/rejected": -10.245260238647461, |
| "step": 990 |
| }, |
| { |
| "epoch": 2.1216, |
| "grad_norm": 0.9763467311859131, |
| "learning_rate": 1.0482946969615495e-06, |
| "logits/chosen": -1.822035551071167, |
| "logits/rejected": -1.673659086227417, |
| "logps/chosen": -214.61703491210938, |
| "logps/rejected": -368.96636962890625, |
| "loss": 0.0653, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0393638610839844, |
| "rewards/margins": 8.412542343139648, |
| "rewards/rejected": -10.451906204223633, |
| "step": 995 |
| }, |
| { |
| "epoch": 2.1322666666666668, |
| "grad_norm": 0.9107277989387512, |
| "learning_rate": 1.0249524555717461e-06, |
| "logits/chosen": -1.7914142608642578, |
| "logits/rejected": -1.6238925457000732, |
| "logps/chosen": -228.9333038330078, |
| "logps/rejected": -378.4297790527344, |
| "loss": 0.0541, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.8040258884429932, |
| "rewards/margins": 8.715888977050781, |
| "rewards/rejected": -10.519915580749512, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.142933333333333, |
| "grad_norm": 0.46706506609916687, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.6658966541290283, |
| "logits/rejected": -1.629685640335083, |
| "logps/chosen": -225.38327026367188, |
| "logps/rejected": -384.2342224121094, |
| "loss": 0.0488, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.277534008026123, |
| "rewards/margins": 8.842560768127441, |
| "rewards/rejected": -11.120096206665039, |
| "step": 1005 |
| }, |
| { |
| "epoch": 2.1536, |
| "grad_norm": 0.696917712688446, |
| "learning_rate": 9.788579010963387e-07, |
| "logits/chosen": -1.8070281744003296, |
| "logits/rejected": -1.6128984689712524, |
| "logps/chosen": -234.9412078857422, |
| "logps/rejected": -374.0914306640625, |
| "loss": 0.0619, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9849834442138672, |
| "rewards/margins": 8.614242553710938, |
| "rewards/rejected": -10.599225997924805, |
| "step": 1010 |
| }, |
| { |
| "epoch": 2.164266666666667, |
| "grad_norm": 3.232445478439331, |
| "learning_rate": 9.56111701030998e-07, |
| "logits/chosen": -1.7992016077041626, |
| "logits/rejected": -1.6329628229141235, |
| "logps/chosen": -223.1928253173828, |
| "logps/rejected": -355.9103698730469, |
| "loss": 0.0846, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.365701198577881, |
| "rewards/margins": 7.757696628570557, |
| "rewards/rejected": -10.123396873474121, |
| "step": 1015 |
| }, |
| { |
| "epoch": 2.174933333333333, |
| "grad_norm": 2.396444797515869, |
| "learning_rate": 9.335702501071459e-07, |
| "logits/chosen": -1.786289930343628, |
| "logits/rejected": -1.632266640663147, |
| "logps/chosen": -243.3270263671875, |
| "logps/rejected": -354.66802978515625, |
| "loss": 0.0683, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8624069690704346, |
| "rewards/margins": 8.355226516723633, |
| "rewards/rejected": -10.217633247375488, |
| "step": 1020 |
| }, |
| { |
| "epoch": 2.1856, |
| "grad_norm": 4.311215400695801, |
| "learning_rate": 9.112365377526636e-07, |
| "logits/chosen": -1.7827332019805908, |
| "logits/rejected": -1.6588077545166016, |
| "logps/chosen": -219.1776123046875, |
| "logps/rejected": -376.1236572265625, |
| "loss": 0.0534, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.2685389518737793, |
| "rewards/margins": 8.695510864257812, |
| "rewards/rejected": -10.964048385620117, |
| "step": 1025 |
| }, |
| { |
| "epoch": 2.196266666666667, |
| "grad_norm": 1.4199345111846924, |
| "learning_rate": 8.891135258453195e-07, |
| "logits/chosen": -1.6969398260116577, |
| "logits/rejected": -1.6353518962860107, |
| "logps/chosen": -217.14682006835938, |
| "logps/rejected": -352.71136474609375, |
| "loss": 0.0646, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6732425689697266, |
| "rewards/margins": 8.477716445922852, |
| "rewards/rejected": -10.150958061218262, |
| "step": 1030 |
| }, |
| { |
| "epoch": 2.206933333333333, |
| "grad_norm": 1.0060111284255981, |
| "learning_rate": 8.672041483199709e-07, |
| "logits/chosen": -1.7296987771987915, |
| "logits/rejected": -1.6749894618988037, |
| "logps/chosen": -212.9332275390625, |
| "logps/rejected": -356.7399597167969, |
| "loss": 0.0892, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9000186920166016, |
| "rewards/margins": 8.221338272094727, |
| "rewards/rejected": -10.121357917785645, |
| "step": 1035 |
| }, |
| { |
| "epoch": 2.2176, |
| "grad_norm": 1.1529102325439453, |
| "learning_rate": 8.455113107794652e-07, |
| "logits/chosen": -1.7899577617645264, |
| "logits/rejected": -1.684852957725525, |
| "logps/chosen": -208.1083984375, |
| "logps/rejected": -348.6238708496094, |
| "loss": 0.0486, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.258780002593994, |
| "rewards/margins": 8.805041313171387, |
| "rewards/rejected": -11.063821792602539, |
| "step": 1040 |
| }, |
| { |
| "epoch": 2.228266666666667, |
| "grad_norm": 2.7815699577331543, |
| "learning_rate": 8.240378901093035e-07, |
| "logits/chosen": -1.6827160120010376, |
| "logits/rejected": -1.5520236492156982, |
| "logps/chosen": -235.8473358154297, |
| "logps/rejected": -384.362060546875, |
| "loss": 0.075, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.263546943664551, |
| "rewards/margins": 8.731700897216797, |
| "rewards/rejected": -10.995248794555664, |
| "step": 1045 |
| }, |
| { |
| "epoch": 2.238933333333333, |
| "grad_norm": 3.8146376609802246, |
| "learning_rate": 8.027867340961113e-07, |
| "logits/chosen": -1.6788311004638672, |
| "logits/rejected": -1.5772473812103271, |
| "logps/chosen": -233.0138397216797, |
| "logps/rejected": -352.76666259765625, |
| "loss": 0.0815, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.020775318145752, |
| "rewards/margins": 8.058589935302734, |
| "rewards/rejected": -10.079363822937012, |
| "step": 1050 |
| }, |
| { |
| "epoch": 2.2496, |
| "grad_norm": 4.936167240142822, |
| "learning_rate": 7.817606610499639e-07, |
| "logits/chosen": -1.7382968664169312, |
| "logits/rejected": -1.6289379596710205, |
| "logps/chosen": -243.76571655273438, |
| "logps/rejected": -367.06884765625, |
| "loss": 0.0747, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.0522563457489014, |
| "rewards/margins": 8.087520599365234, |
| "rewards/rejected": -10.139778137207031, |
| "step": 1055 |
| }, |
| { |
| "epoch": 2.2602666666666664, |
| "grad_norm": 1.672697901725769, |
| "learning_rate": 7.609624594306278e-07, |
| "logits/chosen": -1.8423585891723633, |
| "logits/rejected": -1.6809555292129517, |
| "logps/chosen": -247.50497436523438, |
| "logps/rejected": -400.91009521484375, |
| "loss": 0.0697, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.359173536300659, |
| "rewards/margins": 8.937664985656738, |
| "rewards/rejected": -11.29683780670166, |
| "step": 1060 |
| }, |
| { |
| "epoch": 2.2709333333333332, |
| "grad_norm": 1.9851620197296143, |
| "learning_rate": 7.403948874777556e-07, |
| "logits/chosen": -1.6603851318359375, |
| "logits/rejected": -1.6031215190887451, |
| "logps/chosen": -215.9097442626953, |
| "logps/rejected": -357.3408508300781, |
| "loss": 0.0697, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0506255626678467, |
| "rewards/margins": 8.248406410217285, |
| "rewards/rejected": -10.299031257629395, |
| "step": 1065 |
| }, |
| { |
| "epoch": 2.2816, |
| "grad_norm": 0.7464717030525208, |
| "learning_rate": 7.20060672845091e-07, |
| "logits/chosen": -1.6729190349578857, |
| "logits/rejected": -1.677329421043396, |
| "logps/chosen": -238.68331909179688, |
| "logps/rejected": -372.63372802734375, |
| "loss": 0.0629, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.739391803741455, |
| "rewards/margins": 8.386183738708496, |
| "rewards/rejected": -11.125576972961426, |
| "step": 1070 |
| }, |
| { |
| "epoch": 2.2922666666666665, |
| "grad_norm": 2.3664517402648926, |
| "learning_rate": 6.99962512238733e-07, |
| "logits/chosen": -1.6504055261611938, |
| "logits/rejected": -1.5495134592056274, |
| "logps/chosen": -220.272216796875, |
| "logps/rejected": -367.61602783203125, |
| "loss": 0.0527, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9944871664047241, |
| "rewards/margins": 9.167341232299805, |
| "rewards/rejected": -11.161828994750977, |
| "step": 1075 |
| }, |
| { |
| "epoch": 2.3029333333333333, |
| "grad_norm": 1.6238492727279663, |
| "learning_rate": 6.801030710594949e-07, |
| "logits/chosen": -1.6580880880355835, |
| "logits/rejected": -1.5914041996002197, |
| "logps/chosen": -217.9814910888672, |
| "logps/rejected": -347.48876953125, |
| "loss": 0.0803, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2583582401275635, |
| "rewards/margins": 8.29969596862793, |
| "rewards/rejected": -10.558053970336914, |
| "step": 1080 |
| }, |
| { |
| "epoch": 2.3136, |
| "grad_norm": 2.321399688720703, |
| "learning_rate": 6.604849830494264e-07, |
| "logits/chosen": -1.8174232244491577, |
| "logits/rejected": -1.685241460800171, |
| "logps/chosen": -238.77615356445312, |
| "logps/rejected": -348.0065002441406, |
| "loss": 0.0858, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.586581230163574, |
| "rewards/margins": 7.984179496765137, |
| "rewards/rejected": -10.570761680603027, |
| "step": 1085 |
| }, |
| { |
| "epoch": 2.3242666666666665, |
| "grad_norm": 0.6217744946479797, |
| "learning_rate": 6.411108499425253e-07, |
| "logits/chosen": -1.7700458765029907, |
| "logits/rejected": -1.6312090158462524, |
| "logps/chosen": -242.84988403320312, |
| "logps/rejected": -364.59869384765625, |
| "loss": 0.0587, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.2468154430389404, |
| "rewards/margins": 8.542348861694336, |
| "rewards/rejected": -10.789164543151855, |
| "step": 1090 |
| }, |
| { |
| "epoch": 2.3349333333333333, |
| "grad_norm": 1.4033160209655762, |
| "learning_rate": 6.219832411197013e-07, |
| "logits/chosen": -1.6557588577270508, |
| "logits/rejected": -1.5285712480545044, |
| "logps/chosen": -232.84140014648438, |
| "logps/rejected": -368.0865783691406, |
| "loss": 0.0784, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9303604364395142, |
| "rewards/margins": 8.661425590515137, |
| "rewards/rejected": -10.591785430908203, |
| "step": 1095 |
| }, |
| { |
| "epoch": 2.3456, |
| "grad_norm": 1.5255765914916992, |
| "learning_rate": 6.031046932680229e-07, |
| "logits/chosen": -1.7217681407928467, |
| "logits/rejected": -1.5818337202072144, |
| "logps/chosen": -236.8732147216797, |
| "logps/rejected": -360.173583984375, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.275178909301758, |
| "rewards/margins": 7.931247711181641, |
| "rewards/rejected": -10.206426620483398, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.3562666666666665, |
| "grad_norm": 0.8869664669036865, |
| "learning_rate": 5.844777100443091e-07, |
| "logits/chosen": -1.769890546798706, |
| "logits/rejected": -1.6314458847045898, |
| "logps/chosen": -225.7227325439453, |
| "logps/rejected": -370.30084228515625, |
| "loss": 0.0573, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.3325419425964355, |
| "rewards/margins": 9.121394157409668, |
| "rewards/rejected": -11.453936576843262, |
| "step": 1105 |
| }, |
| { |
| "epoch": 2.3669333333333333, |
| "grad_norm": 1.9305953979492188, |
| "learning_rate": 5.66104761743092e-07, |
| "logits/chosen": -1.8521592617034912, |
| "logits/rejected": -1.786136269569397, |
| "logps/chosen": -237.04428100585938, |
| "logps/rejected": -393.07232666015625, |
| "loss": 0.0569, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.924665689468384, |
| "rewards/margins": 9.040776252746582, |
| "rewards/rejected": -11.965442657470703, |
| "step": 1110 |
| }, |
| { |
| "epoch": 2.3776, |
| "grad_norm": 1.5030986070632935, |
| "learning_rate": 5.479882849690102e-07, |
| "logits/chosen": -1.7002397775650024, |
| "logits/rejected": -1.6100225448608398, |
| "logps/chosen": -249.65884399414062, |
| "logps/rejected": -375.28472900390625, |
| "loss": 0.055, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.4748988151550293, |
| "rewards/margins": 8.668333053588867, |
| "rewards/rejected": -11.143232345581055, |
| "step": 1115 |
| }, |
| { |
| "epoch": 2.3882666666666665, |
| "grad_norm": 3.2749836444854736, |
| "learning_rate": 5.301306823136687e-07, |
| "logits/chosen": -1.6064075231552124, |
| "logits/rejected": -1.5443446636199951, |
| "logps/chosen": -222.10498046875, |
| "logps/rejected": -354.99224853515625, |
| "loss": 0.1063, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.446816921234131, |
| "rewards/margins": 7.639315605163574, |
| "rewards/rejected": -10.086133003234863, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.3989333333333334, |
| "grad_norm": 6.177357196807861, |
| "learning_rate": 5.125343220370088e-07, |
| "logits/chosen": -1.7167507410049438, |
| "logits/rejected": -1.5875396728515625, |
| "logps/chosen": -251.5364532470703, |
| "logps/rejected": -350.0692138671875, |
| "loss": 0.1133, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.6598141193389893, |
| "rewards/margins": 7.746614933013916, |
| "rewards/rejected": -10.406428337097168, |
| "step": 1125 |
| }, |
| { |
| "epoch": 2.4096, |
| "grad_norm": 2.799936056137085, |
| "learning_rate": 4.952015377532329e-07, |
| "logits/chosen": -1.8022699356079102, |
| "logits/rejected": -1.648284912109375, |
| "logps/chosen": -219.56723022460938, |
| "logps/rejected": -375.37835693359375, |
| "loss": 0.0589, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.597977638244629, |
| "rewards/margins": 9.261964797973633, |
| "rewards/rejected": -11.859942436218262, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.4202666666666666, |
| "grad_norm": 0.8679878115653992, |
| "learning_rate": 4.781346281213204e-07, |
| "logits/chosen": -1.758948564529419, |
| "logits/rejected": -1.6984516382217407, |
| "logps/chosen": -216.8728790283203, |
| "logps/rejected": -372.5053405761719, |
| "loss": 0.0526, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3770511150360107, |
| "rewards/margins": 9.284753799438477, |
| "rewards/rejected": -11.66180419921875, |
| "step": 1135 |
| }, |
| { |
| "epoch": 2.4309333333333334, |
| "grad_norm": 3.049687147140503, |
| "learning_rate": 4.613358565401818e-07, |
| "logits/chosen": -1.703428864479065, |
| "logits/rejected": -1.647216796875, |
| "logps/chosen": -251.83456420898438, |
| "logps/rejected": -382.45513916015625, |
| "loss": 0.0735, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.692182779312134, |
| "rewards/margins": 7.887045860290527, |
| "rewards/rejected": -10.579229354858398, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.4416, |
| "grad_norm": 1.0848246812820435, |
| "learning_rate": 4.448074508484906e-07, |
| "logits/chosen": -1.7292941808700562, |
| "logits/rejected": -1.5465002059936523, |
| "logps/chosen": -243.04476928710938, |
| "logps/rejected": -368.3805236816406, |
| "loss": 0.0597, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3163161277770996, |
| "rewards/margins": 8.721205711364746, |
| "rewards/rejected": -11.037521362304688, |
| "step": 1145 |
| }, |
| { |
| "epoch": 2.4522666666666666, |
| "grad_norm": 1.5578293800354004, |
| "learning_rate": 4.2855160302922777e-07, |
| "logits/chosen": -1.7324931621551514, |
| "logits/rejected": -1.696868896484375, |
| "logps/chosen": -221.2848358154297, |
| "logps/rejected": -381.62359619140625, |
| "loss": 0.0938, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.416508197784424, |
| "rewards/margins": 9.063284873962402, |
| "rewards/rejected": -11.479792594909668, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.4629333333333334, |
| "grad_norm": 2.7087442874908447, |
| "learning_rate": 4.125704689189819e-07, |
| "logits/chosen": -1.6965996026992798, |
| "logits/rejected": -1.5242478847503662, |
| "logps/chosen": -222.7657928466797, |
| "logps/rejected": -364.8160095214844, |
| "loss": 0.0673, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0459980964660645, |
| "rewards/margins": 8.586442947387695, |
| "rewards/rejected": -10.632440567016602, |
| "step": 1155 |
| }, |
| { |
| "epoch": 2.4736000000000002, |
| "grad_norm": 4.279755592346191, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.6694886684417725, |
| "logits/rejected": -1.6094839572906494, |
| "logps/chosen": -223.513916015625, |
| "logps/rejected": -360.60211181640625, |
| "loss": 0.0563, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.202442169189453, |
| "rewards/margins": 8.466798782348633, |
| "rewards/rejected": -10.669241905212402, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.4842666666666666, |
| "grad_norm": 3.6986382007598877, |
| "learning_rate": 3.8144078272934676e-07, |
| "logits/chosen": -1.7204971313476562, |
| "logits/rejected": -1.6476894617080688, |
| "logps/chosen": -246.6821746826172, |
| "logps/rejected": -372.75634765625, |
| "loss": 0.0671, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.262946128845215, |
| "rewards/margins": 9.07579231262207, |
| "rewards/rejected": -11.338738441467285, |
| "step": 1165 |
| }, |
| { |
| "epoch": 2.4949333333333334, |
| "grad_norm": 0.5171156525611877, |
| "learning_rate": 3.6629635904222943e-07, |
| "logits/chosen": -1.7819716930389404, |
| "logits/rejected": -1.6632438898086548, |
| "logps/chosen": -226.53421020507812, |
| "logps/rejected": -402.94598388671875, |
| "loss": 0.048, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.1981759071350098, |
| "rewards/margins": 9.268115043640137, |
| "rewards/rejected": -11.466291427612305, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.5056000000000003, |
| "grad_norm": 1.4313619136810303, |
| "learning_rate": 3.514349053011698e-07, |
| "logits/chosen": -1.7268043756484985, |
| "logits/rejected": -1.5567543506622314, |
| "logps/chosen": -242.3755645751953, |
| "logps/rejected": -378.0379333496094, |
| "loss": 0.0647, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5210442543029785, |
| "rewards/margins": 8.706852912902832, |
| "rewards/rejected": -11.227896690368652, |
| "step": 1175 |
| }, |
| { |
| "epoch": 2.5162666666666667, |
| "grad_norm": 4.2524638175964355, |
| "learning_rate": 3.3685839241941133e-07, |
| "logits/chosen": -1.6792681217193604, |
| "logits/rejected": -1.6164538860321045, |
| "logps/chosen": -227.9314727783203, |
| "logps/rejected": -347.4278259277344, |
| "loss": 0.0808, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.1522090435028076, |
| "rewards/margins": 8.178491592407227, |
| "rewards/rejected": -10.33070182800293, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.5269333333333335, |
| "grad_norm": 4.731504917144775, |
| "learning_rate": 3.2256875352158435e-07, |
| "logits/chosen": -1.698481798171997, |
| "logits/rejected": -1.6122955083847046, |
| "logps/chosen": -238.3931427001953, |
| "logps/rejected": -390.96356201171875, |
| "loss": 0.0562, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2728371620178223, |
| "rewards/margins": 9.056289672851562, |
| "rewards/rejected": -11.329126358032227, |
| "step": 1185 |
| }, |
| { |
| "epoch": 2.5376, |
| "grad_norm": 2.2026448249816895, |
| "learning_rate": 3.085678836873399e-07, |
| "logits/chosen": -1.700873613357544, |
| "logits/rejected": -1.6878290176391602, |
| "logps/chosen": -203.16343688964844, |
| "logps/rejected": -371.3619079589844, |
| "loss": 0.0637, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.1309585571289062, |
| "rewards/margins": 8.881011009216309, |
| "rewards/rejected": -11.011968612670898, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.5482666666666667, |
| "grad_norm": 2.0885634422302246, |
| "learning_rate": 2.9485763970002106e-07, |
| "logits/chosen": -1.6919275522232056, |
| "logits/rejected": -1.6454477310180664, |
| "logps/chosen": -229.4306640625, |
| "logps/rejected": -385.67364501953125, |
| "loss": 0.0581, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.0748720169067383, |
| "rewards/margins": 9.040914535522461, |
| "rewards/rejected": -12.1157865524292, |
| "step": 1195 |
| }, |
| { |
| "epoch": 2.558933333333333, |
| "grad_norm": 1.6398587226867676, |
| "learning_rate": 2.8143983980042174e-07, |
| "logits/chosen": -1.6853199005126953, |
| "logits/rejected": -1.563265085220337, |
| "logps/chosen": -220.6924591064453, |
| "logps/rejected": -361.0828552246094, |
| "loss": 0.0533, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5653061866760254, |
| "rewards/margins": 8.984647750854492, |
| "rewards/rejected": -11.549954414367676, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.5696, |
| "grad_norm": 4.250986099243164, |
| "learning_rate": 2.683162634456515e-07, |
| "logits/chosen": -1.7149680852890015, |
| "logits/rejected": -1.6375681161880493, |
| "logps/chosen": -243.39767456054688, |
| "logps/rejected": -375.77197265625, |
| "loss": 0.122, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.3229522705078125, |
| "rewards/margins": 8.259476661682129, |
| "rewards/rejected": -10.582427978515625, |
| "step": 1205 |
| }, |
| { |
| "epoch": 2.5802666666666667, |
| "grad_norm": 1.7875478267669678, |
| "learning_rate": 2.5548865107314606e-07, |
| "logits/chosen": -1.6222026348114014, |
| "logits/rejected": -1.5698870420455933, |
| "logps/chosen": -252.62234497070312, |
| "logps/rejected": -371.35968017578125, |
| "loss": 0.0474, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.9672263860702515, |
| "rewards/margins": 8.441937446594238, |
| "rewards/rejected": -10.409163475036621, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.590933333333333, |
| "grad_norm": 2.756495475769043, |
| "learning_rate": 2.429587038698511e-07, |
| "logits/chosen": -1.7179148197174072, |
| "logits/rejected": -1.6547319889068604, |
| "logps/chosen": -215.96231079101562, |
| "logps/rejected": -383.422607421875, |
| "loss": 0.0606, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.6292073726654053, |
| "rewards/margins": 9.533937454223633, |
| "rewards/rejected": -12.1631441116333, |
| "step": 1215 |
| }, |
| { |
| "epoch": 2.6016, |
| "grad_norm": 2.755079984664917, |
| "learning_rate": 2.307280835466144e-07, |
| "logits/chosen": -1.7297523021697998, |
| "logits/rejected": -1.5602505207061768, |
| "logps/chosen": -226.3815155029297, |
| "logps/rejected": -379.71246337890625, |
| "loss": 0.0489, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.6925208568573, |
| "rewards/margins": 8.994084358215332, |
| "rewards/rejected": -11.686605453491211, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.6122666666666667, |
| "grad_norm": 2.1932475566864014, |
| "learning_rate": 2.1879841211780856e-07, |
| "logits/chosen": -1.6469560861587524, |
| "logits/rejected": -1.5588574409484863, |
| "logps/chosen": -237.40298461914062, |
| "logps/rejected": -373.273681640625, |
| "loss": 0.067, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.41096568107605, |
| "rewards/margins": 8.6922607421875, |
| "rewards/rejected": -11.103225708007812, |
| "step": 1225 |
| }, |
| { |
| "epoch": 2.622933333333333, |
| "grad_norm": 2.3217146396636963, |
| "learning_rate": 2.0717127168622202e-07, |
| "logits/chosen": -1.7379156351089478, |
| "logits/rejected": -1.5808634757995605, |
| "logps/chosen": -245.385498046875, |
| "logps/rejected": -379.40460205078125, |
| "loss": 0.0561, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5605995655059814, |
| "rewards/margins": 8.971899032592773, |
| "rewards/rejected": -11.532498359680176, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.6336, |
| "grad_norm": 3.9729578495025635, |
| "learning_rate": 1.958482042332424e-07, |
| "logits/chosen": -1.7232568264007568, |
| "logits/rejected": -1.5390961170196533, |
| "logps/chosen": -262.4111328125, |
| "logps/rejected": -380.0039978027344, |
| "loss": 0.0991, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.5589394569396973, |
| "rewards/margins": 8.578977584838867, |
| "rewards/rejected": -11.137918472290039, |
| "step": 1235 |
| }, |
| { |
| "epoch": 2.6442666666666668, |
| "grad_norm": 3.108950138092041, |
| "learning_rate": 1.8483071141435937e-07, |
| "logits/chosen": -1.6421682834625244, |
| "logits/rejected": -1.5635117292404175, |
| "logps/chosen": -269.7666015625, |
| "logps/rejected": -369.1126403808594, |
| "loss": 0.0626, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.4174113273620605, |
| "rewards/margins": 8.311561584472656, |
| "rewards/rejected": -10.728974342346191, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.654933333333333, |
| "grad_norm": 2.487255811691284, |
| "learning_rate": 1.741202543600176e-07, |
| "logits/chosen": -1.7232882976531982, |
| "logits/rejected": -1.6298853158950806, |
| "logps/chosen": -225.5282440185547, |
| "logps/rejected": -367.0820007324219, |
| "loss": 0.0481, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.8948845863342285, |
| "rewards/margins": 8.565133094787598, |
| "rewards/rejected": -11.460018157958984, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.6656, |
| "grad_norm": 3.5590980052948, |
| "learning_rate": 1.6371825348184155e-07, |
| "logits/chosen": -1.6784608364105225, |
| "logits/rejected": -1.5763369798660278, |
| "logps/chosen": -233.04873657226562, |
| "logps/rejected": -362.5445861816406, |
| "loss": 0.0588, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.41444730758667, |
| "rewards/margins": 8.470220565795898, |
| "rewards/rejected": -10.884668350219727, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.676266666666667, |
| "grad_norm": 2.1251602172851562, |
| "learning_rate": 1.5362608828426246e-07, |
| "logits/chosen": -1.7499914169311523, |
| "logits/rejected": -1.6538130044937134, |
| "logps/chosen": -247.6387939453125, |
| "logps/rejected": -385.8589782714844, |
| "loss": 0.0527, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.267554759979248, |
| "rewards/margins": 9.327815055847168, |
| "rewards/rejected": -11.595370292663574, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.686933333333333, |
| "grad_norm": 3.002373456954956, |
| "learning_rate": 1.438450971815686e-07, |
| "logits/chosen": -1.7843786478042603, |
| "logits/rejected": -1.6649062633514404, |
| "logps/chosen": -237.3309326171875, |
| "logps/rejected": -372.9043884277344, |
| "loss": 0.0639, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.4575018882751465, |
| "rewards/margins": 9.09184741973877, |
| "rewards/rejected": -11.549348831176758, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.6976, |
| "grad_norm": 3.6394524574279785, |
| "learning_rate": 1.3437657732040783e-07, |
| "logits/chosen": -1.7269392013549805, |
| "logits/rejected": -1.5753428936004639, |
| "logps/chosen": -228.05859375, |
| "logps/rejected": -347.77587890625, |
| "loss": 0.0659, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.337057590484619, |
| "rewards/margins": 8.416463851928711, |
| "rewards/rejected": -10.753522872924805, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.708266666666667, |
| "grad_norm": 2.0372815132141113, |
| "learning_rate": 1.2522178440776022e-07, |
| "logits/chosen": -1.7152751684188843, |
| "logits/rejected": -1.6060600280761719, |
| "logps/chosen": -223.6414337158203, |
| "logps/rejected": -367.9722595214844, |
| "loss": 0.0512, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0669002532958984, |
| "rewards/margins": 8.808296203613281, |
| "rewards/rejected": -10.87519645690918, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.718933333333333, |
| "grad_norm": 3.3202083110809326, |
| "learning_rate": 1.1638193254440655e-07, |
| "logits/chosen": -1.7251523733139038, |
| "logits/rejected": -1.6463289260864258, |
| "logps/chosen": -242.5394287109375, |
| "logps/rejected": -384.67828369140625, |
| "loss": 0.0626, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.65033221244812, |
| "rewards/margins": 9.154813766479492, |
| "rewards/rejected": -11.805147171020508, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.7296, |
| "grad_norm": 3.912275791168213, |
| "learning_rate": 1.0785819406391562e-07, |
| "logits/chosen": -1.629055380821228, |
| "logits/rejected": -1.6328773498535156, |
| "logps/chosen": -222.7128448486328, |
| "logps/rejected": -362.33697509765625, |
| "loss": 0.0828, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.1510047912597656, |
| "rewards/margins": 8.43962287902832, |
| "rewards/rejected": -10.59062671661377, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.740266666666667, |
| "grad_norm": 1.5474649667739868, |
| "learning_rate": 9.965169937717095e-08, |
| "logits/chosen": -1.731776475906372, |
| "logits/rejected": -1.6290785074234009, |
| "logps/chosen": -220.5186004638672, |
| "logps/rejected": -371.43017578125, |
| "loss": 0.0489, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.538206100463867, |
| "rewards/margins": 8.95645809173584, |
| "rewards/rejected": -11.494664192199707, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.7509333333333332, |
| "grad_norm": 4.089476585388184, |
| "learning_rate": 9.176353682245676e-08, |
| "logits/chosen": -1.7597144842147827, |
| "logits/rejected": -1.7338988780975342, |
| "logps/chosen": -222.9378204345703, |
| "logps/rejected": -379.48046875, |
| "loss": 0.0821, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.5998435020446777, |
| "rewards/margins": 8.580057144165039, |
| "rewards/rejected": -11.179900169372559, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.7616, |
| "grad_norm": 2.3440282344818115, |
| "learning_rate": 8.41947525211223e-08, |
| "logits/chosen": -1.6561254262924194, |
| "logits/rejected": -1.5427234172821045, |
| "logps/chosen": -236.984130859375, |
| "logps/rejected": -358.4666442871094, |
| "loss": 0.0705, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.1874051094055176, |
| "rewards/margins": 8.178415298461914, |
| "rewards/rejected": -10.36582088470459, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.772266666666667, |
| "grad_norm": 0.8540581464767456, |
| "learning_rate": 7.694635023884789e-08, |
| "logits/chosen": -1.6888628005981445, |
| "logits/rejected": -1.6380894184112549, |
| "logps/chosen": -244.80453491210938, |
| "logps/rejected": -398.01416015625, |
| "loss": 0.0572, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.498230457305908, |
| "rewards/margins": 9.02320671081543, |
| "rewards/rejected": -11.521438598632812, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7829333333333333, |
| "grad_norm": 2.1657631397247314, |
| "learning_rate": 7.00192912525241e-08, |
| "logits/chosen": -1.7026500701904297, |
| "logits/rejected": -1.6592811346054077, |
| "logps/chosen": -236.1211395263672, |
| "logps/rejected": -372.11700439453125, |
| "loss": 0.0853, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.4645047187805176, |
| "rewards/margins": 8.029794692993164, |
| "rewards/rejected": -10.494298934936523, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.7936, |
| "grad_norm": 2.2320377826690674, |
| "learning_rate": 6.341449422277047e-08, |
| "logits/chosen": -1.6398401260375977, |
| "logits/rejected": -1.586236834526062, |
| "logps/chosen": -264.7616882324219, |
| "logps/rejected": -387.8482971191406, |
| "loss": 0.0761, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.7749290466308594, |
| "rewards/margins": 8.451501846313477, |
| "rewards/rejected": -11.226430892944336, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.804266666666667, |
| "grad_norm": 2.414036989212036, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.7663352489471436, |
| "logits/rejected": -1.7004578113555908, |
| "logps/chosen": -232.11264038085938, |
| "logps/rejected": -388.02520751953125, |
| "loss": 0.0685, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.7615132331848145, |
| "rewards/margins": 9.383472442626953, |
| "rewards/rejected": -12.144986152648926, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.8149333333333333, |
| "grad_norm": 3.1423819065093994, |
| "learning_rate": 5.117514686876379e-08, |
| "logits/chosen": -1.639988660812378, |
| "logits/rejected": -1.6136987209320068, |
| "logps/chosen": -226.6444854736328, |
| "logps/rejected": -338.81243896484375, |
| "loss": 0.0958, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9987974166870117, |
| "rewards/margins": 7.690194606781006, |
| "rewards/rejected": -9.688993453979492, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.8256, |
| "grad_norm": 3.009913206100464, |
| "learning_rate": 4.5542219716255274e-08, |
| "logits/chosen": -1.726658582687378, |
| "logits/rejected": -1.6036049127578735, |
| "logps/chosen": -248.6483154296875, |
| "logps/rejected": -389.01104736328125, |
| "loss": 0.0751, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6450247764587402, |
| "rewards/margins": 9.053154945373535, |
| "rewards/rejected": -11.698179244995117, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.836266666666667, |
| "grad_norm": 1.047540545463562, |
| "learning_rate": 4.023480064854174e-08, |
| "logits/chosen": -1.7443851232528687, |
| "logits/rejected": -1.6466442346572876, |
| "logps/chosen": -232.3708038330078, |
| "logps/rejected": -355.1757507324219, |
| "loss": 0.0386, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.3548583984375, |
| "rewards/margins": 8.295844078063965, |
| "rewards/rejected": -10.650702476501465, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.8469333333333333, |
| "grad_norm": 2.1108577251434326, |
| "learning_rate": 3.525359353098584e-08, |
| "logits/chosen": -1.711523413658142, |
| "logits/rejected": -1.564855694770813, |
| "logps/chosen": -234.50955200195312, |
| "logps/rejected": -401.87237548828125, |
| "loss": 0.085, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5529706478118896, |
| "rewards/margins": 9.332552909851074, |
| "rewards/rejected": -11.885522842407227, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.8576, |
| "grad_norm": 1.4761799573898315, |
| "learning_rate": 3.059925896700094e-08, |
| "logits/chosen": -1.7107757329940796, |
| "logits/rejected": -1.5677356719970703, |
| "logps/chosen": -256.050048828125, |
| "logps/rejected": -387.1049499511719, |
| "loss": 0.0517, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.124401330947876, |
| "rewards/margins": 9.434419631958008, |
| "rewards/rejected": -11.558820724487305, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.8682666666666665, |
| "grad_norm": 2.649878978729248, |
| "learning_rate": 2.6272414210443964e-08, |
| "logits/chosen": -1.6712446212768555, |
| "logits/rejected": -1.666788101196289, |
| "logps/chosen": -246.9146270751953, |
| "logps/rejected": -387.96697998046875, |
| "loss": 0.107, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.460017681121826, |
| "rewards/margins": 8.867338180541992, |
| "rewards/rejected": -11.32735538482666, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.8789333333333333, |
| "grad_norm": 0.556817889213562, |
| "learning_rate": 2.2273633083753632e-08, |
| "logits/chosen": -1.7067201137542725, |
| "logits/rejected": -1.613102674484253, |
| "logps/chosen": -216.364501953125, |
| "logps/rejected": -377.43341064453125, |
| "loss": 0.0453, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5222439765930176, |
| "rewards/margins": 9.433998107910156, |
| "rewards/rejected": -11.956242561340332, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.8895999999999997, |
| "grad_norm": 2.3840444087982178, |
| "learning_rate": 1.8603445901851647e-08, |
| "logits/chosen": -1.6174293756484985, |
| "logits/rejected": -1.5605263710021973, |
| "logps/chosen": -234.5871124267578, |
| "logps/rejected": -393.0818176269531, |
| "loss": 0.053, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.399482250213623, |
| "rewards/margins": 8.842782020568848, |
| "rewards/rejected": -11.242264747619629, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.9002666666666665, |
| "grad_norm": 1.3605575561523438, |
| "learning_rate": 1.5262339401813375e-08, |
| "logits/chosen": -1.721879005432129, |
| "logits/rejected": -1.6476589441299438, |
| "logps/chosen": -225.7846221923828, |
| "logps/rejected": -390.84075927734375, |
| "loss": 0.0683, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.652186632156372, |
| "rewards/margins": 9.073946952819824, |
| "rewards/rejected": -11.726133346557617, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.9109333333333334, |
| "grad_norm": 7.382095813751221, |
| "learning_rate": 1.2250756678315601e-08, |
| "logits/chosen": -1.7093260288238525, |
| "logits/rejected": -1.6412489414215088, |
| "logps/chosen": -242.4256591796875, |
| "logps/rejected": -390.5251159667969, |
| "loss": 0.0874, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4845237731933594, |
| "rewards/margins": 8.939850807189941, |
| "rewards/rejected": -11.424372673034668, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.9215999999999998, |
| "grad_norm": 3.5759291648864746, |
| "learning_rate": 9.569097124875193e-09, |
| "logits/chosen": -1.6812433004379272, |
| "logits/rejected": -1.6351463794708252, |
| "logps/chosen": -224.98519897460938, |
| "logps/rejected": -373.64129638671875, |
| "loss": 0.0635, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5327999591827393, |
| "rewards/margins": 8.734055519104004, |
| "rewards/rejected": -11.266853332519531, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.9322666666666666, |
| "grad_norm": 3.4827802181243896, |
| "learning_rate": 7.217716380881479e-09, |
| "logits/chosen": -1.7382152080535889, |
| "logits/rejected": -1.634007215499878, |
| "logps/chosen": -261.80853271484375, |
| "logps/rejected": -366.9189758300781, |
| "loss": 0.0837, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.2536139488220215, |
| "rewards/margins": 8.559202194213867, |
| "rewards/rejected": -10.812816619873047, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.9429333333333334, |
| "grad_norm": 4.551513195037842, |
| "learning_rate": 5.196926284430359e-09, |
| "logits/chosen": -1.6665912866592407, |
| "logits/rejected": -1.5187146663665771, |
| "logps/chosen": -244.35867309570312, |
| "logps/rejected": -395.45672607421875, |
| "loss": 0.051, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5061800479888916, |
| "rewards/margins": 8.809569358825684, |
| "rewards/rejected": -11.31574821472168, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.9536, |
| "grad_norm": 2.610435724258423, |
| "learning_rate": 3.5069948309712775e-09, |
| "logits/chosen": -1.6568130254745483, |
| "logits/rejected": -1.6110321283340454, |
| "logps/chosen": -270.39569091796875, |
| "logps/rejected": -387.37115478515625, |
| "loss": 0.0588, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6436781883239746, |
| "rewards/margins": 8.328022956848145, |
| "rewards/rejected": -10.971700668334961, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.9642666666666666, |
| "grad_norm": 1.42137610912323, |
| "learning_rate": 2.1481461377634294e-09, |
| "logits/chosen": -1.6595335006713867, |
| "logits/rejected": -1.6308250427246094, |
| "logps/chosen": -232.0425262451172, |
| "logps/rejected": -376.16619873046875, |
| "loss": 0.0508, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.516741991043091, |
| "rewards/margins": 8.540366172790527, |
| "rewards/rejected": -11.057107925415039, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.9749333333333334, |
| "grad_norm": 0.8431697487831116, |
| "learning_rate": 1.120560414154259e-09, |
| "logits/chosen": -1.7065174579620361, |
| "logits/rejected": -1.6119663715362549, |
| "logps/chosen": -232.8285675048828, |
| "logps/rejected": -367.2099914550781, |
| "loss": 0.0596, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5289416313171387, |
| "rewards/margins": 8.318876266479492, |
| "rewards/rejected": -10.847818374633789, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.9856, |
| "grad_norm": 3.3298075199127197, |
| "learning_rate": 4.2437393768079983e-10, |
| "logits/chosen": -1.663039207458496, |
| "logits/rejected": -1.571150779724121, |
| "logps/chosen": -250.9148406982422, |
| "logps/rejected": -356.40277099609375, |
| "loss": 0.0692, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.4644951820373535, |
| "rewards/margins": 8.116125106811523, |
| "rewards/rejected": -10.580621719360352, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.9962666666666666, |
| "grad_norm": 2.234276056289673, |
| "learning_rate": 5.967903599635171e-11, |
| "logits/chosen": -1.6961066722869873, |
| "logits/rejected": -1.5509659051895142, |
| "logps/chosen": -247.11538696289062, |
| "logps/rejected": -370.01708984375, |
| "loss": 0.0646, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5652105808258057, |
| "rewards/margins": 8.701112747192383, |
| "rewards/rejected": -11.266324043273926, |
| "step": 1405 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1407, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
