| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1875, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.008, |
| "grad_norm": 3.09834361076355, |
| "learning_rate": 3.5087719298245616e-07, |
| "logits/chosen": -1.2909044027328491, |
| "logits/rejected": -1.2319936752319336, |
| "logps/chosen": -232.99636840820312, |
| "logps/rejected": -240.0132293701172, |
| "loss": 0.693, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -1.5878351860010298e-06, |
| "rewards/margins": 0.0003593349829316139, |
| "rewards/rejected": -0.00036092280060984194, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.016, |
| "grad_norm": 2.9631168842315674, |
| "learning_rate": 7.894736842105263e-07, |
| "logits/chosen": -1.2395670413970947, |
| "logits/rejected": -1.1628472805023193, |
| "logps/chosen": -226.4220428466797, |
| "logps/rejected": -245.43753051757812, |
| "loss": 0.6909, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.0036325000692158937, |
| "rewards/margins": 0.004563910886645317, |
| "rewards/rejected": -0.0009314107592217624, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.024, |
| "grad_norm": 3.6685404777526855, |
| "learning_rate": 1.2280701754385965e-06, |
| "logits/chosen": -1.2391703128814697, |
| "logits/rejected": -1.15793776512146, |
| "logps/chosen": -224.14266967773438, |
| "logps/rejected": -234.9234619140625, |
| "loss": 0.6851, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.01015846710652113, |
| "rewards/margins": 0.016449259594082832, |
| "rewards/rejected": -0.006290791090577841, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 2.7006146907806396, |
| "learning_rate": 1.6666666666666667e-06, |
| "logits/chosen": -1.122662901878357, |
| "logits/rejected": -1.1195858716964722, |
| "logps/chosen": -234.66134643554688, |
| "logps/rejected": -241.30050659179688, |
| "loss": 0.6757, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.0218190997838974, |
| "rewards/margins": 0.03629850223660469, |
| "rewards/rejected": -0.014479401521384716, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.04, |
| "grad_norm": 2.9753403663635254, |
| "learning_rate": 2.105263157894737e-06, |
| "logits/chosen": -1.18116295337677, |
| "logits/rejected": -1.2049816846847534, |
| "logps/chosen": -215.8043212890625, |
| "logps/rejected": -255.64266967773438, |
| "loss": 0.668, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": 0.032455407083034515, |
| "rewards/margins": 0.05338490009307861, |
| "rewards/rejected": -0.0209294892847538, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.048, |
| "grad_norm": 3.15077543258667, |
| "learning_rate": 2.5438596491228075e-06, |
| "logits/chosen": -1.1878756284713745, |
| "logits/rejected": -1.193882703781128, |
| "logps/chosen": -217.12744140625, |
| "logps/rejected": -248.48825073242188, |
| "loss": 0.6515, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 0.056911785155534744, |
| "rewards/margins": 0.0923050194978714, |
| "rewards/rejected": -0.03539323806762695, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.056, |
| "grad_norm": 2.79844069480896, |
| "learning_rate": 2.9824561403508774e-06, |
| "logits/chosen": -1.183730125427246, |
| "logits/rejected": -1.173300862312317, |
| "logps/chosen": -231.12142944335938, |
| "logps/rejected": -251.42605590820312, |
| "loss": 0.6357, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": 0.07155142724514008, |
| "rewards/margins": 0.13084718585014343, |
| "rewards/rejected": -0.05929575487971306, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 2.861743450164795, |
| "learning_rate": 3.421052631578948e-06, |
| "logits/chosen": -1.2913212776184082, |
| "logits/rejected": -1.1513500213623047, |
| "logps/chosen": -223.00509643554688, |
| "logps/rejected": -215.6059112548828, |
| "loss": 0.627, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.09407249093055725, |
| "rewards/margins": 0.15628233551979065, |
| "rewards/rejected": -0.0622098445892334, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.072, |
| "grad_norm": 2.4972827434539795, |
| "learning_rate": 3.859649122807018e-06, |
| "logits/chosen": -1.1650872230529785, |
| "logits/rejected": -1.0375068187713623, |
| "logps/chosen": -228.2639617919922, |
| "logps/rejected": -230.9693145751953, |
| "loss": 0.5934, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.1547546088695526, |
| "rewards/margins": 0.26725736260414124, |
| "rewards/rejected": -0.11250276863574982, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 2.606940746307373, |
| "learning_rate": 4.298245614035088e-06, |
| "logits/chosen": -1.171435832977295, |
| "logits/rejected": -1.1679085493087769, |
| "logps/chosen": -247.4260711669922, |
| "logps/rejected": -261.73394775390625, |
| "loss": 0.5431, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.25676974654197693, |
| "rewards/margins": 0.4268248677253723, |
| "rewards/rejected": -0.1700551062822342, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.088, |
| "grad_norm": 2.8133466243743896, |
| "learning_rate": 4.736842105263158e-06, |
| "logits/chosen": -1.1443212032318115, |
| "logits/rejected": -1.1487455368041992, |
| "logps/chosen": -217.5292510986328, |
| "logps/rejected": -251.6279296875, |
| "loss": 0.5568, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.22170846164226532, |
| "rewards/margins": 0.42460116744041443, |
| "rewards/rejected": -0.20289269089698792, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 2.8709912300109863, |
| "learning_rate": 4.999985069252202e-06, |
| "logits/chosen": -1.2766786813735962, |
| "logits/rejected": -1.2180373668670654, |
| "logps/chosen": -235.04623413085938, |
| "logps/rejected": -271.3938293457031, |
| "loss": 0.5274, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": 0.2577889561653137, |
| "rewards/margins": 0.5365679264068604, |
| "rewards/rejected": -0.27877891063690186, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.104, |
| "grad_norm": 2.896838903427124, |
| "learning_rate": 4.999817100387611e-06, |
| "logits/chosen": -1.275023102760315, |
| "logits/rejected": -1.273930311203003, |
| "logps/chosen": -218.29830932617188, |
| "logps/rejected": -236.113525390625, |
| "loss": 0.5154, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.18201565742492676, |
| "rewards/margins": 0.5186901092529297, |
| "rewards/rejected": -0.33667445182800293, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.112, |
| "grad_norm": 2.841238021850586, |
| "learning_rate": 4.99946251180492e-06, |
| "logits/chosen": -1.3415954113006592, |
| "logits/rejected": -1.3333144187927246, |
| "logps/chosen": -240.9877471923828, |
| "logps/rejected": -274.54022216796875, |
| "loss": 0.5186, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": 0.14453572034835815, |
| "rewards/margins": 0.6278968453407288, |
| "rewards/rejected": -0.4833610951900482, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 2.8624284267425537, |
| "learning_rate": 4.998921329975353e-06, |
| "logits/chosen": -1.3028557300567627, |
| "logits/rejected": -1.198012113571167, |
| "logps/chosen": -213.59176635742188, |
| "logps/rejected": -253.2032470703125, |
| "loss": 0.5051, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.1062125712633133, |
| "rewards/margins": 0.6504680514335632, |
| "rewards/rejected": -0.5442554950714111, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 3.075573682785034, |
| "learning_rate": 4.998193595299945e-06, |
| "logits/chosen": -1.4071934223175049, |
| "logits/rejected": -1.2867439985275269, |
| "logps/chosen": -223.06655883789062, |
| "logps/rejected": -265.8553771972656, |
| "loss": 0.4527, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.2724152207374573, |
| "rewards/margins": 1.0262385606765747, |
| "rewards/rejected": -0.7538233995437622, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.136, |
| "grad_norm": 2.3981781005859375, |
| "learning_rate": 4.997279362106528e-06, |
| "logits/chosen": -1.3612762689590454, |
| "logits/rejected": -1.279844045639038, |
| "logps/chosen": -212.7271270751953, |
| "logps/rejected": -265.90081787109375, |
| "loss": 0.4299, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.16929033398628235, |
| "rewards/margins": 1.0672941207885742, |
| "rewards/rejected": -0.8980037569999695, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.144, |
| "grad_norm": 2.9816811084747314, |
| "learning_rate": 4.9961786986456705e-06, |
| "logits/chosen": -1.3840779066085815, |
| "logits/rejected": -1.4094316959381104, |
| "logps/chosen": -213.59060668945312, |
| "logps/rejected": -260.33258056640625, |
| "loss": 0.4511, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.11396338790655136, |
| "rewards/margins": 1.0669071674346924, |
| "rewards/rejected": -0.9529438018798828, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.152, |
| "grad_norm": 2.093193769454956, |
| "learning_rate": 4.994891687085587e-06, |
| "logits/chosen": -1.332556128501892, |
| "logits/rejected": -1.307656168937683, |
| "logps/chosen": -216.29476928710938, |
| "logps/rejected": -251.3971710205078, |
| "loss": 0.3924, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.09258659183979034, |
| "rewards/margins": 1.270951747894287, |
| "rewards/rejected": -1.1783651113510132, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 2.8752410411834717, |
| "learning_rate": 4.9934184235060005e-06, |
| "logits/chosen": -1.5339454412460327, |
| "logits/rejected": -1.3785521984100342, |
| "logps/chosen": -239.418701171875, |
| "logps/rejected": -255.296875, |
| "loss": 0.416, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.05207740142941475, |
| "rewards/margins": 1.1651036739349365, |
| "rewards/rejected": -1.1130263805389404, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.168, |
| "grad_norm": 3.262073040008545, |
| "learning_rate": 4.9917590178909695e-06, |
| "logits/chosen": -1.3999998569488525, |
| "logits/rejected": -1.367429494857788, |
| "logps/chosen": -226.4331817626953, |
| "logps/rejected": -264.509521484375, |
| "loss": 0.4265, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.008939206600189209, |
| "rewards/margins": 1.2716344594955444, |
| "rewards/rejected": -1.2805736064910889, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.176, |
| "grad_norm": 2.390469551086426, |
| "learning_rate": 4.989913594120682e-06, |
| "logits/chosen": -1.4643319845199585, |
| "logits/rejected": -1.393023133277893, |
| "logps/chosen": -249.9105987548828, |
| "logps/rejected": -269.37591552734375, |
| "loss": 0.4418, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.04763683304190636, |
| "rewards/margins": 1.3064950704574585, |
| "rewards/rejected": -1.354131817817688, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.184, |
| "grad_norm": 3.1575868129730225, |
| "learning_rate": 4.987882289962201e-06, |
| "logits/chosen": -1.3801255226135254, |
| "logits/rejected": -1.2841880321502686, |
| "logps/chosen": -244.7362823486328, |
| "logps/rejected": -268.76727294921875, |
| "loss": 0.3836, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.015139693394303322, |
| "rewards/margins": 1.4781547784805298, |
| "rewards/rejected": -1.4630149602890015, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 2.9875669479370117, |
| "learning_rate": 4.9856652570591865e-06, |
| "logits/chosen": -1.4804288148880005, |
| "logits/rejected": -1.4396897554397583, |
| "logps/chosen": -223.47317504882812, |
| "logps/rejected": -274.21429443359375, |
| "loss": 0.4113, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.02329862117767334, |
| "rewards/margins": 1.192557454109192, |
| "rewards/rejected": -1.2158560752868652, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 11.367328643798828, |
| "learning_rate": 4.983262660920567e-06, |
| "logits/chosen": -1.5214124917984009, |
| "logits/rejected": -1.478496789932251, |
| "logps/chosen": -284.20745849609375, |
| "logps/rejected": -264.2999267578125, |
| "loss": 0.3841, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.17885152995586395, |
| "rewards/margins": 1.4730366468429565, |
| "rewards/rejected": -1.2941851615905762, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.208, |
| "grad_norm": 2.774146795272827, |
| "learning_rate": 4.980674680908192e-06, |
| "logits/chosen": -1.523785948753357, |
| "logits/rejected": -1.4983959197998047, |
| "logps/chosen": -229.0098876953125, |
| "logps/rejected": -248.9108123779297, |
| "loss": 0.3995, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.1047826036810875, |
| "rewards/margins": 1.3212553262710571, |
| "rewards/rejected": -1.216472864151001, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.216, |
| "grad_norm": 3.826845645904541, |
| "learning_rate": 4.9779015102234354e-06, |
| "logits/chosen": -1.429242491722107, |
| "logits/rejected": -1.4095516204833984, |
| "logps/chosen": -210.46408081054688, |
| "logps/rejected": -243.783935546875, |
| "loss": 0.4262, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 0.008653700351715088, |
| "rewards/margins": 1.4706300497055054, |
| "rewards/rejected": -1.461976408958435, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 3.50723934173584, |
| "learning_rate": 4.9749433558927755e-06, |
| "logits/chosen": -1.4507659673690796, |
| "logits/rejected": -1.4398767948150635, |
| "logps/chosen": -236.15261840820312, |
| "logps/rejected": -261.0746154785156, |
| "loss": 0.461, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.05946246534585953, |
| "rewards/margins": 1.1294766664505005, |
| "rewards/rejected": -1.188939094543457, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.232, |
| "grad_norm": 2.5489661693573, |
| "learning_rate": 4.971800438752339e-06, |
| "logits/chosen": -1.422628402709961, |
| "logits/rejected": -1.389582633972168, |
| "logps/chosen": -224.6511688232422, |
| "logps/rejected": -267.15472412109375, |
| "loss": 0.3153, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.26400551199913025, |
| "rewards/margins": 1.8636376857757568, |
| "rewards/rejected": -1.5996323823928833, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 3.3601999282836914, |
| "learning_rate": 4.968472993431418e-06, |
| "logits/chosen": -1.5419387817382812, |
| "logits/rejected": -1.5043963193893433, |
| "logps/chosen": -214.88528442382812, |
| "logps/rejected": -248.4092254638672, |
| "loss": 0.3794, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.07963865250349045, |
| "rewards/margins": 1.558951497077942, |
| "rewards/rejected": -1.479312777519226, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.248, |
| "grad_norm": 3.492460012435913, |
| "learning_rate": 4.964961268334947e-06, |
| "logits/chosen": -1.407315731048584, |
| "logits/rejected": -1.3751634359359741, |
| "logps/chosen": -226.08737182617188, |
| "logps/rejected": -245.1667938232422, |
| "loss": 0.4431, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 0.07714741677045822, |
| "rewards/margins": 1.2392067909240723, |
| "rewards/rejected": -1.1620593070983887, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 2.7841556072235107, |
| "learning_rate": 4.961265525624965e-06, |
| "logits/chosen": -1.5322622060775757, |
| "logits/rejected": -1.4266688823699951, |
| "logps/chosen": -217.79513549804688, |
| "logps/rejected": -250.97207641601562, |
| "loss": 0.3799, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.034860264509916306, |
| "rewards/margins": 1.4884862899780273, |
| "rewards/rejected": -1.5233466625213623, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.264, |
| "grad_norm": 3.2082886695861816, |
| "learning_rate": 4.957386041201043e-06, |
| "logits/chosen": -1.5398249626159668, |
| "logits/rejected": -1.465033769607544, |
| "logps/chosen": -228.68197631835938, |
| "logps/rejected": -235.70950317382812, |
| "loss": 0.3659, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 0.12753064930438995, |
| "rewards/margins": 1.6183111667633057, |
| "rewards/rejected": -1.4907804727554321, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.272, |
| "grad_norm": 3.9094443321228027, |
| "learning_rate": 4.953323104679683e-06, |
| "logits/chosen": -1.416504979133606, |
| "logits/rejected": -1.3369841575622559, |
| "logps/chosen": -224.4219512939453, |
| "logps/rejected": -256.59808349609375, |
| "loss": 0.373, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.28729182481765747, |
| "rewards/margins": 1.6720726490020752, |
| "rewards/rejected": -1.384780764579773, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.28, |
| "grad_norm": 2.4305460453033447, |
| "learning_rate": 4.949077019372704e-06, |
| "logits/chosen": -1.4719858169555664, |
| "logits/rejected": -1.369411587715149, |
| "logps/chosen": -203.30699157714844, |
| "logps/rejected": -248.2655792236328, |
| "loss": 0.3418, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.168064683675766, |
| "rewards/margins": 1.683985948562622, |
| "rewards/rejected": -1.5159209966659546, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 3.1803479194641113, |
| "learning_rate": 4.944648102264594e-06, |
| "logits/chosen": -1.4664204120635986, |
| "logits/rejected": -1.4561368227005005, |
| "logps/chosen": -235.5093994140625, |
| "logps/rejected": -286.78265380859375, |
| "loss": 0.3208, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.2242468148469925, |
| "rewards/margins": 1.8115650415420532, |
| "rewards/rejected": -1.5873181819915771, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.296, |
| "grad_norm": 2.9232826232910156, |
| "learning_rate": 4.940036683988847e-06, |
| "logits/chosen": -1.6076456308364868, |
| "logits/rejected": -1.520662546157837, |
| "logps/chosen": -232.9312286376953, |
| "logps/rejected": -275.1305236816406, |
| "loss": 0.3502, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 0.04518136382102966, |
| "rewards/margins": 1.9395564794540405, |
| "rewards/rejected": -1.894375205039978, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.304, |
| "grad_norm": 3.534334182739258, |
| "learning_rate": 4.9352431088032785e-06, |
| "logits/chosen": -1.5013444423675537, |
| "logits/rejected": -1.4597349166870117, |
| "logps/chosen": -248.06784057617188, |
| "logps/rejected": -275.9266052246094, |
| "loss": 0.3619, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.07667362689971924, |
| "rewards/margins": 1.9556586742401123, |
| "rewards/rejected": -1.878985047340393, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.312, |
| "grad_norm": 2.96315336227417, |
| "learning_rate": 4.9302677345643316e-06, |
| "logits/chosen": -1.4589729309082031, |
| "logits/rejected": -1.4188172817230225, |
| "logps/chosen": -241.571044921875, |
| "logps/rejected": -264.0114440917969, |
| "loss": 0.3528, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.13401958346366882, |
| "rewards/margins": 2.0827994346618652, |
| "rewards/rejected": -1.9487800598144531, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 3.3248233795166016, |
| "learning_rate": 4.925110932700353e-06, |
| "logits/chosen": -1.4326791763305664, |
| "logits/rejected": -1.393992304801941, |
| "logps/chosen": -243.5026397705078, |
| "logps/rejected": -279.20361328125, |
| "loss": 0.2832, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.26788315176963806, |
| "rewards/margins": 2.33567476272583, |
| "rewards/rejected": -2.067791700363159, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.328, |
| "grad_norm": 2.753101348876953, |
| "learning_rate": 4.919773088183873e-06, |
| "logits/chosen": -1.4298564195632935, |
| "logits/rejected": -1.3792085647583008, |
| "logps/chosen": -232.48947143554688, |
| "logps/rejected": -265.4277648925781, |
| "loss": 0.3057, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.29389330744743347, |
| "rewards/margins": 2.062680721282959, |
| "rewards/rejected": -1.768787145614624, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.336, |
| "grad_norm": 3.2286505699157715, |
| "learning_rate": 4.914254599502859e-06, |
| "logits/chosen": -1.6037689447402954, |
| "logits/rejected": -1.5768611431121826, |
| "logps/chosen": -221.110595703125, |
| "logps/rejected": -259.8968200683594, |
| "loss": 0.3139, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.1620008498430252, |
| "rewards/margins": 2.205214262008667, |
| "rewards/rejected": -2.043213367462158, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.344, |
| "grad_norm": 3.4793143272399902, |
| "learning_rate": 4.908555878630973e-06, |
| "logits/chosen": -1.410553216934204, |
| "logits/rejected": -1.416738510131836, |
| "logps/chosen": -198.44369506835938, |
| "logps/rejected": -268.19049072265625, |
| "loss": 0.286, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.47130662202835083, |
| "rewards/margins": 2.549254894256592, |
| "rewards/rejected": -2.0779480934143066, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 2.717116355895996, |
| "learning_rate": 4.902677350996812e-06, |
| "logits/chosen": -1.5396114587783813, |
| "logits/rejected": -1.4347336292266846, |
| "logps/chosen": -226.77890014648438, |
| "logps/rejected": -249.70449829101562, |
| "loss": 0.3559, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.35638585686683655, |
| "rewards/margins": 2.339998960494995, |
| "rewards/rejected": -1.9836132526397705, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 6.953813552856445, |
| "learning_rate": 4.896619455452149e-06, |
| "logits/chosen": -1.4302887916564941, |
| "logits/rejected": -1.3422778844833374, |
| "logps/chosen": -226.4547576904297, |
| "logps/rejected": -247.8949737548828, |
| "loss": 0.3402, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 0.4336090087890625, |
| "rewards/margins": 2.148460865020752, |
| "rewards/rejected": -1.714851975440979, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.368, |
| "grad_norm": 4.407371520996094, |
| "learning_rate": 4.890382644239176e-06, |
| "logits/chosen": -1.5161406993865967, |
| "logits/rejected": -1.4464107751846313, |
| "logps/chosen": -225.75973510742188, |
| "logps/rejected": -264.4316711425781, |
| "loss": 0.3814, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.34316977858543396, |
| "rewards/margins": 1.8401553630828857, |
| "rewards/rejected": -1.496985673904419, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.376, |
| "grad_norm": 4.450686931610107, |
| "learning_rate": 4.8839673829567345e-06, |
| "logits/chosen": -1.5348962545394897, |
| "logits/rejected": -1.4852139949798584, |
| "logps/chosen": -223.89297485351562, |
| "logps/rejected": -273.54058837890625, |
| "loss": 0.346, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.3010365962982178, |
| "rewards/margins": 2.130527973175049, |
| "rewards/rejected": -1.8294916152954102, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 3.7453107833862305, |
| "learning_rate": 4.877374150525563e-06, |
| "logits/chosen": -1.524099349975586, |
| "logits/rejected": -1.4040982723236084, |
| "logps/chosen": -218.1953125, |
| "logps/rejected": -271.7344665527344, |
| "loss": 0.3333, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.542862057685852, |
| "rewards/margins": 2.4943270683288574, |
| "rewards/rejected": -1.9514648914337158, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.392, |
| "grad_norm": 3.430736541748047, |
| "learning_rate": 4.870603439152544e-06, |
| "logits/chosen": -1.4076268672943115, |
| "logits/rejected": -1.3688628673553467, |
| "logps/chosen": -238.49276733398438, |
| "logps/rejected": -286.45098876953125, |
| "loss": 0.2818, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.3373296856880188, |
| "rewards/margins": 2.4323055744171143, |
| "rewards/rejected": -2.0949759483337402, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 3.630533218383789, |
| "learning_rate": 4.863655754293953e-06, |
| "logits/chosen": -1.5296860933303833, |
| "logits/rejected": -1.5080636739730835, |
| "logps/chosen": -240.5646514892578, |
| "logps/rejected": -269.9687805175781, |
| "loss": 0.3282, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.3692770302295685, |
| "rewards/margins": 2.3015217781066895, |
| "rewards/rejected": -1.932244896888733, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.408, |
| "grad_norm": 3.693142890930176, |
| "learning_rate": 4.856531614617734e-06, |
| "logits/chosen": -1.6221294403076172, |
| "logits/rejected": -1.5056837797164917, |
| "logps/chosen": -217.1576690673828, |
| "logps/rejected": -270.16436767578125, |
| "loss": 0.2793, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.34854626655578613, |
| "rewards/margins": 2.3167076110839844, |
| "rewards/rejected": -1.9681613445281982, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 3.790282726287842, |
| "learning_rate": 4.849231551964771e-06, |
| "logits/chosen": -1.5149017572402954, |
| "logits/rejected": -1.4539215564727783, |
| "logps/chosen": -216.43994140625, |
| "logps/rejected": -245.41830444335938, |
| "loss": 0.3103, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.6397329568862915, |
| "rewards/margins": 2.408543109893799, |
| "rewards/rejected": -1.7688102722167969, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.424, |
| "grad_norm": 2.5389716625213623, |
| "learning_rate": 4.841756111309188e-06, |
| "logits/chosen": -1.5661799907684326, |
| "logits/rejected": -1.412497639656067, |
| "logps/chosen": -233.9298858642578, |
| "logps/rejected": -276.6418762207031, |
| "loss": 0.2872, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.543883204460144, |
| "rewards/margins": 2.6593985557556152, |
| "rewards/rejected": -2.1155149936676025, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.432, |
| "grad_norm": 3.379960060119629, |
| "learning_rate": 4.834105850717664e-06, |
| "logits/chosen": -1.558663010597229, |
| "logits/rejected": -1.4975134134292603, |
| "logps/chosen": -216.4776611328125, |
| "logps/rejected": -252.5549774169922, |
| "loss": 0.337, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.36121705174446106, |
| "rewards/margins": 2.108269214630127, |
| "rewards/rejected": -1.7470521926879883, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.44, |
| "grad_norm": 3.4550364017486572, |
| "learning_rate": 4.826281341307772e-06, |
| "logits/chosen": -1.5177628993988037, |
| "logits/rejected": -1.4909249544143677, |
| "logps/chosen": -211.82522583007812, |
| "logps/rejected": -275.4071960449219, |
| "loss": 0.2673, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.43875017762184143, |
| "rewards/margins": 2.2563743591308594, |
| "rewards/rejected": -1.8176243305206299, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 2.3109264373779297, |
| "learning_rate": 4.818283167205341e-06, |
| "logits/chosen": -1.4933243989944458, |
| "logits/rejected": -1.476614236831665, |
| "logps/chosen": -211.17626953125, |
| "logps/rejected": -257.49737548828125, |
| "loss": 0.284, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.6001291275024414, |
| "rewards/margins": 2.5724263191223145, |
| "rewards/rejected": -1.9722973108291626, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.456, |
| "grad_norm": 2.1601908206939697, |
| "learning_rate": 4.8101119255008555e-06, |
| "logits/chosen": -1.6040397882461548, |
| "logits/rejected": -1.5376040935516357, |
| "logps/chosen": -216.3963165283203, |
| "logps/rejected": -285.90106201171875, |
| "loss": 0.2571, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.6661025881767273, |
| "rewards/margins": 2.8501198291778564, |
| "rewards/rejected": -2.1840171813964844, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.464, |
| "grad_norm": 2.8724215030670166, |
| "learning_rate": 4.801768226204871e-06, |
| "logits/chosen": -1.4982980489730835, |
| "logits/rejected": -1.5107654333114624, |
| "logps/chosen": -197.4161376953125, |
| "logps/rejected": -261.05780029296875, |
| "loss": 0.3096, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.3717041313648224, |
| "rewards/margins": 2.240090847015381, |
| "rewards/rejected": -1.8683868646621704, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.472, |
| "grad_norm": 4.054579734802246, |
| "learning_rate": 4.7932526922024835e-06, |
| "logits/chosen": -1.58639395236969, |
| "logits/rejected": -1.514135479927063, |
| "logps/chosen": -224.87435913085938, |
| "logps/rejected": -274.8253479003906, |
| "loss": 0.2683, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.2706889510154724, |
| "rewards/margins": 2.824399471282959, |
| "rewards/rejected": -2.553710460662842, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 2.7186853885650635, |
| "learning_rate": 4.784565959206822e-06, |
| "logits/chosen": -1.6050167083740234, |
| "logits/rejected": -1.5349472761154175, |
| "logps/chosen": -233.95419311523438, |
| "logps/rejected": -278.12042236328125, |
| "loss": 0.2701, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.08019473403692245, |
| "rewards/margins": 2.7215523719787598, |
| "rewards/rejected": -2.641357421875, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.488, |
| "grad_norm": 2.830869436264038, |
| "learning_rate": 4.775708675711597e-06, |
| "logits/chosen": -1.6498082876205444, |
| "logits/rejected": -1.54813551902771, |
| "logps/chosen": -216.80239868164062, |
| "logps/rejected": -275.505126953125, |
| "loss": 0.2549, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.19069986045360565, |
| "rewards/margins": 2.954810380935669, |
| "rewards/rejected": -2.764110565185547, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.496, |
| "grad_norm": 4.789516925811768, |
| "learning_rate": 4.766681502942681e-06, |
| "logits/chosen": -1.5943660736083984, |
| "logits/rejected": -1.6171715259552002, |
| "logps/chosen": -214.1531219482422, |
| "logps/rejected": -270.62957763671875, |
| "loss": 0.2825, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.21599332988262177, |
| "rewards/margins": 2.9400386810302734, |
| "rewards/rejected": -2.7240452766418457, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.504, |
| "grad_norm": 3.499081611633301, |
| "learning_rate": 4.757485114808755e-06, |
| "logits/chosen": -1.6780385971069336, |
| "logits/rejected": -1.5276051759719849, |
| "logps/chosen": -226.2732391357422, |
| "logps/rejected": -281.07330322265625, |
| "loss": 0.2719, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.1714359074831009, |
| "rewards/margins": 2.7567970752716064, |
| "rewards/rejected": -2.5853612422943115, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 3.7923743724823, |
| "learning_rate": 4.748120197850989e-06, |
| "logits/chosen": -1.5633914470672607, |
| "logits/rejected": -1.542341947555542, |
| "logps/chosen": -240.93814086914062, |
| "logps/rejected": -263.87066650390625, |
| "loss": 0.2843, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.3062919080257416, |
| "rewards/margins": 2.8927149772644043, |
| "rewards/rejected": -2.586423397064209, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.52, |
| "grad_norm": 2.524531841278076, |
| "learning_rate": 4.738587451191796e-06, |
| "logits/chosen": -1.5463718175888062, |
| "logits/rejected": -1.4387627840042114, |
| "logps/chosen": -236.073486328125, |
| "logps/rejected": -272.9033203125, |
| "loss": 0.2858, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.3569238483905792, |
| "rewards/margins": 2.676743268966675, |
| "rewards/rejected": -2.319819450378418, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.528, |
| "grad_norm": 4.112205505371094, |
| "learning_rate": 4.728887586482638e-06, |
| "logits/chosen": -1.5366756916046143, |
| "logits/rejected": -1.4600120782852173, |
| "logps/chosen": -245.8846435546875, |
| "logps/rejected": -273.06353759765625, |
| "loss": 0.2451, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.3740922808647156, |
| "rewards/margins": 2.8029963970184326, |
| "rewards/rejected": -2.4289040565490723, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.536, |
| "grad_norm": 2.5791103839874268, |
| "learning_rate": 4.719021327850898e-06, |
| "logits/chosen": -1.4344203472137451, |
| "logits/rejected": -1.4434177875518799, |
| "logps/chosen": -225.71279907226562, |
| "logps/rejected": -262.5049743652344, |
| "loss": 0.2681, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.1705799102783203, |
| "rewards/margins": 2.7896885871887207, |
| "rewards/rejected": -2.6191084384918213, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 3.0821750164031982, |
| "learning_rate": 4.708989411845826e-06, |
| "logits/chosen": -1.6164867877960205, |
| "logits/rejected": -1.4977359771728516, |
| "logps/chosen": -228.20065307617188, |
| "logps/rejected": -264.6740417480469, |
| "loss": 0.2676, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.2144383192062378, |
| "rewards/margins": 2.7585530281066895, |
| "rewards/rejected": -2.544114589691162, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.552, |
| "grad_norm": 3.665308713912964, |
| "learning_rate": 4.698792587383545e-06, |
| "logits/chosen": -1.679888129234314, |
| "logits/rejected": -1.5331531763076782, |
| "logps/chosen": -228.595947265625, |
| "logps/rejected": -261.4881896972656, |
| "loss": 0.2912, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.19902127981185913, |
| "rewards/margins": 2.8159170150756836, |
| "rewards/rejected": -2.616895914077759, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.56, |
| "grad_norm": 2.2218050956726074, |
| "learning_rate": 4.68843161569115e-06, |
| "logits/chosen": -1.6025232076644897, |
| "logits/rejected": -1.6274888515472412, |
| "logps/chosen": -221.0915985107422, |
| "logps/rejected": -275.3337707519531, |
| "loss": 0.2624, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.23736414313316345, |
| "rewards/margins": 2.8529913425445557, |
| "rewards/rejected": -2.6156270503997803, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.568, |
| "grad_norm": 3.0726404190063477, |
| "learning_rate": 4.6779072702498765e-06, |
| "logits/chosen": -1.496206521987915, |
| "logits/rejected": -1.482466459274292, |
| "logps/chosen": -219.1163330078125, |
| "logps/rejected": -258.2294006347656, |
| "loss": 0.2819, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.44380149245262146, |
| "rewards/margins": 2.6352720260620117, |
| "rewards/rejected": -2.1914706230163574, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 3.823976516723633, |
| "learning_rate": 4.667220336737355e-06, |
| "logits/chosen": -1.5284912586212158, |
| "logits/rejected": -1.4588353633880615, |
| "logps/chosen": -224.3351287841797, |
| "logps/rejected": -264.51531982421875, |
| "loss": 0.2491, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.42943182587623596, |
| "rewards/margins": 2.9441678524017334, |
| "rewards/rejected": -2.5147361755371094, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.584, |
| "grad_norm": 4.347540855407715, |
| "learning_rate": 4.656371612968961e-06, |
| "logits/chosen": -1.55275559425354, |
| "logits/rejected": -1.5454585552215576, |
| "logps/chosen": -213.6840057373047, |
| "logps/rejected": -282.0579528808594, |
| "loss": 0.2327, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.4197406768798828, |
| "rewards/margins": 3.0559322834014893, |
| "rewards/rejected": -2.6361916065216064, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.592, |
| "grad_norm": 5.5594611167907715, |
| "learning_rate": 4.645361908838255e-06, |
| "logits/chosen": -1.6658909320831299, |
| "logits/rejected": -1.5418522357940674, |
| "logps/chosen": -224.8914794921875, |
| "logps/rejected": -266.9699401855469, |
| "loss": 0.3323, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.01982409879565239, |
| "rewards/margins": 2.640495777130127, |
| "rewards/rejected": -2.6206717491149902, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 2.3482155799865723, |
| "learning_rate": 4.634192046256521e-06, |
| "logits/chosen": -1.545961618423462, |
| "logits/rejected": -1.5492537021636963, |
| "logps/chosen": -228.63076782226562, |
| "logps/rejected": -293.1945495605469, |
| "loss": 0.2174, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.24786663055419922, |
| "rewards/margins": 3.227076292037964, |
| "rewards/rejected": -2.9792098999023438, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 2.7039475440979004, |
| "learning_rate": 4.622862859091407e-06, |
| "logits/chosen": -1.5828018188476562, |
| "logits/rejected": -1.5519826412200928, |
| "logps/chosen": -224.43521118164062, |
| "logps/rejected": -289.30133056640625, |
| "loss": 0.2281, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.2096446454524994, |
| "rewards/margins": 3.253903865814209, |
| "rewards/rejected": -3.044259548187256, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.616, |
| "grad_norm": 3.9861464500427246, |
| "learning_rate": 4.611375193104674e-06, |
| "logits/chosen": -1.641060471534729, |
| "logits/rejected": -1.563591718673706, |
| "logps/chosen": -218.3530731201172, |
| "logps/rejected": -275.87127685546875, |
| "loss": 0.2379, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.006566214375197887, |
| "rewards/margins": 3.3008251190185547, |
| "rewards/rejected": -3.2942588329315186, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.624, |
| "grad_norm": 2.931182622909546, |
| "learning_rate": 4.59972990588906e-06, |
| "logits/chosen": -1.462402105331421, |
| "logits/rejected": -1.4504549503326416, |
| "logps/chosen": -237.58432006835938, |
| "logps/rejected": -280.5559997558594, |
| "loss": 0.2608, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.006480635609477758, |
| "rewards/margins": 2.936253786087036, |
| "rewards/rejected": -2.9297728538513184, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.632, |
| "grad_norm": 6.512454032897949, |
| "learning_rate": 4.5879278668042525e-06, |
| "logits/chosen": -1.635387659072876, |
| "logits/rejected": -1.543980360031128, |
| "logps/chosen": -225.7213897705078, |
| "logps/rejected": -257.0222473144531, |
| "loss": 0.2732, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.037258487194776535, |
| "rewards/margins": 2.8508574962615967, |
| "rewards/rejected": -2.8881163597106934, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 2.6228084564208984, |
| "learning_rate": 4.575969956911994e-06, |
| "logits/chosen": -1.600145697593689, |
| "logits/rejected": -1.466813325881958, |
| "logps/chosen": -230.7875213623047, |
| "logps/rejected": -274.317138671875, |
| "loss": 0.2125, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.0649040937423706, |
| "rewards/margins": 3.4587504863739014, |
| "rewards/rejected": -3.5236542224884033, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.648, |
| "grad_norm": 2.864257335662842, |
| "learning_rate": 4.563857068910303e-06, |
| "logits/chosen": -1.520639181137085, |
| "logits/rejected": -1.5240384340286255, |
| "logps/chosen": -219.7958221435547, |
| "logps/rejected": -283.594482421875, |
| "loss": 0.1655, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.11086788028478622, |
| "rewards/margins": 3.5582213401794434, |
| "rewards/rejected": -3.4473540782928467, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.656, |
| "grad_norm": 4.094531536102295, |
| "learning_rate": 4.551590107066833e-06, |
| "logits/chosen": -1.6114097833633423, |
| "logits/rejected": -1.581969976425171, |
| "logps/chosen": -270.63824462890625, |
| "logps/rejected": -291.4254150390625, |
| "loss": 0.2999, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.051137734204530716, |
| "rewards/margins": 3.479069232940674, |
| "rewards/rejected": -3.427931547164917, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.664, |
| "grad_norm": 1.4797621965408325, |
| "learning_rate": 4.5391699871513675e-06, |
| "logits/chosen": -1.6318258047103882, |
| "logits/rejected": -1.512136459350586, |
| "logps/chosen": -241.17367553710938, |
| "logps/rejected": -287.22613525390625, |
| "loss": 0.2384, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.08122474700212479, |
| "rewards/margins": 3.71380877494812, |
| "rewards/rejected": -3.6325843334198, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 3.8422563076019287, |
| "learning_rate": 4.526597636367449e-06, |
| "logits/chosen": -1.7238775491714478, |
| "logits/rejected": -1.6469764709472656, |
| "logps/chosen": -243.2270965576172, |
| "logps/rejected": -272.0723876953125, |
| "loss": 0.2496, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.0937575250864029, |
| "rewards/margins": 3.0788111686706543, |
| "rewards/rejected": -3.1725687980651855, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 4.859435558319092, |
| "learning_rate": 4.5138739932831674e-06, |
| "logits/chosen": -1.600683569908142, |
| "logits/rejected": -1.515218734741211, |
| "logps/chosen": -241.6292724609375, |
| "logps/rejected": -278.32427978515625, |
| "loss": 0.2226, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.06983254849910736, |
| "rewards/margins": 3.874213457107544, |
| "rewards/rejected": -3.8043811321258545, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.688, |
| "grad_norm": 3.252007007598877, |
| "learning_rate": 4.501000007761091e-06, |
| "logits/chosen": -1.6320215463638306, |
| "logits/rejected": -1.5925445556640625, |
| "logps/chosen": -229.80099487304688, |
| "logps/rejected": -283.92095947265625, |
| "loss": 0.2189, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.1597936749458313, |
| "rewards/margins": 3.3600659370422363, |
| "rewards/rejected": -3.2002720832824707, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.696, |
| "grad_norm": 3.6351332664489746, |
| "learning_rate": 4.487976640887349e-06, |
| "logits/chosen": -1.576635718345642, |
| "logits/rejected": -1.564374327659607, |
| "logps/chosen": -223.580322265625, |
| "logps/rejected": -283.73175048828125, |
| "loss": 0.2519, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.35887137055397034, |
| "rewards/margins": 3.7305588722229004, |
| "rewards/rejected": -3.3716869354248047, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 4.0519700050354, |
| "learning_rate": 4.474804864899895e-06, |
| "logits/chosen": -1.527977705001831, |
| "logits/rejected": -1.4295679330825806, |
| "logps/chosen": -207.36312866210938, |
| "logps/rejected": -259.65740966796875, |
| "loss": 0.238, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.36125385761260986, |
| "rewards/margins": 3.2277350425720215, |
| "rewards/rejected": -2.866481304168701, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.712, |
| "grad_norm": 4.860156536102295, |
| "learning_rate": 4.461485663115915e-06, |
| "logits/chosen": -1.6169579029083252, |
| "logits/rejected": -1.5100561380386353, |
| "logps/chosen": -230.5595245361328, |
| "logps/rejected": -272.85198974609375, |
| "loss": 0.3059, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.08551453053951263, |
| "rewards/margins": 3.0663275718688965, |
| "rewards/rejected": -2.980813503265381, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 2.4698750972747803, |
| "learning_rate": 4.448020029858427e-06, |
| "logits/chosen": -1.5895106792449951, |
| "logits/rejected": -1.5054939985275269, |
| "logps/chosen": -215.32870483398438, |
| "logps/rejected": -267.3359375, |
| "loss": 0.2383, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.29027166962623596, |
| "rewards/margins": 3.742069959640503, |
| "rewards/rejected": -3.451798677444458, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.728, |
| "grad_norm": 4.764713764190674, |
| "learning_rate": 4.4344089703820495e-06, |
| "logits/chosen": -1.5860201120376587, |
| "logits/rejected": -1.52178156375885, |
| "logps/chosen": -224.6458740234375, |
| "logps/rejected": -301.4439392089844, |
| "loss": 0.2253, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.401482492685318, |
| "rewards/margins": 3.844498872756958, |
| "rewards/rejected": -3.4430160522460938, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 3.805908441543579, |
| "learning_rate": 4.420653500797952e-06, |
| "logits/chosen": -1.4725072383880615, |
| "logits/rejected": -1.4182043075561523, |
| "logps/chosen": -240.71923828125, |
| "logps/rejected": -270.241943359375, |
| "loss": 0.2165, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.3455933928489685, |
| "rewards/margins": 3.414217710494995, |
| "rewards/rejected": -3.068624258041382, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.744, |
| "grad_norm": 2.8406822681427, |
| "learning_rate": 4.406754647998005e-06, |
| "logits/chosen": -1.6234924793243408, |
| "logits/rejected": -1.4973393678665161, |
| "logps/chosen": -218.62863159179688, |
| "logps/rejected": -281.8992004394531, |
| "loss": 0.1714, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.40566954016685486, |
| "rewards/margins": 3.8715057373046875, |
| "rewards/rejected": -3.4658360481262207, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.752, |
| "grad_norm": 6.241238594055176, |
| "learning_rate": 4.3927134495781155e-06, |
| "logits/chosen": -1.5617748498916626, |
| "logits/rejected": -1.5487487316131592, |
| "logps/chosen": -248.231689453125, |
| "logps/rejected": -287.02960205078125, |
| "loss": 0.2686, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.0831371322274208, |
| "rewards/margins": 3.679551601409912, |
| "rewards/rejected": -3.596414566040039, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.76, |
| "grad_norm": 4.16778039932251, |
| "learning_rate": 4.378530953760768e-06, |
| "logits/chosen": -1.4174320697784424, |
| "logits/rejected": -1.4294613599777222, |
| "logps/chosen": -226.9671173095703, |
| "logps/rejected": -263.05389404296875, |
| "loss": 0.2885, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.02728237584233284, |
| "rewards/margins": 2.947277545928955, |
| "rewards/rejected": -2.919994831085205, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 5.187206268310547, |
| "learning_rate": 4.364208219316771e-06, |
| "logits/chosen": -1.5073765516281128, |
| "logits/rejected": -1.5238683223724365, |
| "logps/chosen": -211.14846801757812, |
| "logps/rejected": -280.1331787109375, |
| "loss": 0.2781, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.08308421075344086, |
| "rewards/margins": 3.373748302459717, |
| "rewards/rejected": -3.4568328857421875, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.776, |
| "grad_norm": 3.847803831100464, |
| "learning_rate": 4.349746315486214e-06, |
| "logits/chosen": -1.4829554557800293, |
| "logits/rejected": -1.4054362773895264, |
| "logps/chosen": -224.22915649414062, |
| "logps/rejected": -282.7549133300781, |
| "loss": 0.242, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.16652753949165344, |
| "rewards/margins": 3.6406772136688232, |
| "rewards/rejected": -3.474149703979492, |
| "step": 485 |
| }, |
| { |
| "epoch": 0.784, |
| "grad_norm": 6.660366058349609, |
| "learning_rate": 4.335146321898651e-06, |
| "logits/chosen": -1.597735047340393, |
| "logits/rejected": -1.5115840435028076, |
| "logps/chosen": -221.6646270751953, |
| "logps/rejected": -284.57452392578125, |
| "loss": 0.3001, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.21003933250904083, |
| "rewards/margins": 3.47546124458313, |
| "rewards/rejected": -3.6855006217956543, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.792, |
| "grad_norm": 3.7531986236572266, |
| "learning_rate": 4.320409328492495e-06, |
| "logits/chosen": -1.5909736156463623, |
| "logits/rejected": -1.5427086353302002, |
| "logps/chosen": -235.1774139404297, |
| "logps/rejected": -293.44940185546875, |
| "loss": 0.2523, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.25116807222366333, |
| "rewards/margins": 3.825078248977661, |
| "rewards/rejected": -3.573909282684326, |
| "step": 495 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 4.075142860412598, |
| "learning_rate": 4.305536435433656e-06, |
| "logits/chosen": -1.5668506622314453, |
| "logits/rejected": -1.5530762672424316, |
| "logps/chosen": -222.394775390625, |
| "logps/rejected": -293.10736083984375, |
| "loss": 0.2185, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.26374197006225586, |
| "rewards/margins": 3.8278470039367676, |
| "rewards/rejected": -3.5641047954559326, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.808, |
| "grad_norm": 3.839890956878662, |
| "learning_rate": 4.290528753033411e-06, |
| "logits/chosen": -1.5763019323349, |
| "logits/rejected": -1.5077393054962158, |
| "logps/chosen": -232.4560546875, |
| "logps/rejected": -289.55499267578125, |
| "loss": 0.2744, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.025719815865159035, |
| "rewards/margins": 3.2603039741516113, |
| "rewards/rejected": -3.2860240936279297, |
| "step": 505 |
| }, |
| { |
| "epoch": 0.816, |
| "grad_norm": 5.276324272155762, |
| "learning_rate": 4.275387401665506e-06, |
| "logits/chosen": -1.667394995689392, |
| "logits/rejected": -1.5322387218475342, |
| "logps/chosen": -217.12167358398438, |
| "logps/rejected": -276.91119384765625, |
| "loss": 0.2641, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.012757187709212303, |
| "rewards/margins": 3.3533809185028076, |
| "rewards/rejected": -3.3406238555908203, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.824, |
| "grad_norm": 2.3465888500213623, |
| "learning_rate": 4.2601135116825295e-06, |
| "logits/chosen": -1.5679190158843994, |
| "logits/rejected": -1.5460660457611084, |
| "logps/chosen": -230.0165252685547, |
| "logps/rejected": -277.22540283203125, |
| "loss": 0.2604, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.008122218772768974, |
| "rewards/margins": 3.359611988067627, |
| "rewards/rejected": -3.351489543914795, |
| "step": 515 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 5.303627014160156, |
| "learning_rate": 4.244708223331519e-06, |
| "logits/chosen": -1.5283758640289307, |
| "logits/rejected": -1.4631974697113037, |
| "logps/chosen": -239.413818359375, |
| "logps/rejected": -262.42071533203125, |
| "loss": 0.2643, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.04543369263410568, |
| "rewards/margins": 3.1686339378356934, |
| "rewards/rejected": -3.2140679359436035, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.84, |
| "grad_norm": 4.114057540893555, |
| "learning_rate": 4.229172686668838e-06, |
| "logits/chosen": -1.6217492818832397, |
| "logits/rejected": -1.57975435256958, |
| "logps/chosen": -229.3615264892578, |
| "logps/rejected": -274.1919250488281, |
| "loss": 0.2762, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.3959481418132782, |
| "rewards/margins": 3.1732304096221924, |
| "rewards/rejected": -3.569178819656372, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.848, |
| "grad_norm": 6.136798858642578, |
| "learning_rate": 4.213508061474327e-06, |
| "logits/chosen": -1.7025260925292969, |
| "logits/rejected": -1.5809694528579712, |
| "logps/chosen": -205.9241180419922, |
| "logps/rejected": -291.3397521972656, |
| "loss": 0.2371, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.26137492060661316, |
| "rewards/margins": 3.5369045734405518, |
| "rewards/rejected": -3.798279285430908, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.856, |
| "grad_norm": 2.6232376098632812, |
| "learning_rate": 4.197715517164716e-06, |
| "logits/chosen": -1.638715147972107, |
| "logits/rejected": -1.539113163948059, |
| "logps/chosen": -238.2532196044922, |
| "logps/rejected": -268.07415771484375, |
| "loss": 0.2408, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.18916237354278564, |
| "rewards/margins": 3.497271776199341, |
| "rewards/rejected": -3.686434268951416, |
| "step": 535 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 3.9021799564361572, |
| "learning_rate": 4.181796232706322e-06, |
| "logits/chosen": -1.6269233226776123, |
| "logits/rejected": -1.5677905082702637, |
| "logps/chosen": -219.751708984375, |
| "logps/rejected": -256.9467468261719, |
| "loss": 0.2363, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3333657681941986, |
| "rewards/margins": 3.4952011108398438, |
| "rewards/rejected": -3.828566789627075, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.872, |
| "grad_norm": 3.2338857650756836, |
| "learning_rate": 4.165751396527044e-06, |
| "logits/chosen": -1.5140718221664429, |
| "logits/rejected": -1.5093212127685547, |
| "logps/chosen": -235.4116668701172, |
| "logps/rejected": -303.4933166503906, |
| "loss": 0.2068, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.029620448127388954, |
| "rewards/margins": 3.950321674346924, |
| "rewards/rejected": -3.9799418449401855, |
| "step": 545 |
| }, |
| { |
| "epoch": 0.88, |
| "grad_norm": 3.0233776569366455, |
| "learning_rate": 4.149582206427634e-06, |
| "logits/chosen": -1.6049916744232178, |
| "logits/rejected": -1.614729642868042, |
| "logps/chosen": -235.3531494140625, |
| "logps/rejected": -303.1663513183594, |
| "loss": 0.2307, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.004267203621566296, |
| "rewards/margins": 3.4955551624298096, |
| "rewards/rejected": -3.49128794670105, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.888, |
| "grad_norm": 4.327015399932861, |
| "learning_rate": 4.133289869492281e-06, |
| "logits/chosen": -1.655649185180664, |
| "logits/rejected": -1.6161320209503174, |
| "logps/chosen": -228.37173461914062, |
| "logps/rejected": -289.50726318359375, |
| "loss": 0.2433, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.171146959066391, |
| "rewards/margins": 3.6598048210144043, |
| "rewards/rejected": -3.8309521675109863, |
| "step": 555 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 2.6060330867767334, |
| "learning_rate": 4.116875601998499e-06, |
| "logits/chosen": -1.5099356174468994, |
| "logits/rejected": -1.533095359802246, |
| "logps/chosen": -209.5348663330078, |
| "logps/rejected": -287.14080810546875, |
| "loss": 0.1912, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.30249691009521484, |
| "rewards/margins": 3.736576557159424, |
| "rewards/rejected": -3.434079647064209, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.904, |
| "grad_norm": 3.044135093688965, |
| "learning_rate": 4.100340629326324e-06, |
| "logits/chosen": -1.59171724319458, |
| "logits/rejected": -1.4727497100830078, |
| "logps/chosen": -231.43966674804688, |
| "logps/rejected": -269.6001281738281, |
| "loss": 0.2462, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.26826876401901245, |
| "rewards/margins": 3.4600913524627686, |
| "rewards/rejected": -3.1918225288391113, |
| "step": 565 |
| }, |
| { |
| "epoch": 0.912, |
| "grad_norm": 2.689462900161743, |
| "learning_rate": 4.083686185866841e-06, |
| "logits/chosen": -1.5678335428237915, |
| "logits/rejected": -1.5138065814971924, |
| "logps/chosen": -239.888671875, |
| "logps/rejected": -290.61932373046875, |
| "loss": 0.1873, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.33111506700515747, |
| "rewards/margins": 3.883434772491455, |
| "rewards/rejected": -3.5523197650909424, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.92, |
| "grad_norm": 2.54464054107666, |
| "learning_rate": 4.0669135149300274e-06, |
| "logits/chosen": -1.5594810247421265, |
| "logits/rejected": -1.4758410453796387, |
| "logps/chosen": -217.28890991210938, |
| "logps/rejected": -288.6588134765625, |
| "loss": 0.2231, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.2762870788574219, |
| "rewards/margins": 3.7494559288024902, |
| "rewards/rejected": -3.4731688499450684, |
| "step": 575 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 3.501030206680298, |
| "learning_rate": 4.050023868651938e-06, |
| "logits/chosen": -1.7346251010894775, |
| "logits/rejected": -1.639661431312561, |
| "logps/chosen": -203.6880340576172, |
| "logps/rejected": -278.46197509765625, |
| "loss": 0.2775, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.0705312117934227, |
| "rewards/margins": 3.4696056842803955, |
| "rewards/rejected": -3.3990745544433594, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.936, |
| "grad_norm": 3.608591318130493, |
| "learning_rate": 4.033018507901229e-06, |
| "logits/chosen": -1.57565438747406, |
| "logits/rejected": -1.4335561990737915, |
| "logps/chosen": -250.42318725585938, |
| "logps/rejected": -307.19134521484375, |
| "loss": 0.2373, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.0479436032474041, |
| "rewards/margins": 3.5765857696533203, |
| "rewards/rejected": -3.52864146232605, |
| "step": 585 |
| }, |
| { |
| "epoch": 0.944, |
| "grad_norm": 4.200185298919678, |
| "learning_rate": 4.015898702185028e-06, |
| "logits/chosen": -1.4519469738006592, |
| "logits/rejected": -1.4604626893997192, |
| "logps/chosen": -236.66989135742188, |
| "logps/rejected": -282.889892578125, |
| "loss": 0.1888, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.17684012651443481, |
| "rewards/margins": 3.7582874298095703, |
| "rewards/rejected": -3.5814476013183594, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.952, |
| "grad_norm": 4.1930975914001465, |
| "learning_rate": 3.998665729554165e-06, |
| "logits/chosen": -1.4375026226043701, |
| "logits/rejected": -1.3792469501495361, |
| "logps/chosen": -226.0066680908203, |
| "logps/rejected": -287.548828125, |
| "loss": 0.2448, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.20100080966949463, |
| "rewards/margins": 4.001578330993652, |
| "rewards/rejected": -3.8005776405334473, |
| "step": 595 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 1.7681405544281006, |
| "learning_rate": 3.9813208765077575e-06, |
| "logits/chosen": -1.5724737644195557, |
| "logits/rejected": -1.559664249420166, |
| "logps/chosen": -230.7184600830078, |
| "logps/rejected": -300.39178466796875, |
| "loss": 0.2479, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.04428700730204582, |
| "rewards/margins": 3.71527099609375, |
| "rewards/rejected": -3.7595577239990234, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.968, |
| "grad_norm": 2.4093072414398193, |
| "learning_rate": 3.96386543789717e-06, |
| "logits/chosen": -1.5758655071258545, |
| "logits/rejected": -1.526741623878479, |
| "logps/chosen": -222.5027618408203, |
| "logps/rejected": -270.0567626953125, |
| "loss": 0.2558, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.05063873529434204, |
| "rewards/margins": 3.403191089630127, |
| "rewards/rejected": -3.4538300037384033, |
| "step": 605 |
| }, |
| { |
| "epoch": 0.976, |
| "grad_norm": 4.265599727630615, |
| "learning_rate": 3.946300716829352e-06, |
| "logits/chosen": -1.5326379537582397, |
| "logits/rejected": -1.479724645614624, |
| "logps/chosen": -223.6086883544922, |
| "logps/rejected": -298.55877685546875, |
| "loss": 0.2146, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.2664863169193268, |
| "rewards/margins": 4.003918647766113, |
| "rewards/rejected": -3.7374324798583984, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.984, |
| "grad_norm": 6.780034065246582, |
| "learning_rate": 3.9286280245695526e-06, |
| "logits/chosen": -1.5027611255645752, |
| "logits/rejected": -1.3937050104141235, |
| "logps/chosen": -216.69552612304688, |
| "logps/rejected": -277.9105224609375, |
| "loss": 0.2438, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.20606736838817596, |
| "rewards/margins": 3.780785083770752, |
| "rewards/rejected": -3.5747172832489014, |
| "step": 615 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 3.8066725730895996, |
| "learning_rate": 3.910848680443432e-06, |
| "logits/chosen": -1.543540596961975, |
| "logits/rejected": -1.5942132472991943, |
| "logps/chosen": -218.4105682373047, |
| "logps/rejected": -291.05810546875, |
| "loss": 0.2594, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.14096443355083466, |
| "rewards/margins": 3.5304324626922607, |
| "rewards/rejected": -3.6713974475860596, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 4.143672466278076, |
| "learning_rate": 3.892964011738571e-06, |
| "logits/chosen": -1.5397661924362183, |
| "logits/rejected": -1.476738691329956, |
| "logps/chosen": -229.1673126220703, |
| "logps/rejected": -285.22662353515625, |
| "loss": 0.2476, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.1350642591714859, |
| "rewards/margins": 3.7622408866882324, |
| "rewards/rejected": -3.627176284790039, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.008, |
| "grad_norm": 2.3031466007232666, |
| "learning_rate": 3.874975353605382e-06, |
| "logits/chosen": -1.4658385515213013, |
| "logits/rejected": -1.352839469909668, |
| "logps/chosen": -234.90890502929688, |
| "logps/rejected": -282.998046875, |
| "loss": 0.2004, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.3079695403575897, |
| "rewards/margins": 3.906391143798828, |
| "rewards/rejected": -3.598421573638916, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.016, |
| "grad_norm": 3.11726975440979, |
| "learning_rate": 3.856884048957435e-06, |
| "logits/chosen": -1.4668641090393066, |
| "logits/rejected": -1.4360945224761963, |
| "logps/chosen": -215.73709106445312, |
| "logps/rejected": -278.59356689453125, |
| "loss": 0.2069, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.2725924849510193, |
| "rewards/margins": 3.960139513015747, |
| "rewards/rejected": -3.687546968460083, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.024, |
| "grad_norm": 3.6341516971588135, |
| "learning_rate": 3.8386914483712085e-06, |
| "logits/chosen": -1.6500327587127686, |
| "logits/rejected": -1.5192759037017822, |
| "logps/chosen": -222.8524627685547, |
| "logps/rejected": -287.26336669921875, |
| "loss": 0.2025, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.19326825439929962, |
| "rewards/margins": 3.847665786743164, |
| "rewards/rejected": -3.6543972492218018, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.032, |
| "grad_norm": 3.898862600326538, |
| "learning_rate": 3.82039890998526e-06, |
| "logits/chosen": -1.578674077987671, |
| "logits/rejected": -1.521864652633667, |
| "logps/chosen": -231.91354370117188, |
| "logps/rejected": -285.1756896972656, |
| "loss": 0.202, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.09104955941438675, |
| "rewards/margins": 3.9102611541748047, |
| "rewards/rejected": -3.81921124458313, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.04, |
| "grad_norm": 3.565178394317627, |
| "learning_rate": 3.8020077993988392e-06, |
| "logits/chosen": -1.6293882131576538, |
| "logits/rejected": -1.5812406539916992, |
| "logps/chosen": -216.76052856445312, |
| "logps/rejected": -261.8741149902344, |
| "loss": 0.2163, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.07969091832637787, |
| "rewards/margins": 3.7176177501678467, |
| "rewards/rejected": -3.6379265785217285, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.048, |
| "grad_norm": 4.372779369354248, |
| "learning_rate": 3.783519489569941e-06, |
| "logits/chosen": -1.6112524271011353, |
| "logits/rejected": -1.5277847051620483, |
| "logps/chosen": -206.58047485351562, |
| "logps/rejected": -293.06402587890625, |
| "loss": 0.1932, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.17493240535259247, |
| "rewards/margins": 4.243735313415527, |
| "rewards/rejected": -4.068803310394287, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.056, |
| "grad_norm": 3.313710927963257, |
| "learning_rate": 3.76493536071281e-06, |
| "logits/chosen": -1.6428502798080444, |
| "logits/rejected": -1.5417510271072388, |
| "logps/chosen": -219.9541473388672, |
| "logps/rejected": -294.42877197265625, |
| "loss": 0.2107, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.21960565447807312, |
| "rewards/margins": 3.8889222145080566, |
| "rewards/rejected": -3.669316053390503, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.064, |
| "grad_norm": 2.9225704669952393, |
| "learning_rate": 3.7462568001948998e-06, |
| "logits/chosen": -1.4473567008972168, |
| "logits/rejected": -1.4605478048324585, |
| "logps/chosen": -243.60775756835938, |
| "logps/rejected": -293.059814453125, |
| "loss": 0.1868, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.031231069937348366, |
| "rewards/margins": 4.10404109954834, |
| "rewards/rejected": -4.07280969619751, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.072, |
| "grad_norm": 5.222462177276611, |
| "learning_rate": 3.727485202433306e-06, |
| "logits/chosen": -1.565495491027832, |
| "logits/rejected": -1.5146604776382446, |
| "logps/chosen": -225.4569854736328, |
| "logps/rejected": -277.9839172363281, |
| "loss": 0.2113, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.06461232900619507, |
| "rewards/margins": 4.261273384094238, |
| "rewards/rejected": -4.325885772705078, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.08, |
| "grad_norm": 3.0117835998535156, |
| "learning_rate": 3.708621968790664e-06, |
| "logits/chosen": -1.4770100116729736, |
| "logits/rejected": -1.4104207754135132, |
| "logps/chosen": -217.6276397705078, |
| "logps/rejected": -272.0361633300781, |
| "loss": 0.1713, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.24164548516273499, |
| "rewards/margins": 4.098926067352295, |
| "rewards/rejected": -3.857280731201172, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.088, |
| "grad_norm": 3.177833318710327, |
| "learning_rate": 3.689668507470538e-06, |
| "logits/chosen": -1.500281572341919, |
| "logits/rejected": -1.4820690155029297, |
| "logps/chosen": -220.1053466796875, |
| "logps/rejected": -289.69903564453125, |
| "loss": 0.1829, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.013184416107833385, |
| "rewards/margins": 4.457877159118652, |
| "rewards/rejected": -4.444692611694336, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.096, |
| "grad_norm": 4.731311321258545, |
| "learning_rate": 3.670626233412286e-06, |
| "logits/chosen": -1.5912206172943115, |
| "logits/rejected": -1.5220589637756348, |
| "logps/chosen": -201.251708984375, |
| "logps/rejected": -270.2958984375, |
| "loss": 0.2014, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.341642826795578, |
| "rewards/margins": 4.2412543296813965, |
| "rewards/rejected": -4.582897186279297, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.104, |
| "grad_norm": 2.3776421546936035, |
| "learning_rate": 3.651496568185437e-06, |
| "logits/chosen": -1.5553052425384521, |
| "logits/rejected": -1.594269037246704, |
| "logps/chosen": -232.0573272705078, |
| "logps/rejected": -281.67572021484375, |
| "loss": 0.1584, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.1348535120487213, |
| "rewards/margins": 4.210672855377197, |
| "rewards/rejected": -4.075818061828613, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.112, |
| "grad_norm": 2.743382215499878, |
| "learning_rate": 3.6322809398835633e-06, |
| "logits/chosen": -1.56232488155365, |
| "logits/rejected": -1.6043574810028076, |
| "logps/chosen": -231.5035400390625, |
| "logps/rejected": -310.3698425292969, |
| "loss": 0.1365, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.3358309268951416, |
| "rewards/margins": 4.229272365570068, |
| "rewards/rejected": -4.565102577209473, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.12, |
| "grad_norm": 5.52133846282959, |
| "learning_rate": 3.6129807830176677e-06, |
| "logits/chosen": -1.5213687419891357, |
| "logits/rejected": -1.427746295928955, |
| "logps/chosen": -221.33627319335938, |
| "logps/rejected": -293.8585510253906, |
| "loss": 0.1724, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.30416998267173767, |
| "rewards/margins": 5.004787445068359, |
| "rewards/rejected": -5.308957099914551, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.1280000000000001, |
| "grad_norm": 2.9546468257904053, |
| "learning_rate": 3.593597538409095e-06, |
| "logits/chosen": -1.595198154449463, |
| "logits/rejected": -1.5737626552581787, |
| "logps/chosen": -228.0746307373047, |
| "logps/rejected": -294.04345703125, |
| "loss": 0.2306, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.3983309268951416, |
| "rewards/margins": 4.113828659057617, |
| "rewards/rejected": -4.512159824371338, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.1360000000000001, |
| "grad_norm": 4.587896347045898, |
| "learning_rate": 3.574132653081966e-06, |
| "logits/chosen": -1.594580888748169, |
| "logits/rejected": -1.5994797945022583, |
| "logps/chosen": -263.15087890625, |
| "logps/rejected": -298.9627990722656, |
| "loss": 0.2129, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.20374688506126404, |
| "rewards/margins": 4.656205654144287, |
| "rewards/rejected": -4.859952449798584, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.144, |
| "grad_norm": 2.239849090576172, |
| "learning_rate": 3.554587580155156e-06, |
| "logits/chosen": -1.5059843063354492, |
| "logits/rejected": -1.4877843856811523, |
| "logps/chosen": -234.9763641357422, |
| "logps/rejected": -300.68017578125, |
| "loss": 0.1998, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.6419249176979065, |
| "rewards/margins": 4.179691791534424, |
| "rewards/rejected": -4.821616172790527, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.152, |
| "grad_norm": 2.508899211883545, |
| "learning_rate": 3.534963778733813e-06, |
| "logits/chosen": -1.5398582220077515, |
| "logits/rejected": -1.4734686613082886, |
| "logps/chosen": -233.4082489013672, |
| "logps/rejected": -288.84893798828125, |
| "loss": 0.228, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.31519871950149536, |
| "rewards/margins": 4.3696699142456055, |
| "rewards/rejected": -4.684868812561035, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.16, |
| "grad_norm": 4.6924543380737305, |
| "learning_rate": 3.5152627138004307e-06, |
| "logits/chosen": -1.5623716115951538, |
| "logits/rejected": -1.501668930053711, |
| "logps/chosen": -218.7466583251953, |
| "logps/rejected": -302.3932800292969, |
| "loss": 0.1949, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.20108382403850555, |
| "rewards/margins": 4.127780914306641, |
| "rewards/rejected": -4.328865051269531, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.168, |
| "grad_norm": 1.3646442890167236, |
| "learning_rate": 3.495485856105483e-06, |
| "logits/chosen": -1.5090452432632446, |
| "logits/rejected": -1.4874252080917358, |
| "logps/chosen": -222.24996948242188, |
| "logps/rejected": -288.701171875, |
| "loss": 0.1532, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.22113752365112305, |
| "rewards/margins": 5.008795261383057, |
| "rewards/rejected": -5.2299323081970215, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.176, |
| "grad_norm": 2.9527175426483154, |
| "learning_rate": 3.4756346820576254e-06, |
| "logits/chosen": -1.5279325246810913, |
| "logits/rejected": -1.4767868518829346, |
| "logps/chosen": -224.994140625, |
| "logps/rejected": -298.9396057128906, |
| "loss": 0.1771, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.06507287174463272, |
| "rewards/margins": 5.222357273101807, |
| "rewards/rejected": -5.287430286407471, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.184, |
| "grad_norm": 3.2858221530914307, |
| "learning_rate": 3.455710673613481e-06, |
| "logits/chosen": -1.5745080709457397, |
| "logits/rejected": -1.5177898406982422, |
| "logps/chosen": -242.11489868164062, |
| "logps/rejected": -295.5982360839844, |
| "loss": 0.176, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.3049863874912262, |
| "rewards/margins": 4.537248611450195, |
| "rewards/rejected": -4.8422346115112305, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.192, |
| "grad_norm": 3.9363486766815186, |
| "learning_rate": 3.4357153181670026e-06, |
| "logits/chosen": -1.469070315361023, |
| "logits/rejected": -1.4067113399505615, |
| "logps/chosen": -249.7563018798828, |
| "logps/rejected": -298.33514404296875, |
| "loss": 0.1697, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.2381099909543991, |
| "rewards/margins": 4.4300432205200195, |
| "rewards/rejected": -4.668152809143066, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.2, |
| "grad_norm": 4.966811180114746, |
| "learning_rate": 3.4156501084384366e-06, |
| "logits/chosen": -1.6039005517959595, |
| "logits/rejected": -1.4729602336883545, |
| "logps/chosen": -273.5932312011719, |
| "logps/rejected": -294.7056579589844, |
| "loss": 0.1801, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.16059216856956482, |
| "rewards/margins": 5.157106876373291, |
| "rewards/rejected": -4.996514797210693, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.208, |
| "grad_norm": 4.185051441192627, |
| "learning_rate": 3.3955165423628837e-06, |
| "logits/chosen": -1.477455496788025, |
| "logits/rejected": -1.4620649814605713, |
| "logps/chosen": -234.0481414794922, |
| "logps/rejected": -306.4354553222656, |
| "loss": 0.1811, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.3949986398220062, |
| "rewards/margins": 5.021819114685059, |
| "rewards/rejected": -5.416817665100098, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.216, |
| "grad_norm": 2.5614964962005615, |
| "learning_rate": 3.375316122978477e-06, |
| "logits/chosen": -1.4299887418746948, |
| "logits/rejected": -1.5205398797988892, |
| "logps/chosen": -221.0004119873047, |
| "logps/rejected": -303.17156982421875, |
| "loss": 0.2033, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.15307825803756714, |
| "rewards/margins": 4.517648696899414, |
| "rewards/rejected": -4.670726776123047, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.224, |
| "grad_norm": 4.064622402191162, |
| "learning_rate": 3.3550503583141726e-06, |
| "logits/chosen": -1.52488374710083, |
| "logits/rejected": -1.53218674659729, |
| "logps/chosen": -217.2790069580078, |
| "logps/rejected": -300.43756103515625, |
| "loss": 0.1503, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.1853751242160797, |
| "rewards/margins": 4.725203037261963, |
| "rewards/rejected": -4.910577774047852, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.232, |
| "grad_norm": 3.794024705886841, |
| "learning_rate": 3.3347207612771685e-06, |
| "logits/chosen": -1.5807452201843262, |
| "logits/rejected": -1.4922082424163818, |
| "logps/chosen": -228.3516387939453, |
| "logps/rejected": -306.33966064453125, |
| "loss": 0.1985, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.4110232889652252, |
| "rewards/margins": 4.793760776519775, |
| "rewards/rejected": -5.204783916473389, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.24, |
| "grad_norm": 2.162344217300415, |
| "learning_rate": 3.3143288495399663e-06, |
| "logits/chosen": -1.5615044832229614, |
| "logits/rejected": -1.5205762386322021, |
| "logps/chosen": -224.5255126953125, |
| "logps/rejected": -294.60888671875, |
| "loss": 0.1701, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.17508749663829803, |
| "rewards/margins": 4.54577112197876, |
| "rewards/rejected": -4.720858573913574, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.248, |
| "grad_norm": 3.240323781967163, |
| "learning_rate": 3.2938761454270652e-06, |
| "logits/chosen": -1.5861835479736328, |
| "logits/rejected": -1.5375291109085083, |
| "logps/chosen": -216.03616333007812, |
| "logps/rejected": -306.78656005859375, |
| "loss": 0.1846, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.41480645537376404, |
| "rewards/margins": 4.762531280517578, |
| "rewards/rejected": -5.177338123321533, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.256, |
| "grad_norm": 6.329495429992676, |
| "learning_rate": 3.2733641758013224e-06, |
| "logits/chosen": -1.5894248485565186, |
| "logits/rejected": -1.486215591430664, |
| "logps/chosen": -236.5580291748047, |
| "logps/rejected": -283.81109619140625, |
| "loss": 0.227, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.3073071837425232, |
| "rewards/margins": 4.490199089050293, |
| "rewards/rejected": -4.797506809234619, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.264, |
| "grad_norm": 3.0770888328552246, |
| "learning_rate": 3.2527944719499614e-06, |
| "logits/chosen": -1.6199833154678345, |
| "logits/rejected": -1.5427840948104858, |
| "logps/chosen": -226.6359405517578, |
| "logps/rejected": -277.6859130859375, |
| "loss": 0.1648, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.3684183955192566, |
| "rewards/margins": 4.242851734161377, |
| "rewards/rejected": -4.611270427703857, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.272, |
| "grad_norm": 3.9166150093078613, |
| "learning_rate": 3.2321685694702604e-06, |
| "logits/chosen": -1.5863549709320068, |
| "logits/rejected": -1.5562946796417236, |
| "logps/chosen": -221.8000030517578, |
| "logps/rejected": -295.9337158203125, |
| "loss": 0.1567, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.2886640429496765, |
| "rewards/margins": 4.546719551086426, |
| "rewards/rejected": -4.835383892059326, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.28, |
| "grad_norm": 3.533409595489502, |
| "learning_rate": 3.211488008154913e-06, |
| "logits/chosen": -1.5623531341552734, |
| "logits/rejected": -1.5109331607818604, |
| "logps/chosen": -241.4813690185547, |
| "logps/rejected": -311.42047119140625, |
| "loss": 0.167, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.16028550267219543, |
| "rewards/margins": 4.911408424377441, |
| "rewards/rejected": -5.071693420410156, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.288, |
| "grad_norm": 4.332025527954102, |
| "learning_rate": 3.190754331877076e-06, |
| "logits/chosen": -1.6541258096694946, |
| "logits/rejected": -1.5064847469329834, |
| "logps/chosen": -229.9279327392578, |
| "logps/rejected": -264.9905700683594, |
| "loss": 0.1983, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.24898216128349304, |
| "rewards/margins": 4.498307704925537, |
| "rewards/rejected": -4.747290134429932, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.296, |
| "grad_norm": 3.0773584842681885, |
| "learning_rate": 3.1699690884751205e-06, |
| "logits/chosen": -1.5797970294952393, |
| "logits/rejected": -1.5334302186965942, |
| "logps/chosen": -223.06967163085938, |
| "logps/rejected": -284.0609436035156, |
| "loss": 0.1923, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.35305580496788025, |
| "rewards/margins": 4.535957336425781, |
| "rewards/rejected": -4.889012336730957, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.304, |
| "grad_norm": 4.666840553283691, |
| "learning_rate": 3.149133829637072e-06, |
| "logits/chosen": -1.4840573072433472, |
| "logits/rejected": -1.4880752563476562, |
| "logps/chosen": -220.59378051757812, |
| "logps/rejected": -295.93572998046875, |
| "loss": 0.1604, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5703553557395935, |
| "rewards/margins": 4.865350246429443, |
| "rewards/rejected": -5.435705661773682, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.312, |
| "grad_norm": 3.065351963043213, |
| "learning_rate": 3.128250110784778e-06, |
| "logits/chosen": -1.6565717458724976, |
| "logits/rejected": -1.481231451034546, |
| "logps/chosen": -255.04312133789062, |
| "logps/rejected": -284.08197021484375, |
| "loss": 0.1835, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.38553112745285034, |
| "rewards/margins": 4.551497459411621, |
| "rewards/rejected": -4.937028884887695, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.32, |
| "grad_norm": 3.423949956893921, |
| "learning_rate": 3.107319490957787e-06, |
| "logits/chosen": -1.566410779953003, |
| "logits/rejected": -1.5307550430297852, |
| "logps/chosen": -235.0450897216797, |
| "logps/rejected": -312.58770751953125, |
| "loss": 0.186, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5675615668296814, |
| "rewards/margins": 4.500984191894531, |
| "rewards/rejected": -5.068546295166016, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.328, |
| "grad_norm": 1.9515182971954346, |
| "learning_rate": 3.0863435326969636e-06, |
| "logits/chosen": -1.5598112344741821, |
| "logits/rejected": -1.480304479598999, |
| "logps/chosen": -249.81643676757812, |
| "logps/rejected": -297.97186279296875, |
| "loss": 0.1759, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.40629854798316956, |
| "rewards/margins": 4.989799499511719, |
| "rewards/rejected": -5.3960981369018555, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.336, |
| "grad_norm": 3.0678558349609375, |
| "learning_rate": 3.065323801927837e-06, |
| "logits/chosen": -1.6347713470458984, |
| "logits/rejected": -1.5678890943527222, |
| "logps/chosen": -239.1768035888672, |
| "logps/rejected": -304.45159912109375, |
| "loss": 0.1399, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4652630686759949, |
| "rewards/margins": 4.925078392028809, |
| "rewards/rejected": -5.390341281890869, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.3439999999999999, |
| "grad_norm": 1.6803035736083984, |
| "learning_rate": 3.044261867843702e-06, |
| "logits/chosen": -1.600610375404358, |
| "logits/rejected": -1.547503113746643, |
| "logps/chosen": -242.60928344726562, |
| "logps/rejected": -299.3851013183594, |
| "loss": 0.1305, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.09166824072599411, |
| "rewards/margins": 4.836394309997559, |
| "rewards/rejected": -4.92806339263916, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.3519999999999999, |
| "grad_norm": 3.992915153503418, |
| "learning_rate": 3.0231593027884713e-06, |
| "logits/chosen": -1.715815544128418, |
| "logits/rejected": -1.7157901525497437, |
| "logps/chosen": -217.67306518554688, |
| "logps/rejected": -314.8918151855469, |
| "loss": 0.2032, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.4734416902065277, |
| "rewards/margins": 4.529386043548584, |
| "rewards/rejected": -5.0028276443481445, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.3599999999999999, |
| "grad_norm": 2.1506552696228027, |
| "learning_rate": 3.0020176821392965e-06, |
| "logits/chosen": -1.5611059665679932, |
| "logits/rejected": -1.5400519371032715, |
| "logps/chosen": -217.276611328125, |
| "logps/rejected": -293.49066162109375, |
| "loss": 0.1648, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.31788164377212524, |
| "rewards/margins": 4.510266304016113, |
| "rewards/rejected": -4.828147888183594, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.3679999999999999, |
| "grad_norm": 5.309419631958008, |
| "learning_rate": 2.9808385841889588e-06, |
| "logits/chosen": -1.4734315872192383, |
| "logits/rejected": -1.484628438949585, |
| "logps/chosen": -231.23458862304688, |
| "logps/rejected": -303.7582092285156, |
| "loss": 0.2078, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.34313979744911194, |
| "rewards/margins": 4.747766494750977, |
| "rewards/rejected": -5.090906620025635, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.376, |
| "grad_norm": 4.189775466918945, |
| "learning_rate": 2.9596235900280463e-06, |
| "logits/chosen": -1.59661865234375, |
| "logits/rejected": -1.595775842666626, |
| "logps/chosen": -241.6797637939453, |
| "logps/rejected": -294.72186279296875, |
| "loss": 0.1996, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.4572688937187195, |
| "rewards/margins": 4.40045166015625, |
| "rewards/rejected": -4.857720851898193, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.384, |
| "grad_norm": 3.6436808109283447, |
| "learning_rate": 2.9383742834269195e-06, |
| "logits/chosen": -1.5166704654693604, |
| "logits/rejected": -1.5094910860061646, |
| "logps/chosen": -213.38858032226562, |
| "logps/rejected": -299.029541015625, |
| "loss": 0.2195, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.3637376129627228, |
| "rewards/margins": 4.762623310089111, |
| "rewards/rejected": -5.126360893249512, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.392, |
| "grad_norm": 3.5518014430999756, |
| "learning_rate": 2.917092250717478e-06, |
| "logits/chosen": -1.548153042793274, |
| "logits/rejected": -1.4777790307998657, |
| "logps/chosen": -221.06396484375, |
| "logps/rejected": -300.5973205566406, |
| "loss": 0.1963, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.16941462457180023, |
| "rewards/margins": 5.223523139953613, |
| "rewards/rejected": -5.392938137054443, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 4.208786487579346, |
| "learning_rate": 2.895779080674734e-06, |
| "logits/chosen": -1.496821403503418, |
| "logits/rejected": -1.44963538646698, |
| "logps/chosen": -246.3695068359375, |
| "logps/rejected": -278.2579345703125, |
| "loss": 0.238, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.2623414099216461, |
| "rewards/margins": 3.9486358165740967, |
| "rewards/rejected": -4.210977077484131, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.408, |
| "grad_norm": 5.761550426483154, |
| "learning_rate": 2.8744363643982043e-06, |
| "logits/chosen": -1.5691251754760742, |
| "logits/rejected": -1.5343207120895386, |
| "logps/chosen": -227.2958526611328, |
| "logps/rejected": -284.8408203125, |
| "loss": 0.2237, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.3449009656906128, |
| "rewards/margins": 4.538619518280029, |
| "rewards/rejected": -4.88352108001709, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.416, |
| "grad_norm": 3.3380019664764404, |
| "learning_rate": 2.853065695193135e-06, |
| "logits/chosen": -1.4605835676193237, |
| "logits/rejected": -1.4047877788543701, |
| "logps/chosen": -232.16897583007812, |
| "logps/rejected": -295.48626708984375, |
| "loss": 0.1473, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.09790012985467911, |
| "rewards/margins": 4.885996341705322, |
| "rewards/rejected": -4.788095951080322, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.424, |
| "grad_norm": 2.6631267070770264, |
| "learning_rate": 2.831668668451549e-06, |
| "logits/chosen": -1.511093258857727, |
| "logits/rejected": -1.4666423797607422, |
| "logps/chosen": -232.2217559814453, |
| "logps/rejected": -291.5470275878906, |
| "loss": 0.1595, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.36790722608566284, |
| "rewards/margins": 4.524458885192871, |
| "rewards/rejected": -4.892366409301758, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.432, |
| "grad_norm": 5.322598457336426, |
| "learning_rate": 2.8102468815331487e-06, |
| "logits/chosen": -1.5681586265563965, |
| "logits/rejected": -1.5516602993011475, |
| "logps/chosen": -214.8710174560547, |
| "logps/rejected": -306.7196044921875, |
| "loss": 0.2156, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.6621019244194031, |
| "rewards/margins": 4.608248710632324, |
| "rewards/rejected": -5.270350933074951, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 3.846139907836914, |
| "learning_rate": 2.788801933646066e-06, |
| "logits/chosen": -1.535198450088501, |
| "logits/rejected": -1.4371068477630615, |
| "logps/chosen": -225.10507202148438, |
| "logps/rejected": -300.16339111328125, |
| "loss": 0.1716, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.22050288319587708, |
| "rewards/margins": 4.9329986572265625, |
| "rewards/rejected": -5.153501987457275, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.448, |
| "grad_norm": 4.434758186340332, |
| "learning_rate": 2.7673354257274775e-06, |
| "logits/chosen": -1.6025463342666626, |
| "logits/rejected": -1.5076074600219727, |
| "logps/chosen": -225.4650421142578, |
| "logps/rejected": -309.60272216796875, |
| "loss": 0.185, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.4188717305660248, |
| "rewards/margins": 4.646775722503662, |
| "rewards/rejected": -5.065647125244141, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.456, |
| "grad_norm": 2.5944976806640625, |
| "learning_rate": 2.7458489603240873e-06, |
| "logits/chosen": -1.6403672695159912, |
| "logits/rejected": -1.5778125524520874, |
| "logps/chosen": -216.8976593017578, |
| "logps/rejected": -304.27142333984375, |
| "loss": 0.1484, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.30255550146102905, |
| "rewards/margins": 5.206709861755371, |
| "rewards/rejected": -5.509265422821045, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.464, |
| "grad_norm": 5.241691589355469, |
| "learning_rate": 2.724344141472493e-06, |
| "logits/chosen": -1.429987907409668, |
| "logits/rejected": -1.3562849760055542, |
| "logps/chosen": -255.3536834716797, |
| "logps/rejected": -304.39044189453125, |
| "loss": 0.2206, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.3938491940498352, |
| "rewards/margins": 4.941486358642578, |
| "rewards/rejected": -5.335335731506348, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.472, |
| "grad_norm": 1.7611676454544067, |
| "learning_rate": 2.702822574579439e-06, |
| "logits/chosen": -1.5172083377838135, |
| "logits/rejected": -1.4978201389312744, |
| "logps/chosen": -227.9757843017578, |
| "logps/rejected": -301.53466796875, |
| "loss": 0.148, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.3590165376663208, |
| "rewards/margins": 5.5135297775268555, |
| "rewards/rejected": -5.872546195983887, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.48, |
| "grad_norm": 2.4033844470977783, |
| "learning_rate": 2.681285866301967e-06, |
| "logits/chosen": -1.4651075601577759, |
| "logits/rejected": -1.4009742736816406, |
| "logps/chosen": -262.92706298828125, |
| "logps/rejected": -274.20477294921875, |
| "loss": 0.1876, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.12432646751403809, |
| "rewards/margins": 4.453019618988037, |
| "rewards/rejected": -4.577345848083496, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.488, |
| "grad_norm": 2.467533826828003, |
| "learning_rate": 2.6597356244274723e-06, |
| "logits/chosen": -1.4565765857696533, |
| "logits/rejected": -1.4273284673690796, |
| "logps/chosen": -222.59634399414062, |
| "logps/rejected": -284.24267578125, |
| "loss": 0.2066, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.47027140855789185, |
| "rewards/margins": 4.3873701095581055, |
| "rewards/rejected": -4.85764217376709, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.496, |
| "grad_norm": 1.8973395824432373, |
| "learning_rate": 2.6381734577536816e-06, |
| "logits/chosen": -1.6343520879745483, |
| "logits/rejected": -1.5240951776504517, |
| "logps/chosen": -237.08358764648438, |
| "logps/rejected": -299.1141662597656, |
| "loss": 0.1203, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.15792664885520935, |
| "rewards/margins": 5.232786655426025, |
| "rewards/rejected": -5.390713214874268, |
| "step": 935 |
| }, |
| { |
| "epoch": 1.504, |
| "grad_norm": 2.850900650024414, |
| "learning_rate": 2.616600975968544e-06, |
| "logits/chosen": -1.5603736639022827, |
| "logits/rejected": -1.4937515258789062, |
| "logps/chosen": -229.7916717529297, |
| "logps/rejected": -306.2135925292969, |
| "loss": 0.1522, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.43655306100845337, |
| "rewards/margins": 5.207942962646484, |
| "rewards/rejected": -5.644496440887451, |
| "step": 940 |
| }, |
| { |
| "epoch": 1.512, |
| "grad_norm": 3.7755420207977295, |
| "learning_rate": 2.59501978953007e-06, |
| "logits/chosen": -1.605474829673767, |
| "logits/rejected": -1.5499731302261353, |
| "logps/chosen": -236.4785919189453, |
| "logps/rejected": -303.67742919921875, |
| "loss": 0.1512, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.48815402388572693, |
| "rewards/margins": 5.069121360778809, |
| "rewards/rejected": -5.557275772094727, |
| "step": 945 |
| }, |
| { |
| "epoch": 1.52, |
| "grad_norm": 7.098239421844482, |
| "learning_rate": 2.5734315095460983e-06, |
| "logits/chosen": -1.5143400430679321, |
| "logits/rejected": -1.5596923828125, |
| "logps/chosen": -232.8845672607422, |
| "logps/rejected": -296.623779296875, |
| "loss": 0.2305, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.8665415048599243, |
| "rewards/margins": 4.796109676361084, |
| "rewards/rejected": -5.662651062011719, |
| "step": 950 |
| }, |
| { |
| "epoch": 1.528, |
| "grad_norm": 3.9633948802948, |
| "learning_rate": 2.551837747654027e-06, |
| "logits/chosen": -1.5993857383728027, |
| "logits/rejected": -1.555641531944275, |
| "logps/chosen": -242.03060913085938, |
| "logps/rejected": -328.293701171875, |
| "loss": 0.1351, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8731464147567749, |
| "rewards/margins": 5.36746072769165, |
| "rewards/rejected": -6.240607261657715, |
| "step": 955 |
| }, |
| { |
| "epoch": 1.536, |
| "grad_norm": 4.446506977081299, |
| "learning_rate": 2.530240115900494e-06, |
| "logits/chosen": -1.533271312713623, |
| "logits/rejected": -1.5505766868591309, |
| "logps/chosen": -222.93154907226562, |
| "logps/rejected": -297.92071533203125, |
| "loss": 0.1835, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.8138089179992676, |
| "rewards/margins": 4.758518218994141, |
| "rewards/rejected": -5.572327136993408, |
| "step": 960 |
| }, |
| { |
| "epoch": 1.544, |
| "grad_norm": 4.153780937194824, |
| "learning_rate": 2.5086402266210403e-06, |
| "logits/chosen": -1.5660960674285889, |
| "logits/rejected": -1.5074437856674194, |
| "logps/chosen": -239.93869018554688, |
| "logps/rejected": -301.6562805175781, |
| "loss": 0.1765, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.2847996652126312, |
| "rewards/margins": 4.576987266540527, |
| "rewards/rejected": -4.861786842346191, |
| "step": 965 |
| }, |
| { |
| "epoch": 1.552, |
| "grad_norm": 2.6983306407928467, |
| "learning_rate": 2.4870396923197347e-06, |
| "logits/chosen": -1.6386797428131104, |
| "logits/rejected": -1.539115071296692, |
| "logps/chosen": -241.8653564453125, |
| "logps/rejected": -293.84979248046875, |
| "loss": 0.2475, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.6399244070053101, |
| "rewards/margins": 3.9493603706359863, |
| "rewards/rejected": -4.589285373687744, |
| "step": 970 |
| }, |
| { |
| "epoch": 1.56, |
| "grad_norm": 2.8271076679229736, |
| "learning_rate": 2.4654401255487993e-06, |
| "logits/chosen": -1.5707069635391235, |
| "logits/rejected": -1.557492971420288, |
| "logps/chosen": -221.89389038085938, |
| "logps/rejected": -299.2413024902344, |
| "loss": 0.1839, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5429846048355103, |
| "rewards/margins": 4.966402530670166, |
| "rewards/rejected": -5.509387016296387, |
| "step": 975 |
| }, |
| { |
| "epoch": 1.568, |
| "grad_norm": 5.555903911590576, |
| "learning_rate": 2.443843138788228e-06, |
| "logits/chosen": -1.6090481281280518, |
| "logits/rejected": -1.4825258255004883, |
| "logps/chosen": -220.1716766357422, |
| "logps/rejected": -300.88458251953125, |
| "loss": 0.157, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.31384724378585815, |
| "rewards/margins": 5.2531962394714355, |
| "rewards/rejected": -5.567043781280518, |
| "step": 980 |
| }, |
| { |
| "epoch": 1.576, |
| "grad_norm": 3.2215209007263184, |
| "learning_rate": 2.4222503443254084e-06, |
| "logits/chosen": -1.5508859157562256, |
| "logits/rejected": -1.5698188543319702, |
| "logps/chosen": -239.60537719726562, |
| "logps/rejected": -307.3664245605469, |
| "loss": 0.2068, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.189010351896286, |
| "rewards/margins": 4.920119285583496, |
| "rewards/rejected": -5.109129905700684, |
| "step": 985 |
| }, |
| { |
| "epoch": 1.584, |
| "grad_norm": 2.4060542583465576, |
| "learning_rate": 2.400663354134758e-06, |
| "logits/chosen": -1.5376317501068115, |
| "logits/rejected": -1.4998356103897095, |
| "logps/chosen": -243.26486206054688, |
| "logps/rejected": -289.48065185546875, |
| "loss": 0.1764, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5606673359870911, |
| "rewards/margins": 4.382869720458984, |
| "rewards/rejected": -4.943536758422852, |
| "step": 990 |
| }, |
| { |
| "epoch": 1.592, |
| "grad_norm": 4.755422592163086, |
| "learning_rate": 2.3790837797573867e-06, |
| "logits/chosen": -1.4841296672821045, |
| "logits/rejected": -1.4592382907867432, |
| "logps/chosen": -235.3893585205078, |
| "logps/rejected": -284.45806884765625, |
| "loss": 0.1904, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.288729190826416, |
| "rewards/margins": 4.829131126403809, |
| "rewards/rejected": -5.117860317230225, |
| "step": 995 |
| }, |
| { |
| "epoch": 1.6, |
| "grad_norm": 3.5496389865875244, |
| "learning_rate": 2.357513232180788e-06, |
| "logits/chosen": -1.6013635396957397, |
| "logits/rejected": -1.4790610074996948, |
| "logps/chosen": -240.17324829101562, |
| "logps/rejected": -303.7333068847656, |
| "loss": 0.1618, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.15540306270122528, |
| "rewards/margins": 5.00370454788208, |
| "rewards/rejected": -5.159107208251953, |
| "step": 1000 |
| }, |
| { |
| "epoch": 1.608, |
| "grad_norm": 4.431177616119385, |
| "learning_rate": 2.335953321718578e-06, |
| "logits/chosen": -1.6200017929077148, |
| "logits/rejected": -1.538055658340454, |
| "logps/chosen": -253.2138671875, |
| "logps/rejected": -305.25054931640625, |
| "loss": 0.1785, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.22772517800331116, |
| "rewards/margins": 4.754574775695801, |
| "rewards/rejected": -4.982300281524658, |
| "step": 1005 |
| }, |
| { |
| "epoch": 1.616, |
| "grad_norm": 4.100212574005127, |
| "learning_rate": 2.3144056578902713e-06, |
| "logits/chosen": -1.4878506660461426, |
| "logits/rejected": -1.4349610805511475, |
| "logps/chosen": -240.9538116455078, |
| "logps/rejected": -279.4476013183594, |
| "loss": 0.1915, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.35914379358291626, |
| "rewards/margins": 4.77241325378418, |
| "rewards/rejected": -5.131556987762451, |
| "step": 1010 |
| }, |
| { |
| "epoch": 1.624, |
| "grad_norm": 4.914249420166016, |
| "learning_rate": 2.292871849301135e-06, |
| "logits/chosen": -1.5851023197174072, |
| "logits/rejected": -1.4445489645004272, |
| "logps/chosen": -245.83212280273438, |
| "logps/rejected": -300.76226806640625, |
| "loss": 0.2035, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.06682246923446655, |
| "rewards/margins": 4.696481227874756, |
| "rewards/rejected": -4.763303756713867, |
| "step": 1015 |
| }, |
| { |
| "epoch": 1.6320000000000001, |
| "grad_norm": 5.474418640136719, |
| "learning_rate": 2.271353503522092e-06, |
| "logits/chosen": -1.4620692729949951, |
| "logits/rejected": -1.474820852279663, |
| "logps/chosen": -234.1145477294922, |
| "logps/rejected": -303.3814392089844, |
| "loss": 0.231, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.4707890450954437, |
| "rewards/margins": 4.186692237854004, |
| "rewards/rejected": -4.6574811935424805, |
| "step": 1020 |
| }, |
| { |
| "epoch": 1.6400000000000001, |
| "grad_norm": 2.9608800411224365, |
| "learning_rate": 2.2498522269697193e-06, |
| "logits/chosen": -1.4659358263015747, |
| "logits/rejected": -1.5075595378875732, |
| "logps/chosen": -234.145751953125, |
| "logps/rejected": -303.431396484375, |
| "loss": 0.1632, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.674322247505188, |
| "rewards/margins": 5.1520586013793945, |
| "rewards/rejected": -5.826380729675293, |
| "step": 1025 |
| }, |
| { |
| "epoch": 1.6480000000000001, |
| "grad_norm": 3.0162484645843506, |
| "learning_rate": 2.228369624786314e-06, |
| "logits/chosen": -1.5545270442962646, |
| "logits/rejected": -1.4817333221435547, |
| "logps/chosen": -243.4051971435547, |
| "logps/rejected": -318.2030334472656, |
| "loss": 0.1455, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.2738165259361267, |
| "rewards/margins": 5.5686564445495605, |
| "rewards/rejected": -5.842472553253174, |
| "step": 1030 |
| }, |
| { |
| "epoch": 1.6560000000000001, |
| "grad_norm": 1.2202463150024414, |
| "learning_rate": 2.2069073007200735e-06, |
| "logits/chosen": -1.5296119451522827, |
| "logits/rejected": -1.4982463121414185, |
| "logps/chosen": -248.0770721435547, |
| "logps/rejected": -293.5713806152344, |
| "loss": 0.1328, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.10386035591363907, |
| "rewards/margins": 5.480264186859131, |
| "rewards/rejected": -5.584124565124512, |
| "step": 1035 |
| }, |
| { |
| "epoch": 1.6640000000000001, |
| "grad_norm": 4.070965766906738, |
| "learning_rate": 2.1854668570053617e-06, |
| "logits/chosen": -1.4422088861465454, |
| "logits/rejected": -1.336922287940979, |
| "logps/chosen": -244.10546875, |
| "logps/rejected": -291.4113464355469, |
| "loss": 0.162, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.21176743507385254, |
| "rewards/margins": 5.223017692565918, |
| "rewards/rejected": -5.434784889221191, |
| "step": 1040 |
| }, |
| { |
| "epoch": 1.6720000000000002, |
| "grad_norm": 2.6828646659851074, |
| "learning_rate": 2.164049894243106e-06, |
| "logits/chosen": -1.521547555923462, |
| "logits/rejected": -1.5329920053482056, |
| "logps/chosen": -220.31912231445312, |
| "logps/rejected": -300.8257141113281, |
| "loss": 0.1597, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8682439923286438, |
| "rewards/margins": 4.75205135345459, |
| "rewards/rejected": -5.620295524597168, |
| "step": 1045 |
| }, |
| { |
| "epoch": 1.6800000000000002, |
| "grad_norm": 5.221405506134033, |
| "learning_rate": 2.1426580112812963e-06, |
| "logits/chosen": -1.4651761054992676, |
| "logits/rejected": -1.4989073276519775, |
| "logps/chosen": -225.73898315429688, |
| "logps/rejected": -306.9569091796875, |
| "loss": 0.1811, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.521946370601654, |
| "rewards/margins": 4.900145530700684, |
| "rewards/rejected": -5.422091960906982, |
| "step": 1050 |
| }, |
| { |
| "epoch": 1.688, |
| "grad_norm": 4.7056684494018555, |
| "learning_rate": 2.1212928050956363e-06, |
| "logits/chosen": -1.4704397916793823, |
| "logits/rejected": -1.3621515035629272, |
| "logps/chosen": -261.0582580566406, |
| "logps/rejected": -309.2656555175781, |
| "loss": 0.1629, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.46547871828079224, |
| "rewards/margins": 5.209592342376709, |
| "rewards/rejected": -5.6750712394714355, |
| "step": 1055 |
| }, |
| { |
| "epoch": 1.696, |
| "grad_norm": 6.003643989562988, |
| "learning_rate": 2.0999558706703156e-06, |
| "logits/chosen": -1.5227447748184204, |
| "logits/rejected": -1.4205873012542725, |
| "logps/chosen": -242.269287109375, |
| "logps/rejected": -301.63250732421875, |
| "loss": 0.1872, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.2986319959163666, |
| "rewards/margins": 4.871571063995361, |
| "rewards/rejected": -5.17020320892334, |
| "step": 1060 |
| }, |
| { |
| "epoch": 1.704, |
| "grad_norm": 3.101877212524414, |
| "learning_rate": 2.0786488008789452e-06, |
| "logits/chosen": -1.5970118045806885, |
| "logits/rejected": -1.5494978427886963, |
| "logps/chosen": -249.52725219726562, |
| "logps/rejected": -304.41656494140625, |
| "loss": 0.1266, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.39330193400382996, |
| "rewards/margins": 5.001396179199219, |
| "rewards/rejected": -5.394698143005371, |
| "step": 1065 |
| }, |
| { |
| "epoch": 1.712, |
| "grad_norm": 1.7850382328033447, |
| "learning_rate": 2.0573731863656378e-06, |
| "logits/chosen": -1.5234953165054321, |
| "logits/rejected": -1.4536514282226562, |
| "logps/chosen": -234.9643096923828, |
| "logps/rejected": -293.68804931640625, |
| "loss": 0.1331, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.3854447305202484, |
| "rewards/margins": 5.071932315826416, |
| "rewards/rejected": -5.457376956939697, |
| "step": 1070 |
| }, |
| { |
| "epoch": 1.72, |
| "grad_norm": 4.341036796569824, |
| "learning_rate": 2.036130615426267e-06, |
| "logits/chosen": -1.541611671447754, |
| "logits/rejected": -1.4438472986221313, |
| "logps/chosen": -226.50082397460938, |
| "logps/rejected": -294.1707458496094, |
| "loss": 0.1548, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2821238040924072, |
| "rewards/margins": 4.956992149353027, |
| "rewards/rejected": -5.239116191864014, |
| "step": 1075 |
| }, |
| { |
| "epoch": 1.728, |
| "grad_norm": 1.9874736070632935, |
| "learning_rate": 2.0149226738898923e-06, |
| "logits/chosen": -1.6126731634140015, |
| "logits/rejected": -1.5012925863265991, |
| "logps/chosen": -241.3867950439453, |
| "logps/rejected": -312.0802917480469, |
| "loss": 0.1447, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.49232107400894165, |
| "rewards/margins": 5.413763523101807, |
| "rewards/rejected": -5.906083583831787, |
| "step": 1080 |
| }, |
| { |
| "epoch": 1.736, |
| "grad_norm": 4.81574821472168, |
| "learning_rate": 1.9937509450003715e-06, |
| "logits/chosen": -1.4321324825286865, |
| "logits/rejected": -1.4526867866516113, |
| "logps/chosen": -215.76388549804688, |
| "logps/rejected": -297.8123474121094, |
| "loss": 0.1816, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.575035572052002, |
| "rewards/margins": 5.639148712158203, |
| "rewards/rejected": -6.214184761047363, |
| "step": 1085 |
| }, |
| { |
| "epoch": 1.744, |
| "grad_norm": 3.8950352668762207, |
| "learning_rate": 1.9726170092981676e-06, |
| "logits/chosen": -1.5303950309753418, |
| "logits/rejected": -1.5153299570083618, |
| "logps/chosen": -237.6295928955078, |
| "logps/rejected": -315.077880859375, |
| "loss": 0.1566, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.65621018409729, |
| "rewards/margins": 5.208491802215576, |
| "rewards/rejected": -5.864701747894287, |
| "step": 1090 |
| }, |
| { |
| "epoch": 1.752, |
| "grad_norm": 3.6696534156799316, |
| "learning_rate": 1.9515224445023553e-06, |
| "logits/chosen": -1.5249470472335815, |
| "logits/rejected": -1.4677118062973022, |
| "logps/chosen": -226.087646484375, |
| "logps/rejected": -301.89190673828125, |
| "loss": 0.1377, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.3383384346961975, |
| "rewards/margins": 5.166504859924316, |
| "rewards/rejected": -5.504843711853027, |
| "step": 1095 |
| }, |
| { |
| "epoch": 1.76, |
| "grad_norm": 3.5852746963500977, |
| "learning_rate": 1.9304688253928398e-06, |
| "logits/chosen": -1.533934235572815, |
| "logits/rejected": -1.5655229091644287, |
| "logps/chosen": -235.65298461914062, |
| "logps/rejected": -301.34503173828125, |
| "loss": 0.1833, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.8953931927680969, |
| "rewards/margins": 5.259232044219971, |
| "rewards/rejected": -6.154624938964844, |
| "step": 1100 |
| }, |
| { |
| "epoch": 1.768, |
| "grad_norm": 5.908403396606445, |
| "learning_rate": 1.909457723692794e-06, |
| "logits/chosen": -1.5542901754379272, |
| "logits/rejected": -1.5842716693878174, |
| "logps/chosen": -244.9988250732422, |
| "logps/rejected": -300.6830139160156, |
| "loss": 0.2367, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.088773488998413, |
| "rewards/margins": 5.178781032562256, |
| "rewards/rejected": -6.267554759979248, |
| "step": 1105 |
| }, |
| { |
| "epoch": 1.776, |
| "grad_norm": 4.3570661544799805, |
| "learning_rate": 1.8884907079513227e-06, |
| "logits/chosen": -1.5214910507202148, |
| "logits/rejected": -1.4828593730926514, |
| "logps/chosen": -231.094970703125, |
| "logps/rejected": -313.3832702636719, |
| "loss": 0.1663, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.30087193846702576, |
| "rewards/margins": 5.7574782371521, |
| "rewards/rejected": -6.058350086212158, |
| "step": 1110 |
| }, |
| { |
| "epoch": 1.784, |
| "grad_norm": 3.1484177112579346, |
| "learning_rate": 1.8675693434263652e-06, |
| "logits/chosen": -1.5721497535705566, |
| "logits/rejected": -1.4944632053375244, |
| "logps/chosen": -221.1279296875, |
| "logps/rejected": -300.9421081542969, |
| "loss": 0.1515, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.32752472162246704, |
| "rewards/margins": 5.808013439178467, |
| "rewards/rejected": -6.135538101196289, |
| "step": 1115 |
| }, |
| { |
| "epoch": 1.792, |
| "grad_norm": 3.595839500427246, |
| "learning_rate": 1.8466951919678491e-06, |
| "logits/chosen": -1.59939444065094, |
| "logits/rejected": -1.5375818014144897, |
| "logps/chosen": -230.9847412109375, |
| "logps/rejected": -309.09027099609375, |
| "loss": 0.1364, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.29122036695480347, |
| "rewards/margins": 5.625614643096924, |
| "rewards/rejected": -5.916835308074951, |
| "step": 1120 |
| }, |
| { |
| "epoch": 1.8, |
| "grad_norm": 3.8204920291900635, |
| "learning_rate": 1.8258698119010832e-06, |
| "logits/chosen": -1.5291309356689453, |
| "logits/rejected": -1.4495049715042114, |
| "logps/chosen": -232.0759735107422, |
| "logps/rejected": -281.865478515625, |
| "loss": 0.1622, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2713313102722168, |
| "rewards/margins": 4.578837871551514, |
| "rewards/rejected": -4.850168704986572, |
| "step": 1125 |
| }, |
| { |
| "epoch": 1.808, |
| "grad_norm": 2.033203363418579, |
| "learning_rate": 1.8050947579104325e-06, |
| "logits/chosen": -1.5407826900482178, |
| "logits/rejected": -1.5142061710357666, |
| "logps/chosen": -227.7032470703125, |
| "logps/rejected": -311.5636291503906, |
| "loss": 0.1278, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.25752535462379456, |
| "rewards/margins": 5.367326736450195, |
| "rewards/rejected": -5.624852180480957, |
| "step": 1130 |
| }, |
| { |
| "epoch": 1.8159999999999998, |
| "grad_norm": 4.951417922973633, |
| "learning_rate": 1.7843715809232486e-06, |
| "logits/chosen": -1.4634641408920288, |
| "logits/rejected": -1.4312846660614014, |
| "logps/chosen": -229.4953155517578, |
| "logps/rejected": -293.6320495605469, |
| "loss": 0.2149, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.5195713639259338, |
| "rewards/margins": 4.903964519500732, |
| "rewards/rejected": -5.4235358238220215, |
| "step": 1135 |
| }, |
| { |
| "epoch": 1.8239999999999998, |
| "grad_norm": 2.566675901412964, |
| "learning_rate": 1.763701827994095e-06, |
| "logits/chosen": -1.4973647594451904, |
| "logits/rejected": -1.424039602279663, |
| "logps/chosen": -219.69979858398438, |
| "logps/rejected": -298.1719665527344, |
| "loss": 0.1902, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.3790625035762787, |
| "rewards/margins": 5.123688697814941, |
| "rewards/rejected": -5.50275182723999, |
| "step": 1140 |
| }, |
| { |
| "epoch": 1.8319999999999999, |
| "grad_norm": 5.082289218902588, |
| "learning_rate": 1.7430870421892459e-06, |
| "logits/chosen": -1.5290133953094482, |
| "logits/rejected": -1.4534924030303955, |
| "logps/chosen": -231.0949249267578, |
| "logps/rejected": -288.7142639160156, |
| "loss": 0.1439, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.3327312469482422, |
| "rewards/margins": 5.081864356994629, |
| "rewards/rejected": -5.414595603942871, |
| "step": 1145 |
| }, |
| { |
| "epoch": 1.8399999999999999, |
| "grad_norm": 2.2908127307891846, |
| "learning_rate": 1.7225287624714987e-06, |
| "logits/chosen": -1.557173728942871, |
| "logits/rejected": -1.451888918876648, |
| "logps/chosen": -237.3270263671875, |
| "logps/rejected": -302.59124755859375, |
| "loss": 0.1289, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.10339298099279404, |
| "rewards/margins": 5.4684553146362305, |
| "rewards/rejected": -5.571847915649414, |
| "step": 1150 |
| }, |
| { |
| "epoch": 1.8479999999999999, |
| "grad_norm": 3.687828540802002, |
| "learning_rate": 1.7020285235852824e-06, |
| "logits/chosen": -1.483363151550293, |
| "logits/rejected": -1.4083940982818604, |
| "logps/chosen": -229.43991088867188, |
| "logps/rejected": -289.58795166015625, |
| "loss": 0.1625, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.04203609377145767, |
| "rewards/margins": 5.190605640411377, |
| "rewards/rejected": -5.23264217376709, |
| "step": 1155 |
| }, |
| { |
| "epoch": 1.8559999999999999, |
| "grad_norm": 3.409076690673828, |
| "learning_rate": 1.6815878559420806e-06, |
| "logits/chosen": -1.637976050376892, |
| "logits/rejected": -1.5942111015319824, |
| "logps/chosen": -233.52560424804688, |
| "logps/rejected": -307.1108093261719, |
| "loss": 0.1353, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.40837258100509644, |
| "rewards/margins": 4.940521240234375, |
| "rewards/rejected": -5.348894119262695, |
| "step": 1160 |
| }, |
| { |
| "epoch": 1.8639999999999999, |
| "grad_norm": 3.731051206588745, |
| "learning_rate": 1.661208285506188e-06, |
| "logits/chosen": -1.5421288013458252, |
| "logits/rejected": -1.4871435165405273, |
| "logps/chosen": -214.5398712158203, |
| "logps/rejected": -283.8194885253906, |
| "loss": 0.2104, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.5290651321411133, |
| "rewards/margins": 4.5201005935668945, |
| "rewards/rejected": -5.04916524887085, |
| "step": 1165 |
| }, |
| { |
| "epoch": 1.8719999999999999, |
| "grad_norm": 4.392094612121582, |
| "learning_rate": 1.6408913336807835e-06, |
| "logits/chosen": -1.7092958688735962, |
| "logits/rejected": -1.677384376525879, |
| "logps/chosen": -239.9275665283203, |
| "logps/rejected": -326.7081604003906, |
| "loss": 0.201, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.9028335809707642, |
| "rewards/margins": 5.153952598571777, |
| "rewards/rejected": -6.05678653717041, |
| "step": 1170 |
| }, |
| { |
| "epoch": 1.88, |
| "grad_norm": 3.3208742141723633, |
| "learning_rate": 1.6206385171943628e-06, |
| "logits/chosen": -1.5975421667099, |
| "logits/rejected": -1.5135239362716675, |
| "logps/chosen": -221.99978637695312, |
| "logps/rejected": -312.03778076171875, |
| "loss": 0.1347, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.614351212978363, |
| "rewards/margins": 5.365405559539795, |
| "rewards/rejected": -5.979756832122803, |
| "step": 1175 |
| }, |
| { |
| "epoch": 1.888, |
| "grad_norm": 4.675105571746826, |
| "learning_rate": 1.6004513479874975e-06, |
| "logits/chosen": -1.5107734203338623, |
| "logits/rejected": -1.4795351028442383, |
| "logps/chosen": -221.3353729248047, |
| "logps/rejected": -302.6809997558594, |
| "loss": 0.1506, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.6793597936630249, |
| "rewards/margins": 5.78696870803833, |
| "rewards/rejected": -6.466329097747803, |
| "step": 1180 |
| }, |
| { |
| "epoch": 1.896, |
| "grad_norm": 3.0278146266937256, |
| "learning_rate": 1.580331333099976e-06, |
| "logits/chosen": -1.589797854423523, |
| "logits/rejected": -1.5239031314849854, |
| "logps/chosen": -228.4359893798828, |
| "logps/rejected": -315.303955078125, |
| "loss": 0.1711, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.9097194671630859, |
| "rewards/margins": 5.420597076416016, |
| "rewards/rejected": -6.330316543579102, |
| "step": 1185 |
| }, |
| { |
| "epoch": 1.904, |
| "grad_norm": 5.7446513175964355, |
| "learning_rate": 1.5602799745582859e-06, |
| "logits/chosen": -1.5522780418395996, |
| "logits/rejected": -1.491706132888794, |
| "logps/chosen": -253.501708984375, |
| "logps/rejected": -303.25872802734375, |
| "loss": 0.1788, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.768315315246582, |
| "rewards/margins": 5.147302150726318, |
| "rewards/rejected": -5.915616989135742, |
| "step": 1190 |
| }, |
| { |
| "epoch": 1.912, |
| "grad_norm": 2.9774773120880127, |
| "learning_rate": 1.5402987692634941e-06, |
| "logits/chosen": -1.564814567565918, |
| "logits/rejected": -1.5448888540267944, |
| "logps/chosen": -217.634033203125, |
| "logps/rejected": -321.3653869628906, |
| "loss": 0.1247, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9152569770812988, |
| "rewards/margins": 5.907143592834473, |
| "rewards/rejected": -6.822400093078613, |
| "step": 1195 |
| }, |
| { |
| "epoch": 1.92, |
| "grad_norm": 3.8479833602905273, |
| "learning_rate": 1.5203892088794888e-06, |
| "logits/chosen": -1.5153319835662842, |
| "logits/rejected": -1.5472588539123535, |
| "logps/chosen": -241.7740020751953, |
| "logps/rejected": -319.2223205566406, |
| "loss": 0.1756, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.827950119972229, |
| "rewards/margins": 5.2503790855407715, |
| "rewards/rejected": -6.078328609466553, |
| "step": 1200 |
| }, |
| { |
| "epoch": 1.928, |
| "grad_norm": 7.263859748840332, |
| "learning_rate": 1.5005527797216284e-06, |
| "logits/chosen": -1.5309240818023682, |
| "logits/rejected": -1.5484015941619873, |
| "logps/chosen": -245.1676483154297, |
| "logps/rejected": -309.8975524902344, |
| "loss": 0.206, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.322698175907135, |
| "rewards/margins": 5.289670944213867, |
| "rewards/rejected": -5.612368583679199, |
| "step": 1205 |
| }, |
| { |
| "epoch": 1.936, |
| "grad_norm": 4.652385711669922, |
| "learning_rate": 1.4807909626457785e-06, |
| "logits/chosen": -1.603051781654358, |
| "logits/rejected": -1.4821627140045166, |
| "logps/chosen": -224.2364044189453, |
| "logps/rejected": -321.984375, |
| "loss": 0.172, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.7000328302383423, |
| "rewards/margins": 5.369967937469482, |
| "rewards/rejected": -6.070000648498535, |
| "step": 1210 |
| }, |
| { |
| "epoch": 1.944, |
| "grad_norm": 2.9950015544891357, |
| "learning_rate": 1.4611052329377629e-06, |
| "logits/chosen": -1.6220756769180298, |
| "logits/rejected": -1.5894696712493896, |
| "logps/chosen": -225.9698486328125, |
| "logps/rejected": -324.79254150390625, |
| "loss": 0.1683, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7203431129455566, |
| "rewards/margins": 5.393028736114502, |
| "rewards/rejected": -6.1133713722229, |
| "step": 1215 |
| }, |
| { |
| "epoch": 1.952, |
| "grad_norm": 2.781445026397705, |
| "learning_rate": 1.4414970602032301e-06, |
| "logits/chosen": -1.4824979305267334, |
| "logits/rejected": -1.455690622329712, |
| "logps/chosen": -222.3482666015625, |
| "logps/rejected": -307.96856689453125, |
| "loss": 0.1571, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.7416940927505493, |
| "rewards/margins": 5.335463523864746, |
| "rewards/rejected": -6.077157974243164, |
| "step": 1220 |
| }, |
| { |
| "epoch": 1.96, |
| "grad_norm": 4.076323986053467, |
| "learning_rate": 1.4219679082579384e-06, |
| "logits/chosen": -1.558056354522705, |
| "logits/rejected": -1.5473124980926514, |
| "logps/chosen": -253.4956512451172, |
| "logps/rejected": -309.33221435546875, |
| "loss": 0.187, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.9098089337348938, |
| "rewards/margins": 5.028618812561035, |
| "rewards/rejected": -5.938427925109863, |
| "step": 1225 |
| }, |
| { |
| "epoch": 1.968, |
| "grad_norm": 4.632643699645996, |
| "learning_rate": 1.4025192350184847e-06, |
| "logits/chosen": -1.5654726028442383, |
| "logits/rejected": -1.52810800075531, |
| "logps/chosen": -239.4210968017578, |
| "logps/rejected": -316.1085510253906, |
| "loss": 0.1592, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7853416204452515, |
| "rewards/margins": 5.446610450744629, |
| "rewards/rejected": -6.231952667236328, |
| "step": 1230 |
| }, |
| { |
| "epoch": 1.976, |
| "grad_norm": 2.3505971431732178, |
| "learning_rate": 1.3831524923934525e-06, |
| "logits/chosen": -1.4985238313674927, |
| "logits/rejected": -1.4004181623458862, |
| "logps/chosen": -241.896240234375, |
| "logps/rejected": -305.42242431640625, |
| "loss": 0.0944, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.4821149706840515, |
| "rewards/margins": 5.794647216796875, |
| "rewards/rejected": -6.27676248550415, |
| "step": 1235 |
| }, |
| { |
| "epoch": 1.984, |
| "grad_norm": 4.241216659545898, |
| "learning_rate": 1.3638691261750402e-06, |
| "logits/chosen": -1.5663890838623047, |
| "logits/rejected": -1.4924695491790771, |
| "logps/chosen": -228.1782989501953, |
| "logps/rejected": -331.6749572753906, |
| "loss": 0.1288, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5289267897605896, |
| "rewards/margins": 5.913106441497803, |
| "rewards/rejected": -6.442032814025879, |
| "step": 1240 |
| }, |
| { |
| "epoch": 1.992, |
| "grad_norm": 4.635298728942871, |
| "learning_rate": 1.3446705759311087e-06, |
| "logits/chosen": -1.6584011316299438, |
| "logits/rejected": -1.5797560214996338, |
| "logps/chosen": -245.9219207763672, |
| "logps/rejected": -326.98760986328125, |
| "loss": 0.1045, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.826259970664978, |
| "rewards/margins": 5.950412273406982, |
| "rewards/rejected": -6.77667236328125, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 2.418168783187866, |
| "learning_rate": 1.325558274897732e-06, |
| "logits/chosen": -1.4989831447601318, |
| "logits/rejected": -1.4638856649398804, |
| "logps/chosen": -236.1473846435547, |
| "logps/rejected": -303.3345031738281, |
| "loss": 0.1753, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.45226964354515076, |
| "rewards/margins": 5.467932224273682, |
| "rewards/rejected": -5.920201778411865, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.008, |
| "grad_norm": 2.998203754425049, |
| "learning_rate": 1.306533649872182e-06, |
| "logits/chosen": -1.517118215560913, |
| "logits/rejected": -1.4960505962371826, |
| "logps/chosen": -224.80142211914062, |
| "logps/rejected": -304.97857666015625, |
| "loss": 0.1276, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.8061330914497375, |
| "rewards/margins": 5.22979736328125, |
| "rewards/rejected": -6.0359296798706055, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.016, |
| "grad_norm": 3.6085073947906494, |
| "learning_rate": 1.287598121106432e-06, |
| "logits/chosen": -1.4973828792572021, |
| "logits/rejected": -1.430394172668457, |
| "logps/chosen": -224.1341094970703, |
| "logps/rejected": -306.37847900390625, |
| "loss": 0.1382, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.8516316413879395, |
| "rewards/margins": 5.601286888122559, |
| "rewards/rejected": -6.45291805267334, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.024, |
| "grad_norm": 5.166800498962402, |
| "learning_rate": 1.2687531022011146e-06, |
| "logits/chosen": -1.5529485940933228, |
| "logits/rejected": -1.5082321166992188, |
| "logps/chosen": -231.6868438720703, |
| "logps/rejected": -289.4835205078125, |
| "loss": 0.2142, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.7622639536857605, |
| "rewards/margins": 5.081564903259277, |
| "rewards/rejected": -5.843829154968262, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.032, |
| "grad_norm": 2.8838765621185303, |
| "learning_rate": 1.2500000000000007e-06, |
| "logits/chosen": -1.5373793840408325, |
| "logits/rejected": -1.491275668144226, |
| "logps/chosen": -218.83438110351562, |
| "logps/rejected": -298.385986328125, |
| "loss": 0.1204, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.4552576541900635, |
| "rewards/margins": 5.728938102722168, |
| "rewards/rejected": -6.184195518493652, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.04, |
| "grad_norm": 3.6019749641418457, |
| "learning_rate": 1.2313402144849714e-06, |
| "logits/chosen": -1.6223344802856445, |
| "logits/rejected": -1.564382791519165, |
| "logps/chosen": -247.25927734375, |
| "logps/rejected": -321.1236877441406, |
| "loss": 0.1288, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.5351258516311646, |
| "rewards/margins": 5.776213645935059, |
| "rewards/rejected": -6.311339378356934, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.048, |
| "grad_norm": 2.6691808700561523, |
| "learning_rate": 1.2127751386715054e-06, |
| "logits/chosen": -1.5619357824325562, |
| "logits/rejected": -1.4246995449066162, |
| "logps/chosen": -250.17330932617188, |
| "logps/rejected": -289.0823669433594, |
| "loss": 0.1265, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.5506324768066406, |
| "rewards/margins": 5.429201126098633, |
| "rewards/rejected": -5.979834079742432, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.056, |
| "grad_norm": 4.023916244506836, |
| "learning_rate": 1.194306158504683e-06, |
| "logits/chosen": -1.542194128036499, |
| "logits/rejected": -1.5818068981170654, |
| "logps/chosen": -224.98544311523438, |
| "logps/rejected": -313.40020751953125, |
| "loss": 0.1482, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6270204782485962, |
| "rewards/margins": 5.459203243255615, |
| "rewards/rejected": -6.08622407913208, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.064, |
| "grad_norm": 3.2508156299591064, |
| "learning_rate": 1.1759346527557247e-06, |
| "logits/chosen": -1.5900170803070068, |
| "logits/rejected": -1.5209767818450928, |
| "logps/chosen": -239.06289672851562, |
| "logps/rejected": -296.8851623535156, |
| "loss": 0.1824, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.7839146852493286, |
| "rewards/margins": 5.1801862716674805, |
| "rewards/rejected": -5.964101314544678, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.072, |
| "grad_norm": 3.451742172241211, |
| "learning_rate": 1.157661992919058e-06, |
| "logits/chosen": -1.497745156288147, |
| "logits/rejected": -1.4466352462768555, |
| "logps/chosen": -246.4713592529297, |
| "logps/rejected": -291.977294921875, |
| "loss": 0.1708, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.40274477005004883, |
| "rewards/margins": 5.257174491882324, |
| "rewards/rejected": -5.659919261932373, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.08, |
| "grad_norm": 3.3363468647003174, |
| "learning_rate": 1.1394895431099315e-06, |
| "logits/chosen": -1.462850570678711, |
| "logits/rejected": -1.4387768507003784, |
| "logps/chosen": -214.2821807861328, |
| "logps/rejected": -312.31256103515625, |
| "loss": 0.0987, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3282471299171448, |
| "rewards/margins": 6.343912601470947, |
| "rewards/rejected": -6.672159671783447, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.088, |
| "grad_norm": 2.328258514404297, |
| "learning_rate": 1.1214186599625835e-06, |
| "logits/chosen": -1.402923583984375, |
| "logits/rejected": -1.4672176837921143, |
| "logps/chosen": -240.6547393798828, |
| "logps/rejected": -320.3161926269531, |
| "loss": 0.1345, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8091422319412231, |
| "rewards/margins": 5.365943908691406, |
| "rewards/rejected": -6.17508602142334, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.096, |
| "grad_norm": 3.9099833965301514, |
| "learning_rate": 1.103450692528954e-06, |
| "logits/chosen": -1.4001861810684204, |
| "logits/rejected": -1.4040039777755737, |
| "logps/chosen": -253.8073272705078, |
| "logps/rejected": -298.1487121582031, |
| "loss": 0.1378, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8334349393844604, |
| "rewards/margins": 5.020366191864014, |
| "rewards/rejected": -5.8538007736206055, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.104, |
| "grad_norm": 4.8871541023254395, |
| "learning_rate": 1.0855869821779883e-06, |
| "logits/chosen": -1.5196176767349243, |
| "logits/rejected": -1.50358247756958, |
| "logps/chosen": -228.2346649169922, |
| "logps/rejected": -301.0938415527344, |
| "loss": 0.1661, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.9291476011276245, |
| "rewards/margins": 5.45242977142334, |
| "rewards/rejected": -6.381577968597412, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.112, |
| "grad_norm": 4.641075134277344, |
| "learning_rate": 1.0678288624954836e-06, |
| "logits/chosen": -1.5506222248077393, |
| "logits/rejected": -1.538140892982483, |
| "logps/chosen": -275.78460693359375, |
| "logps/rejected": -298.97076416015625, |
| "loss": 0.1659, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.011925202794373035, |
| "rewards/margins": 5.404875755310059, |
| "rewards/rejected": -5.392950057983398, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.12, |
| "grad_norm": 3.5139901638031006, |
| "learning_rate": 1.0501776591845472e-06, |
| "logits/chosen": -1.521786093711853, |
| "logits/rejected": -1.509730577468872, |
| "logps/chosen": -223.95529174804688, |
| "logps/rejected": -306.37933349609375, |
| "loss": 0.1303, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5522927641868591, |
| "rewards/margins": 5.402201175689697, |
| "rewards/rejected": -5.954493522644043, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.128, |
| "grad_norm": 3.935023069381714, |
| "learning_rate": 1.0326346899666143e-06, |
| "logits/chosen": -1.5349467992782593, |
| "logits/rejected": -1.4667823314666748, |
| "logps/chosen": -227.30422973632812, |
| "logps/rejected": -298.933349609375, |
| "loss": 0.1265, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.4221525192260742, |
| "rewards/margins": 5.432391166687012, |
| "rewards/rejected": -5.854543209075928, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.136, |
| "grad_norm": 3.250990152359009, |
| "learning_rate": 1.0152012644830897e-06, |
| "logits/chosen": -1.5740466117858887, |
| "logits/rejected": -1.6187541484832764, |
| "logps/chosen": -225.9763641357422, |
| "logps/rejected": -285.09185791015625, |
| "loss": 0.1386, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.5500262379646301, |
| "rewards/margins": 5.309136867523193, |
| "rewards/rejected": -5.859163284301758, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.144, |
| "grad_norm": 3.963165521621704, |
| "learning_rate": 9.978786841975647e-07, |
| "logits/chosen": -1.5362905263900757, |
| "logits/rejected": -1.494563341140747, |
| "logps/chosen": -238.83505249023438, |
| "logps/rejected": -307.7918701171875, |
| "loss": 0.1578, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7797427177429199, |
| "rewards/margins": 5.327605247497559, |
| "rewards/rejected": -6.107347011566162, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.152, |
| "grad_norm": 4.277246952056885, |
| "learning_rate": 9.806682422986716e-07, |
| "logits/chosen": -1.4467289447784424, |
| "logits/rejected": -1.4939007759094238, |
| "logps/chosen": -235.6219024658203, |
| "logps/rejected": -302.406982421875, |
| "loss": 0.1333, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5323159098625183, |
| "rewards/margins": 5.708221912384033, |
| "rewards/rejected": -6.240537166595459, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.16, |
| "grad_norm": 3.088682174682617, |
| "learning_rate": 9.635712236035305e-07, |
| "logits/chosen": -1.4703865051269531, |
| "logits/rejected": -1.4268938302993774, |
| "logps/chosen": -239.4414825439453, |
| "logps/rejected": -310.94677734375, |
| "loss": 0.1358, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5711382031440735, |
| "rewards/margins": 5.83718729019165, |
| "rewards/rejected": -6.408326148986816, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.168, |
| "grad_norm": 2.9433059692382812, |
| "learning_rate": 9.465889044618463e-07, |
| "logits/chosen": -1.531912922859192, |
| "logits/rejected": -1.5826823711395264, |
| "logps/chosen": -226.4390869140625, |
| "logps/rejected": -294.73016357421875, |
| "loss": 0.158, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0183833837509155, |
| "rewards/margins": 4.735301971435547, |
| "rewards/rejected": -5.753684997558594, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.176, |
| "grad_norm": 3.169440507888794, |
| "learning_rate": 9.297225526606105e-07, |
| "logits/chosen": -1.4751956462860107, |
| "logits/rejected": -1.4470738172531128, |
| "logps/chosen": -252.1983642578125, |
| "logps/rejected": -320.13812255859375, |
| "loss": 0.1171, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7644977569580078, |
| "rewards/margins": 5.812073707580566, |
| "rewards/rejected": -6.576571464538574, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.184, |
| "grad_norm": 4.91829252243042, |
| "learning_rate": 9.129734273294718e-07, |
| "logits/chosen": -1.552290439605713, |
| "logits/rejected": -1.4868769645690918, |
| "logps/chosen": -230.2556610107422, |
| "logps/rejected": -291.697998046875, |
| "loss": 0.1257, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.046489953994751, |
| "rewards/margins": 5.263110160827637, |
| "rewards/rejected": -6.309600830078125, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.192, |
| "grad_norm": 2.0276551246643066, |
| "learning_rate": 8.963427788467241e-07, |
| "logits/chosen": -1.545638084411621, |
| "logits/rejected": -1.466631531715393, |
| "logps/chosen": -244.7844696044922, |
| "logps/rejected": -315.06695556640625, |
| "loss": 0.1714, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.9884885549545288, |
| "rewards/margins": 5.733240604400635, |
| "rewards/rejected": -6.721728324890137, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.2, |
| "grad_norm": 5.268774509429932, |
| "learning_rate": 8.79831848745972e-07, |
| "logits/chosen": -1.597069501876831, |
| "logits/rejected": -1.5245897769927979, |
| "logps/chosen": -248.6392059326172, |
| "logps/rejected": -304.13348388671875, |
| "loss": 0.1752, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.6362749338150024, |
| "rewards/margins": 5.747987270355225, |
| "rewards/rejected": -6.384262561798096, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.208, |
| "grad_norm": 4.227468013763428, |
| "learning_rate": 8.634418696234403e-07, |
| "logits/chosen": -1.6100578308105469, |
| "logits/rejected": -1.5145816802978516, |
| "logps/chosen": -260.8665771484375, |
| "logps/rejected": -310.1653747558594, |
| "loss": 0.1859, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.0051674842834473, |
| "rewards/margins": 5.3024678230285645, |
| "rewards/rejected": -6.307635307312012, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.216, |
| "grad_norm": 3.3430335521698, |
| "learning_rate": 8.471740650459582e-07, |
| "logits/chosen": -1.4245984554290771, |
| "logits/rejected": -1.4522405862808228, |
| "logps/chosen": -240.0626983642578, |
| "logps/rejected": -317.7297058105469, |
| "loss": 0.1313, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.5361242294311523, |
| "rewards/margins": 5.816006660461426, |
| "rewards/rejected": -6.352130889892578, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.224, |
| "grad_norm": 3.8488659858703613, |
| "learning_rate": 8.310296494596159e-07, |
| "logits/chosen": -1.4325063228607178, |
| "logits/rejected": -1.4604406356811523, |
| "logps/chosen": -232.338134765625, |
| "logps/rejected": -300.801025390625, |
| "loss": 0.1457, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9337561726570129, |
| "rewards/margins": 5.740461349487305, |
| "rewards/rejected": -6.674217224121094, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.232, |
| "grad_norm": 3.3606622219085693, |
| "learning_rate": 8.150098280991029e-07, |
| "logits/chosen": -1.61603581905365, |
| "logits/rejected": -1.4698944091796875, |
| "logps/chosen": -201.53733825683594, |
| "logps/rejected": -290.5418395996094, |
| "loss": 0.1196, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5469184517860413, |
| "rewards/margins": 5.2674431800842285, |
| "rewards/rejected": -5.814361572265625, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.24, |
| "grad_norm": 4.717289447784424, |
| "learning_rate": 7.991157968977322e-07, |
| "logits/chosen": -1.5912104845046997, |
| "logits/rejected": -1.5626710653305054, |
| "logps/chosen": -228.1504669189453, |
| "logps/rejected": -326.4696350097656, |
| "loss": 0.1317, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.556774914264679, |
| "rewards/margins": 6.516512393951416, |
| "rewards/rejected": -7.0732879638671875, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.248, |
| "grad_norm": 2.843961477279663, |
| "learning_rate": 7.833487423981581e-07, |
| "logits/chosen": -1.484765887260437, |
| "logits/rejected": -1.4198788404464722, |
| "logps/chosen": -228.2584991455078, |
| "logps/rejected": -315.5864562988281, |
| "loss": 0.1104, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5225152969360352, |
| "rewards/margins": 6.307417869567871, |
| "rewards/rejected": -6.829933166503906, |
| "step": 1405 |
| }, |
| { |
| "epoch": 2.2560000000000002, |
| "grad_norm": 5.61122465133667, |
| "learning_rate": 7.677098416638037e-07, |
| "logits/chosen": -1.5627809762954712, |
| "logits/rejected": -1.561023235321045, |
| "logps/chosen": -237.8609619140625, |
| "logps/rejected": -313.85479736328125, |
| "loss": 0.1684, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.603573203086853, |
| "rewards/margins": 5.463527202606201, |
| "rewards/rejected": -6.067100524902344, |
| "step": 1410 |
| }, |
| { |
| "epoch": 2.2640000000000002, |
| "grad_norm": 3.832843542098999, |
| "learning_rate": 7.522002621909785e-07, |
| "logits/chosen": -1.5515294075012207, |
| "logits/rejected": -1.5249848365783691, |
| "logps/chosen": -210.19668579101562, |
| "logps/rejected": -317.81121826171875, |
| "loss": 0.1313, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8036311864852905, |
| "rewards/margins": 5.415702819824219, |
| "rewards/rejected": -6.219333648681641, |
| "step": 1415 |
| }, |
| { |
| "epoch": 2.2720000000000002, |
| "grad_norm": 3.504119396209717, |
| "learning_rate": 7.368211618217325e-07, |
| "logits/chosen": -1.482468843460083, |
| "logits/rejected": -1.4689217805862427, |
| "logps/chosen": -237.25418090820312, |
| "logps/rejected": -307.9796447753906, |
| "loss": 0.1392, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.6051761507987976, |
| "rewards/margins": 5.398853302001953, |
| "rewards/rejected": -6.004029273986816, |
| "step": 1420 |
| }, |
| { |
| "epoch": 2.2800000000000002, |
| "grad_norm": 4.0742316246032715, |
| "learning_rate": 7.21573688657408e-07, |
| "logits/chosen": -1.5382550954818726, |
| "logits/rejected": -1.5822103023529053, |
| "logps/chosen": -234.6659698486328, |
| "logps/rejected": -320.8883972167969, |
| "loss": 0.1264, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8635808825492859, |
| "rewards/margins": 5.727967262268066, |
| "rewards/rejected": -6.591547966003418, |
| "step": 1425 |
| }, |
| { |
| "epoch": 2.288, |
| "grad_norm": 3.2900421619415283, |
| "learning_rate": 7.064589809729416e-07, |
| "logits/chosen": -1.6166538000106812, |
| "logits/rejected": -1.5299408435821533, |
| "logps/chosen": -240.4681396484375, |
| "logps/rejected": -300.2424011230469, |
| "loss": 0.1137, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8580220937728882, |
| "rewards/margins": 5.814321041107178, |
| "rewards/rejected": -6.6723432540893555, |
| "step": 1430 |
| }, |
| { |
| "epoch": 2.296, |
| "grad_norm": 4.590370178222656, |
| "learning_rate": 6.914781671318757e-07, |
| "logits/chosen": -1.5603392124176025, |
| "logits/rejected": -1.5085687637329102, |
| "logps/chosen": -240.8662872314453, |
| "logps/rejected": -297.9051208496094, |
| "loss": 0.1545, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9626073837280273, |
| "rewards/margins": 5.510343551635742, |
| "rewards/rejected": -6.4729509353637695, |
| "step": 1435 |
| }, |
| { |
| "epoch": 2.304, |
| "grad_norm": 1.9299755096435547, |
| "learning_rate": 6.766323655021356e-07, |
| "logits/chosen": -1.4613937139511108, |
| "logits/rejected": -1.464815616607666, |
| "logps/chosen": -219.6249542236328, |
| "logps/rejected": -291.37982177734375, |
| "loss": 0.144, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7187808752059937, |
| "rewards/margins": 6.116919040679932, |
| "rewards/rejected": -6.835699558258057, |
| "step": 1440 |
| }, |
| { |
| "epoch": 2.312, |
| "grad_norm": 1.7471683025360107, |
| "learning_rate": 6.619226843725266e-07, |
| "logits/chosen": -1.518546462059021, |
| "logits/rejected": -1.4668691158294678, |
| "logps/chosen": -213.170166015625, |
| "logps/rejected": -317.2686462402344, |
| "loss": 0.1152, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.5413722395896912, |
| "rewards/margins": 6.646666526794434, |
| "rewards/rejected": -7.188039302825928, |
| "step": 1445 |
| }, |
| { |
| "epoch": 2.32, |
| "grad_norm": 4.943786144256592, |
| "learning_rate": 6.473502218700081e-07, |
| "logits/chosen": -1.5848660469055176, |
| "logits/rejected": -1.5765035152435303, |
| "logps/chosen": -231.70864868164062, |
| "logps/rejected": -317.9561767578125, |
| "loss": 0.1248, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.9403325319290161, |
| "rewards/margins": 5.609653949737549, |
| "rewards/rejected": -6.549986362457275, |
| "step": 1450 |
| }, |
| { |
| "epoch": 2.328, |
| "grad_norm": 6.270254611968994, |
| "learning_rate": 6.329160658777062e-07, |
| "logits/chosen": -1.6108720302581787, |
| "logits/rejected": -1.5802674293518066, |
| "logps/chosen": -233.30233764648438, |
| "logps/rejected": -310.03814697265625, |
| "loss": 0.1612, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.1865160465240479, |
| "rewards/margins": 6.1157379150390625, |
| "rewards/rejected": -7.302254676818848, |
| "step": 1455 |
| }, |
| { |
| "epoch": 2.336, |
| "grad_norm": 5.8520379066467285, |
| "learning_rate": 6.18621293953704e-07, |
| "logits/chosen": -1.5015791654586792, |
| "logits/rejected": -1.458094835281372, |
| "logps/chosen": -245.0795135498047, |
| "logps/rejected": -312.7300109863281, |
| "loss": 0.1316, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.6283895373344421, |
| "rewards/margins": 5.754814147949219, |
| "rewards/rejected": -6.383203983306885, |
| "step": 1460 |
| }, |
| { |
| "epoch": 2.344, |
| "grad_norm": 4.313529014587402, |
| "learning_rate": 6.044669732505968e-07, |
| "logits/chosen": -1.4273951053619385, |
| "logits/rejected": -1.4270398616790771, |
| "logps/chosen": -232.2421417236328, |
| "logps/rejected": -308.38275146484375, |
| "loss": 0.157, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.1415029764175415, |
| "rewards/margins": 5.1364593505859375, |
| "rewards/rejected": -6.277962684631348, |
| "step": 1465 |
| }, |
| { |
| "epoch": 2.352, |
| "grad_norm": 3.829115629196167, |
| "learning_rate": 5.904541604358257e-07, |
| "logits/chosen": -1.5568525791168213, |
| "logits/rejected": -1.4775985479354858, |
| "logps/chosen": -231.17343139648438, |
| "logps/rejected": -319.28338623046875, |
| "loss": 0.1103, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6411401033401489, |
| "rewards/margins": 6.239230632781982, |
| "rewards/rejected": -6.880370140075684, |
| "step": 1470 |
| }, |
| { |
| "epoch": 2.36, |
| "grad_norm": 3.9916186332702637, |
| "learning_rate": 5.765839016127953e-07, |
| "logits/chosen": -1.601308822631836, |
| "logits/rejected": -1.556640386581421, |
| "logps/chosen": -247.08151245117188, |
| "logps/rejected": -326.2941589355469, |
| "loss": 0.0974, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.1435987949371338, |
| "rewards/margins": 6.010471343994141, |
| "rewards/rejected": -7.1540703773498535, |
| "step": 1475 |
| }, |
| { |
| "epoch": 2.368, |
| "grad_norm": 4.295379638671875, |
| "learning_rate": 5.628572322427755e-07, |
| "logits/chosen": -1.4031133651733398, |
| "logits/rejected": -1.4401230812072754, |
| "logps/chosen": -246.1109161376953, |
| "logps/rejected": -311.51519775390625, |
| "loss": 0.1096, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9082221984863281, |
| "rewards/margins": 6.309326648712158, |
| "rewards/rejected": -7.217548370361328, |
| "step": 1480 |
| }, |
| { |
| "epoch": 2.376, |
| "grad_norm": 5.539330005645752, |
| "learning_rate": 5.492751770676042e-07, |
| "logits/chosen": -1.4508850574493408, |
| "logits/rejected": -1.484483003616333, |
| "logps/chosen": -248.3240966796875, |
| "logps/rejected": -311.86114501953125, |
| "loss": 0.1696, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7610406875610352, |
| "rewards/margins": 5.5662055015563965, |
| "rewards/rejected": -6.327246189117432, |
| "step": 1485 |
| }, |
| { |
| "epoch": 2.384, |
| "grad_norm": 2.899242639541626, |
| "learning_rate": 5.358387500331844e-07, |
| "logits/chosen": -1.550159215927124, |
| "logits/rejected": -1.5097694396972656, |
| "logps/chosen": -220.2747039794922, |
| "logps/rejected": -304.1933288574219, |
| "loss": 0.1453, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.11124587059021, |
| "rewards/margins": 5.5194854736328125, |
| "rewards/rejected": -6.630731105804443, |
| "step": 1490 |
| }, |
| { |
| "epoch": 2.392, |
| "grad_norm": 4.426836967468262, |
| "learning_rate": 5.22548954213791e-07, |
| "logits/chosen": -1.5275931358337402, |
| "logits/rejected": -1.5510616302490234, |
| "logps/chosen": -265.9932556152344, |
| "logps/rejected": -331.5732116699219, |
| "loss": 0.1398, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.0064342021942139, |
| "rewards/margins": 6.052700996398926, |
| "rewards/rejected": -7.059134483337402, |
| "step": 1495 |
| }, |
| { |
| "epoch": 2.4, |
| "grad_norm": 2.684803009033203, |
| "learning_rate": 5.094067817371878e-07, |
| "logits/chosen": -1.4548851251602173, |
| "logits/rejected": -1.4815185070037842, |
| "logps/chosen": -228.2082977294922, |
| "logps/rejected": -318.86248779296875, |
| "loss": 0.1197, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7952719926834106, |
| "rewards/margins": 6.384288787841797, |
| "rewards/rejected": -7.179561614990234, |
| "step": 1500 |
| }, |
| { |
| "epoch": 2.408, |
| "grad_norm": 2.261495590209961, |
| "learning_rate": 4.964132137105617e-07, |
| "logits/chosen": -1.4797651767730713, |
| "logits/rejected": -1.440944790840149, |
| "logps/chosen": -236.2601776123047, |
| "logps/rejected": -323.2953186035156, |
| "loss": 0.1415, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6942367553710938, |
| "rewards/margins": 5.794956207275391, |
| "rewards/rejected": -6.489192962646484, |
| "step": 1505 |
| }, |
| { |
| "epoch": 2.416, |
| "grad_norm": 3.655900716781616, |
| "learning_rate": 4.835692201472794e-07, |
| "logits/chosen": -1.5585110187530518, |
| "logits/rejected": -1.4639651775360107, |
| "logps/chosen": -221.63577270507812, |
| "logps/rejected": -317.4681396484375, |
| "loss": 0.1161, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.9286325573921204, |
| "rewards/margins": 5.880457878112793, |
| "rewards/rejected": -6.809090614318848, |
| "step": 1510 |
| }, |
| { |
| "epoch": 2.424, |
| "grad_norm": 4.601859092712402, |
| "learning_rate": 4.7087575989447353e-07, |
| "logits/chosen": -1.5128328800201416, |
| "logits/rejected": -1.4157400131225586, |
| "logps/chosen": -252.9822235107422, |
| "logps/rejected": -314.6139221191406, |
| "loss": 0.1445, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.8977339863777161, |
| "rewards/margins": 5.779736518859863, |
| "rewards/rejected": -6.6774702072143555, |
| "step": 1515 |
| }, |
| { |
| "epoch": 2.432, |
| "grad_norm": 4.9197869300842285, |
| "learning_rate": 4.583337805614602e-07, |
| "logits/chosen": -1.4999001026153564, |
| "logits/rejected": -1.4343955516815186, |
| "logps/chosen": -229.39651489257812, |
| "logps/rejected": -317.10858154296875, |
| "loss": 0.1504, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.8339191675186157, |
| "rewards/margins": 5.6455535888671875, |
| "rewards/rejected": -6.479473114013672, |
| "step": 1520 |
| }, |
| { |
| "epoch": 2.44, |
| "grad_norm": 4.315069198608398, |
| "learning_rate": 4.4594421844899844e-07, |
| "logits/chosen": -1.4863693714141846, |
| "logits/rejected": -1.4807065725326538, |
| "logps/chosen": -227.49496459960938, |
| "logps/rejected": -310.77850341796875, |
| "loss": 0.1039, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.9351770281791687, |
| "rewards/margins": 6.132818698883057, |
| "rewards/rejected": -7.067995548248291, |
| "step": 1525 |
| }, |
| { |
| "epoch": 2.448, |
| "grad_norm": 4.87631368637085, |
| "learning_rate": 4.33707998479391e-07, |
| "logits/chosen": -1.582213282585144, |
| "logits/rejected": -1.6060279607772827, |
| "logps/chosen": -299.22735595703125, |
| "logps/rejected": -332.704833984375, |
| "loss": 0.2081, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.0234695672988892, |
| "rewards/margins": 5.895220756530762, |
| "rewards/rejected": -6.9186906814575195, |
| "step": 1530 |
| }, |
| { |
| "epoch": 2.456, |
| "grad_norm": 3.5169260501861572, |
| "learning_rate": 4.2162603412743585e-07, |
| "logits/chosen": -1.4920437335968018, |
| "logits/rejected": -1.474491834640503, |
| "logps/chosen": -229.24575805664062, |
| "logps/rejected": -305.94171142578125, |
| "loss": 0.1227, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8777539134025574, |
| "rewards/margins": 5.8880133628845215, |
| "rewards/rejected": -6.765768527984619, |
| "step": 1535 |
| }, |
| { |
| "epoch": 2.464, |
| "grad_norm": 3.1716701984405518, |
| "learning_rate": 4.0969922735223296e-07, |
| "logits/chosen": -1.5159512758255005, |
| "logits/rejected": -1.4225298166275024, |
| "logps/chosen": -221.02584838867188, |
| "logps/rejected": -322.4046630859375, |
| "loss": 0.126, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9317782521247864, |
| "rewards/margins": 6.3952813148498535, |
| "rewards/rejected": -7.327059268951416, |
| "step": 1540 |
| }, |
| { |
| "epoch": 2.472, |
| "grad_norm": 1.5373523235321045, |
| "learning_rate": 3.979284685298493e-07, |
| "logits/chosen": -1.5270391702651978, |
| "logits/rejected": -1.4918533563613892, |
| "logps/chosen": -231.8376922607422, |
| "logps/rejected": -322.9443359375, |
| "loss": 0.0849, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.8065363764762878, |
| "rewards/margins": 6.097052574157715, |
| "rewards/rejected": -6.903588771820068, |
| "step": 1545 |
| }, |
| { |
| "epoch": 2.48, |
| "grad_norm": 5.075746536254883, |
| "learning_rate": 3.863146363868495e-07, |
| "logits/chosen": -1.5676982402801514, |
| "logits/rejected": -1.4815164804458618, |
| "logps/chosen": -226.5551300048828, |
| "logps/rejected": -311.35284423828125, |
| "loss": 0.1273, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.0178006887435913, |
| "rewards/margins": 6.11976957321167, |
| "rewards/rejected": -7.137570381164551, |
| "step": 1550 |
| }, |
| { |
| "epoch": 2.488, |
| "grad_norm": 4.667273998260498, |
| "learning_rate": 3.7485859793469636e-07, |
| "logits/chosen": -1.4717382192611694, |
| "logits/rejected": -1.4203176498413086, |
| "logps/chosen": -251.059814453125, |
| "logps/rejected": -314.29156494140625, |
| "loss": 0.1622, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6194905042648315, |
| "rewards/margins": 5.366819381713867, |
| "rewards/rejected": -5.986310005187988, |
| "step": 1555 |
| }, |
| { |
| "epoch": 2.496, |
| "grad_norm": 4.541383266448975, |
| "learning_rate": 3.6356120840502553e-07, |
| "logits/chosen": -1.598649024963379, |
| "logits/rejected": -1.6106332540512085, |
| "logps/chosen": -227.8941650390625, |
| "logps/rejected": -312.3360290527344, |
| "loss": 0.1947, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.984524130821228, |
| "rewards/margins": 5.360861778259277, |
| "rewards/rejected": -6.345385551452637, |
| "step": 1560 |
| }, |
| { |
| "epoch": 2.504, |
| "grad_norm": 3.541747570037842, |
| "learning_rate": 3.52423311185798e-07, |
| "logits/chosen": -1.4794892072677612, |
| "logits/rejected": -1.4599707126617432, |
| "logps/chosen": -223.511962890625, |
| "logps/rejected": -308.583984375, |
| "loss": 0.143, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.2889810800552368, |
| "rewards/margins": 5.206359386444092, |
| "rewards/rejected": -6.495339870452881, |
| "step": 1565 |
| }, |
| { |
| "epoch": 2.512, |
| "grad_norm": 4.896753311157227, |
| "learning_rate": 3.414457377583413e-07, |
| "logits/chosen": -1.512119174003601, |
| "logits/rejected": -1.4517587423324585, |
| "logps/chosen": -250.6962127685547, |
| "logps/rejected": -331.4056701660156, |
| "loss": 0.1387, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.8094693422317505, |
| "rewards/margins": 6.361907958984375, |
| "rewards/rejected": -7.171376705169678, |
| "step": 1570 |
| }, |
| { |
| "epoch": 2.52, |
| "grad_norm": 5.767399787902832, |
| "learning_rate": 3.30629307635274e-07, |
| "logits/chosen": -1.5372960567474365, |
| "logits/rejected": -1.5459281206130981, |
| "logps/chosen": -222.1968994140625, |
| "logps/rejected": -318.656005859375, |
| "loss": 0.1104, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.6385549306869507, |
| "rewards/margins": 6.328636169433594, |
| "rewards/rejected": -6.967191219329834, |
| "step": 1575 |
| }, |
| { |
| "epoch": 2.528, |
| "grad_norm": 2.8931169509887695, |
| "learning_rate": 3.199748282993273e-07, |
| "logits/chosen": -1.5647623538970947, |
| "logits/rejected": -1.4885820150375366, |
| "logps/chosen": -230.0967559814453, |
| "logps/rejected": -316.1094665527344, |
| "loss": 0.1245, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.6826328039169312, |
| "rewards/margins": 6.465601921081543, |
| "rewards/rejected": -7.148235321044922, |
| "step": 1580 |
| }, |
| { |
| "epoch": 2.536, |
| "grad_norm": 3.3651797771453857, |
| "learning_rate": 3.094830951430644e-07, |
| "logits/chosen": -1.4947978258132935, |
| "logits/rejected": -1.4919826984405518, |
| "logps/chosen": -247.35824584960938, |
| "logps/rejected": -329.46697998046875, |
| "loss": 0.1327, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8214322924613953, |
| "rewards/margins": 6.117218017578125, |
| "rewards/rejected": -6.938650608062744, |
| "step": 1585 |
| }, |
| { |
| "epoch": 2.544, |
| "grad_norm": 3.0831105709075928, |
| "learning_rate": 2.9915489140950076e-07, |
| "logits/chosen": -1.534934401512146, |
| "logits/rejected": -1.5211741924285889, |
| "logps/chosen": -238.5296173095703, |
| "logps/rejected": -291.08160400390625, |
| "loss": 0.1872, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.9827302694320679, |
| "rewards/margins": 5.219890117645264, |
| "rewards/rejected": -6.202620506286621, |
| "step": 1590 |
| }, |
| { |
| "epoch": 2.552, |
| "grad_norm": 3.8352949619293213, |
| "learning_rate": 2.889909881336328e-07, |
| "logits/chosen": -1.5629241466522217, |
| "logits/rejected": -1.5492066144943237, |
| "logps/chosen": -246.606201171875, |
| "logps/rejected": -315.3060607910156, |
| "loss": 0.1732, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -1.1226515769958496, |
| "rewards/margins": 5.307157039642334, |
| "rewards/rejected": -6.4298095703125, |
| "step": 1595 |
| }, |
| { |
| "epoch": 2.56, |
| "grad_norm": 3.378814458847046, |
| "learning_rate": 2.789921440848778e-07, |
| "logits/chosen": -1.693416953086853, |
| "logits/rejected": -1.556857705116272, |
| "logps/chosen": -233.92196655273438, |
| "logps/rejected": -321.92962646484375, |
| "loss": 0.1014, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.9284855127334595, |
| "rewards/margins": 6.132410049438477, |
| "rewards/rejected": -7.0608954429626465, |
| "step": 1600 |
| }, |
| { |
| "epoch": 2.568, |
| "grad_norm": 4.796236038208008, |
| "learning_rate": 2.691591057104287e-07, |
| "logits/chosen": -1.5096356868743896, |
| "logits/rejected": -1.532952070236206, |
| "logps/chosen": -236.99169921875, |
| "logps/rejected": -332.6849060058594, |
| "loss": 0.1642, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.5096395611763, |
| "rewards/margins": 5.59578800201416, |
| "rewards/rejected": -6.1054277420043945, |
| "step": 1605 |
| }, |
| { |
| "epoch": 2.576, |
| "grad_norm": 3.768148899078369, |
| "learning_rate": 2.594926070795295e-07, |
| "logits/chosen": -1.5512020587921143, |
| "logits/rejected": -1.5035789012908936, |
| "logps/chosen": -243.709716796875, |
| "logps/rejected": -319.43084716796875, |
| "loss": 0.1359, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9202622175216675, |
| "rewards/margins": 5.834639072418213, |
| "rewards/rejected": -6.754900932312012, |
| "step": 1610 |
| }, |
| { |
| "epoch": 2.584, |
| "grad_norm": 4.090240001678467, |
| "learning_rate": 2.4999336982867496e-07, |
| "logits/chosen": -1.4447572231292725, |
| "logits/rejected": -1.467057228088379, |
| "logps/chosen": -246.8234405517578, |
| "logps/rejected": -334.2413330078125, |
| "loss": 0.141, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9848871231079102, |
| "rewards/margins": 5.94996452331543, |
| "rewards/rejected": -6.934852600097656, |
| "step": 1615 |
| }, |
| { |
| "epoch": 2.592, |
| "grad_norm": 3.5356523990631104, |
| "learning_rate": 2.4066210310773776e-07, |
| "logits/chosen": -1.4008734226226807, |
| "logits/rejected": -1.4272077083587646, |
| "logps/chosen": -227.53982543945312, |
| "logps/rejected": -310.771240234375, |
| "loss": 0.1112, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.43771782517433167, |
| "rewards/margins": 5.975172996520996, |
| "rewards/rejected": -6.412891387939453, |
| "step": 1620 |
| }, |
| { |
| "epoch": 2.6, |
| "grad_norm": 1.1229864358901978, |
| "learning_rate": 2.3149950352702787e-07, |
| "logits/chosen": -1.575382947921753, |
| "logits/rejected": -1.5440458059310913, |
| "logps/chosen": -227.5854034423828, |
| "logps/rejected": -332.78656005859375, |
| "loss": 0.0806, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -0.9323962926864624, |
| "rewards/margins": 6.606331825256348, |
| "rewards/rejected": -7.538727760314941, |
| "step": 1625 |
| }, |
| { |
| "epoch": 2.608, |
| "grad_norm": 2.3300342559814453, |
| "learning_rate": 2.2250625510528783e-07, |
| "logits/chosen": -1.48783278465271, |
| "logits/rejected": -1.398735761642456, |
| "logps/chosen": -235.2317657470703, |
| "logps/rejected": -308.76959228515625, |
| "loss": 0.1099, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5314217805862427, |
| "rewards/margins": 6.1185760498046875, |
| "rewards/rejected": -6.649998664855957, |
| "step": 1630 |
| }, |
| { |
| "epoch": 2.616, |
| "grad_norm": 3.574950933456421, |
| "learning_rate": 2.136830292186298e-07, |
| "logits/chosen": -1.4672324657440186, |
| "logits/rejected": -1.4376590251922607, |
| "logps/chosen": -249.235107421875, |
| "logps/rejected": -325.9244384765625, |
| "loss": 0.1538, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7882421612739563, |
| "rewards/margins": 5.851679801940918, |
| "rewards/rejected": -6.63992166519165, |
| "step": 1635 |
| }, |
| { |
| "epoch": 2.624, |
| "grad_norm": 1.543809413909912, |
| "learning_rate": 2.050304845504139e-07, |
| "logits/chosen": -1.525334119796753, |
| "logits/rejected": -1.3860852718353271, |
| "logps/chosen": -259.4710693359375, |
| "logps/rejected": -307.67340087890625, |
| "loss": 0.1104, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.8463247418403625, |
| "rewards/margins": 5.842609405517578, |
| "rewards/rejected": -6.688934326171875, |
| "step": 1640 |
| }, |
| { |
| "epoch": 2.632, |
| "grad_norm": 2.3976762294769287, |
| "learning_rate": 1.9654926704207627e-07, |
| "logits/chosen": -1.4831944704055786, |
| "logits/rejected": -1.4159176349639893, |
| "logps/chosen": -228.45761108398438, |
| "logps/rejected": -317.77362060546875, |
| "loss": 0.1252, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.6478134393692017, |
| "rewards/margins": 5.69929313659668, |
| "rewards/rejected": -6.347106456756592, |
| "step": 1645 |
| }, |
| { |
| "epoch": 2.64, |
| "grad_norm": 2.3702242374420166, |
| "learning_rate": 1.882400098449058e-07, |
| "logits/chosen": -1.5494359731674194, |
| "logits/rejected": -1.5194289684295654, |
| "logps/chosen": -249.11532592773438, |
| "logps/rejected": -323.60675048828125, |
| "loss": 0.1229, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.8728028535842896, |
| "rewards/margins": 5.897127628326416, |
| "rewards/rejected": -6.769930839538574, |
| "step": 1650 |
| }, |
| { |
| "epoch": 2.648, |
| "grad_norm": 5.607757091522217, |
| "learning_rate": 1.8010333327277947e-07, |
| "logits/chosen": -1.5770350694656372, |
| "logits/rejected": -1.5135689973831177, |
| "logps/chosen": -245.56640625, |
| "logps/rejected": -333.17462158203125, |
| "loss": 0.145, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.2801220417022705, |
| "rewards/margins": 5.944527626037598, |
| "rewards/rejected": -7.2246503829956055, |
| "step": 1655 |
| }, |
| { |
| "epoch": 2.656, |
| "grad_norm": 6.220284461975098, |
| "learning_rate": 1.7213984475585143e-07, |
| "logits/chosen": -1.499732255935669, |
| "logits/rejected": -1.4527981281280518, |
| "logps/chosen": -209.24880981445312, |
| "logps/rejected": -297.71746826171875, |
| "loss": 0.1497, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8399412035942078, |
| "rewards/margins": 5.599820613861084, |
| "rewards/rejected": -6.439762115478516, |
| "step": 1660 |
| }, |
| { |
| "epoch": 2.664, |
| "grad_norm": 6.647799491882324, |
| "learning_rate": 1.6435013879520844e-07, |
| "logits/chosen": -1.5817861557006836, |
| "logits/rejected": -1.5299128293991089, |
| "logps/chosen": -240.9619598388672, |
| "logps/rejected": -315.337158203125, |
| "loss": 0.1264, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8431243896484375, |
| "rewards/margins": 5.834400177001953, |
| "rewards/rejected": -6.677524566650391, |
| "step": 1665 |
| }, |
| { |
| "epoch": 2.672, |
| "grad_norm": 3.162820816040039, |
| "learning_rate": 1.5673479691848736e-07, |
| "logits/chosen": -1.538938045501709, |
| "logits/rejected": -1.5266144275665283, |
| "logps/chosen": -255.1186065673828, |
| "logps/rejected": -306.49041748046875, |
| "loss": 0.116, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8456069827079773, |
| "rewards/margins": 6.127451419830322, |
| "rewards/rejected": -6.973057746887207, |
| "step": 1670 |
| }, |
| { |
| "epoch": 2.68, |
| "grad_norm": 3.5515389442443848, |
| "learning_rate": 1.492943876364611e-07, |
| "logits/chosen": -1.576812744140625, |
| "logits/rejected": -1.541210412979126, |
| "logps/chosen": -245.21884155273438, |
| "logps/rejected": -319.4032897949219, |
| "loss": 0.122, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8992266654968262, |
| "rewards/margins": 5.852095603942871, |
| "rewards/rejected": -6.7513227462768555, |
| "step": 1675 |
| }, |
| { |
| "epoch": 2.6879999999999997, |
| "grad_norm": 3.37399959564209, |
| "learning_rate": 1.4202946640060138e-07, |
| "logits/chosen": -1.4246790409088135, |
| "logits/rejected": -1.4544460773468018, |
| "logps/chosen": -229.1632843017578, |
| "logps/rejected": -313.3404846191406, |
| "loss": 0.1375, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.7347529530525208, |
| "rewards/margins": 5.777396202087402, |
| "rewards/rejected": -6.512149810791016, |
| "step": 1680 |
| }, |
| { |
| "epoch": 2.6959999999999997, |
| "grad_norm": 2.765533685684204, |
| "learning_rate": 1.3494057556160567e-07, |
| "logits/chosen": -1.5078999996185303, |
| "logits/rejected": -1.4551175832748413, |
| "logps/chosen": -241.14511108398438, |
| "logps/rejected": -319.65020751953125, |
| "loss": 0.1085, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.8278571367263794, |
| "rewards/margins": 5.884353160858154, |
| "rewards/rejected": -6.712210655212402, |
| "step": 1685 |
| }, |
| { |
| "epoch": 2.7039999999999997, |
| "grad_norm": 5.154428482055664, |
| "learning_rate": 1.2802824432891658e-07, |
| "logits/chosen": -1.5341852903366089, |
| "logits/rejected": -1.4802993535995483, |
| "logps/chosen": -264.14093017578125, |
| "logps/rejected": -332.47259521484375, |
| "loss": 0.1298, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7193272113800049, |
| "rewards/margins": 5.93912410736084, |
| "rewards/rejected": -6.658452033996582, |
| "step": 1690 |
| }, |
| { |
| "epoch": 2.7119999999999997, |
| "grad_norm": 5.017862319946289, |
| "learning_rate": 1.2129298873120753e-07, |
| "logits/chosen": -1.5598084926605225, |
| "logits/rejected": -1.4870166778564453, |
| "logps/chosen": -231.30032348632812, |
| "logps/rejected": -324.11279296875, |
| "loss": 0.1291, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8461719751358032, |
| "rewards/margins": 6.092933654785156, |
| "rewards/rejected": -6.939105987548828, |
| "step": 1695 |
| }, |
| { |
| "epoch": 2.7199999999999998, |
| "grad_norm": 2.480734348297119, |
| "learning_rate": 1.14735311577866e-07, |
| "logits/chosen": -1.5418061017990112, |
| "logits/rejected": -1.558050274848938, |
| "logps/chosen": -249.1891632080078, |
| "logps/rejected": -333.8502502441406, |
| "loss": 0.1441, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1636663675308228, |
| "rewards/margins": 6.228588104248047, |
| "rewards/rejected": -7.392254829406738, |
| "step": 1700 |
| }, |
| { |
| "epoch": 2.7279999999999998, |
| "grad_norm": 4.858855724334717, |
| "learning_rate": 1.0835570242145072e-07, |
| "logits/chosen": -1.6297658681869507, |
| "logits/rejected": -1.5588557720184326, |
| "logps/chosen": -239.9254913330078, |
| "logps/rejected": -313.71563720703125, |
| "loss": 0.1529, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.8668500781059265, |
| "rewards/margins": 5.69160795211792, |
| "rewards/rejected": -6.558457851409912, |
| "step": 1705 |
| }, |
| { |
| "epoch": 2.7359999999999998, |
| "grad_norm": 3.9852185249328613, |
| "learning_rate": 1.0215463752115101e-07, |
| "logits/chosen": -1.5099657773971558, |
| "logits/rejected": -1.4762407541275024, |
| "logps/chosen": -220.3232421875, |
| "logps/rejected": -319.54681396484375, |
| "loss": 0.1223, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7317541241645813, |
| "rewards/margins": 6.199631214141846, |
| "rewards/rejected": -6.931385040283203, |
| "step": 1710 |
| }, |
| { |
| "epoch": 2.7439999999999998, |
| "grad_norm": 1.988715410232544, |
| "learning_rate": 9.613257980722685e-08, |
| "logits/chosen": -1.5078150033950806, |
| "logits/rejected": -1.4630861282348633, |
| "logps/chosen": -242.2015838623047, |
| "logps/rejected": -299.0679931640625, |
| "loss": 0.0811, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5184379816055298, |
| "rewards/margins": 5.81303071975708, |
| "rewards/rejected": -6.33146858215332, |
| "step": 1715 |
| }, |
| { |
| "epoch": 2.752, |
| "grad_norm": 4.002151966094971, |
| "learning_rate": 9.028997884645535e-08, |
| "logits/chosen": -1.4705991744995117, |
| "logits/rejected": -1.4686193466186523, |
| "logps/chosen": -226.935791015625, |
| "logps/rejected": -305.57562255859375, |
| "loss": 0.1356, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.1761690378189087, |
| "rewards/margins": 5.607525825500488, |
| "rewards/rejected": -6.783695220947266, |
| "step": 1720 |
| }, |
| { |
| "epoch": 2.76, |
| "grad_norm": 1.376519799232483, |
| "learning_rate": 8.462727080856243e-08, |
| "logits/chosen": -1.5141279697418213, |
| "logits/rejected": -1.448982834815979, |
| "logps/chosen": -237.4274444580078, |
| "logps/rejected": -339.56768798828125, |
| "loss": 0.1158, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.5523132085800171, |
| "rewards/margins": 6.665265083312988, |
| "rewards/rejected": -7.2175774574279785, |
| "step": 1725 |
| }, |
| { |
| "epoch": 2.768, |
| "grad_norm": 2.3542063236236572, |
| "learning_rate": 7.914487843366875e-08, |
| "logits/chosen": -1.5205110311508179, |
| "logits/rejected": -1.4258232116699219, |
| "logps/chosen": -232.4739227294922, |
| "logps/rejected": -308.33062744140625, |
| "loss": 0.1493, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8891655206680298, |
| "rewards/margins": 5.613663673400879, |
| "rewards/rejected": -6.502829074859619, |
| "step": 1730 |
| }, |
| { |
| "epoch": 2.776, |
| "grad_norm": 1.9824843406677246, |
| "learning_rate": 7.384321100072339e-08, |
| "logits/chosen": -1.5084909200668335, |
| "logits/rejected": -1.4419305324554443, |
| "logps/chosen": -244.23532104492188, |
| "logps/rejected": -316.4294128417969, |
| "loss": 0.1651, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.8520871996879578, |
| "rewards/margins": 6.0770697593688965, |
| "rewards/rejected": -6.9291582107543945, |
| "step": 1735 |
| }, |
| { |
| "epoch": 2.784, |
| "grad_norm": 4.31235408782959, |
| "learning_rate": 6.872266429695518e-08, |
| "logits/chosen": -1.5736935138702393, |
| "logits/rejected": -1.4683692455291748, |
| "logps/chosen": -243.3426513671875, |
| "logps/rejected": -316.1590576171875, |
| "loss": 0.1086, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.9821755290031433, |
| "rewards/margins": 5.919834136962891, |
| "rewards/rejected": -6.9020094871521, |
| "step": 1740 |
| }, |
| { |
| "epoch": 2.792, |
| "grad_norm": 6.580287456512451, |
| "learning_rate": 6.37836205883241e-08, |
| "logits/chosen": -1.4502264261245728, |
| "logits/rejected": -1.4019014835357666, |
| "logps/chosen": -237.04660034179688, |
| "logps/rejected": -304.73858642578125, |
| "loss": 0.1768, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6265815496444702, |
| "rewards/margins": 5.869175910949707, |
| "rewards/rejected": -6.495758056640625, |
| "step": 1745 |
| }, |
| { |
| "epoch": 2.8, |
| "grad_norm": 3.6616616249084473, |
| "learning_rate": 5.9026448590982497e-08, |
| "logits/chosen": -1.5317808389663696, |
| "logits/rejected": -1.4841121435165405, |
| "logps/chosen": -234.8379669189453, |
| "logps/rejected": -316.34454345703125, |
| "loss": 0.1134, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.7818082571029663, |
| "rewards/margins": 6.074831962585449, |
| "rewards/rejected": -6.856640815734863, |
| "step": 1750 |
| }, |
| { |
| "epoch": 2.808, |
| "grad_norm": 2.6014156341552734, |
| "learning_rate": 5.4451503443751774e-08, |
| "logits/chosen": -1.4140145778656006, |
| "logits/rejected": -1.4126405715942383, |
| "logps/chosen": -236.1201629638672, |
| "logps/rejected": -313.41497802734375, |
| "loss": 0.0918, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9264975786209106, |
| "rewards/margins": 6.3627729415893555, |
| "rewards/rejected": -7.289270877838135, |
| "step": 1755 |
| }, |
| { |
| "epoch": 2.816, |
| "grad_norm": 4.039976119995117, |
| "learning_rate": 5.0059126681607226e-08, |
| "logits/chosen": -1.4606797695159912, |
| "logits/rejected": -1.4615904092788696, |
| "logps/chosen": -233.7847137451172, |
| "logps/rejected": -324.12091064453125, |
| "loss": 0.121, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.6958805322647095, |
| "rewards/margins": 6.251741886138916, |
| "rewards/rejected": -6.9476213455200195, |
| "step": 1760 |
| }, |
| { |
| "epoch": 2.824, |
| "grad_norm": 3.214773654937744, |
| "learning_rate": 4.5849646210184837e-08, |
| "logits/chosen": -1.4053151607513428, |
| "logits/rejected": -1.4549171924591064, |
| "logps/chosen": -231.1645965576172, |
| "logps/rejected": -330.7877197265625, |
| "loss": 0.1145, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.609539806842804, |
| "rewards/margins": 6.281281471252441, |
| "rewards/rejected": -6.8908209800720215, |
| "step": 1765 |
| }, |
| { |
| "epoch": 2.832, |
| "grad_norm": 1.2576606273651123, |
| "learning_rate": 4.1823376281297515e-08, |
| "logits/chosen": -1.3523794412612915, |
| "logits/rejected": -1.399296522140503, |
| "logps/chosen": -261.13092041015625, |
| "logps/rejected": -322.9598083496094, |
| "loss": 0.1283, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.6175804734230042, |
| "rewards/margins": 6.264570713043213, |
| "rewards/rejected": -6.882150173187256, |
| "step": 1770 |
| }, |
| { |
| "epoch": 2.84, |
| "grad_norm": 2.263697385787964, |
| "learning_rate": 3.798061746947995e-08, |
| "logits/chosen": -1.4639291763305664, |
| "logits/rejected": -1.4517295360565186, |
| "logps/chosen": -242.2957763671875, |
| "logps/rejected": -311.65545654296875, |
| "loss": 0.1298, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6775432825088501, |
| "rewards/margins": 5.772078990936279, |
| "rewards/rejected": -6.449623107910156, |
| "step": 1775 |
| }, |
| { |
| "epoch": 2.848, |
| "grad_norm": 5.509952545166016, |
| "learning_rate": 3.432165664954551e-08, |
| "logits/chosen": -1.5598200559616089, |
| "logits/rejected": -1.474449872970581, |
| "logps/chosen": -258.42950439453125, |
| "logps/rejected": -309.0158386230469, |
| "loss": 0.1269, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.9016896486282349, |
| "rewards/margins": 5.584256649017334, |
| "rewards/rejected": -6.4859466552734375, |
| "step": 1780 |
| }, |
| { |
| "epoch": 2.856, |
| "grad_norm": 5.17020845413208, |
| "learning_rate": 3.0846766975173583e-08, |
| "logits/chosen": -1.5272047519683838, |
| "logits/rejected": -1.4640551805496216, |
| "logps/chosen": -224.8867645263672, |
| "logps/rejected": -301.6604919433594, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.9279806017875671, |
| "rewards/margins": 5.865192890167236, |
| "rewards/rejected": -6.793173313140869, |
| "step": 1785 |
| }, |
| { |
| "epoch": 2.864, |
| "grad_norm": 5.173901081085205, |
| "learning_rate": 2.7556207858514262e-08, |
| "logits/chosen": -1.5167121887207031, |
| "logits/rejected": -1.535841464996338, |
| "logps/chosen": -249.1966552734375, |
| "logps/rejected": -297.0778503417969, |
| "loss": 0.1406, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.1131664514541626, |
| "rewards/margins": 5.719702243804932, |
| "rewards/rejected": -6.832869052886963, |
| "step": 1790 |
| }, |
| { |
| "epoch": 2.872, |
| "grad_norm": 2.5215299129486084, |
| "learning_rate": 2.4450224950826607e-08, |
| "logits/chosen": -1.543928623199463, |
| "logits/rejected": -1.4860496520996094, |
| "logps/chosen": -220.95565795898438, |
| "logps/rejected": -311.44158935546875, |
| "loss": 0.1465, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9338169097900391, |
| "rewards/margins": 5.805626392364502, |
| "rewards/rejected": -6.739443302154541, |
| "step": 1795 |
| }, |
| { |
| "epoch": 2.88, |
| "grad_norm": 2.975391387939453, |
| "learning_rate": 2.15290501241347e-08, |
| "logits/chosen": -1.5811344385147095, |
| "logits/rejected": -1.5057846307754517, |
| "logps/chosen": -236.87191772460938, |
| "logps/rejected": -321.338134765625, |
| "loss": 0.1335, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7364934682846069, |
| "rewards/margins": 6.344755172729492, |
| "rewards/rejected": -7.081249237060547, |
| "step": 1800 |
| }, |
| { |
| "epoch": 2.888, |
| "grad_norm": 3.126237154006958, |
| "learning_rate": 1.8792901453923428e-08, |
| "logits/chosen": -1.5378950834274292, |
| "logits/rejected": -1.516510248184204, |
| "logps/chosen": -237.1420135498047, |
| "logps/rejected": -311.3038635253906, |
| "loss": 0.1549, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9023133516311646, |
| "rewards/margins": 5.623795032501221, |
| "rewards/rejected": -6.526108741760254, |
| "step": 1805 |
| }, |
| { |
| "epoch": 2.896, |
| "grad_norm": 4.797680854797363, |
| "learning_rate": 1.624198320285264e-08, |
| "logits/chosen": -1.5249507427215576, |
| "logits/rejected": -1.4735972881317139, |
| "logps/chosen": -234.7667999267578, |
| "logps/rejected": -326.597412109375, |
| "loss": 0.1449, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.2055195569992065, |
| "rewards/margins": 5.8126044273376465, |
| "rewards/rejected": -7.018124580383301, |
| "step": 1810 |
| }, |
| { |
| "epoch": 2.904, |
| "grad_norm": 1.3551347255706787, |
| "learning_rate": 1.3876485805513762e-08, |
| "logits/chosen": -1.5904200077056885, |
| "logits/rejected": -1.6041524410247803, |
| "logps/chosen": -225.4705810546875, |
| "logps/rejected": -329.77874755859375, |
| "loss": 0.118, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1419427394866943, |
| "rewards/margins": 6.027765274047852, |
| "rewards/rejected": -7.169708251953125, |
| "step": 1815 |
| }, |
| { |
| "epoch": 2.912, |
| "grad_norm": 3.9451160430908203, |
| "learning_rate": 1.169658585420813e-08, |
| "logits/chosen": -1.4940643310546875, |
| "logits/rejected": -1.5199902057647705, |
| "logps/chosen": -231.20944213867188, |
| "logps/rejected": -320.60662841796875, |
| "loss": 0.1453, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.0270144939422607, |
| "rewards/margins": 6.1724395751953125, |
| "rewards/rejected": -7.199454307556152, |
| "step": 1820 |
| }, |
| { |
| "epoch": 2.92, |
| "grad_norm": 2.1993324756622314, |
| "learning_rate": 9.702446085768091e-09, |
| "logits/chosen": -1.514919400215149, |
| "logits/rejected": -1.537259817123413, |
| "logps/chosen": -227.4259033203125, |
| "logps/rejected": -331.2008361816406, |
| "loss": 0.1286, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.0264793634414673, |
| "rewards/margins": 6.270672798156738, |
| "rewards/rejected": -7.297152519226074, |
| "step": 1825 |
| }, |
| { |
| "epoch": 2.928, |
| "grad_norm": 5.0035080909729, |
| "learning_rate": 7.894215369405322e-09, |
| "logits/chosen": -1.4085063934326172, |
| "logits/rejected": -1.3085895776748657, |
| "logps/chosen": -240.96902465820312, |
| "logps/rejected": -309.6005859375, |
| "loss": 0.1369, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.6670558452606201, |
| "rewards/margins": 5.737823486328125, |
| "rewards/rejected": -6.404879570007324, |
| "step": 1830 |
| }, |
| { |
| "epoch": 2.936, |
| "grad_norm": 2.903502941131592, |
| "learning_rate": 6.272028695598065e-09, |
| "logits/chosen": -1.4993667602539062, |
| "logits/rejected": -1.482932209968567, |
| "logps/chosen": -225.55258178710938, |
| "logps/rejected": -322.9141540527344, |
| "loss": 0.1414, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6842883825302124, |
| "rewards/margins": 6.101261615753174, |
| "rewards/rejected": -6.785550117492676, |
| "step": 1835 |
| }, |
| { |
| "epoch": 2.944, |
| "grad_norm": 6.005549907684326, |
| "learning_rate": 4.836007166014179e-09, |
| "logits/chosen": -1.5267841815948486, |
| "logits/rejected": -1.5386269092559814, |
| "logps/chosen": -230.11038208007812, |
| "logps/rejected": -331.4931335449219, |
| "loss": 0.151, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1418485641479492, |
| "rewards/margins": 6.155372619628906, |
| "rewards/rejected": -7.297222137451172, |
| "step": 1840 |
| }, |
| { |
| "epoch": 2.952, |
| "grad_norm": 3.431831121444702, |
| "learning_rate": 3.5862579844697652e-09, |
| "logits/chosen": -1.5528335571289062, |
| "logits/rejected": -1.5695273876190186, |
| "logps/chosen": -237.58261108398438, |
| "logps/rejected": -302.52191162109375, |
| "loss": 0.1869, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.3783166408538818, |
| "rewards/margins": 5.019209861755371, |
| "rewards/rejected": -6.397526264190674, |
| "step": 1845 |
| }, |
| { |
| "epoch": 2.96, |
| "grad_norm": 3.1080453395843506, |
| "learning_rate": 2.522874448926682e-09, |
| "logits/chosen": -1.5317953824996948, |
| "logits/rejected": -1.4420945644378662, |
| "logps/chosen": -249.25967407226562, |
| "logps/rejected": -301.2223815917969, |
| "loss": 0.1357, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8196800947189331, |
| "rewards/margins": 5.4106550216674805, |
| "rewards/rejected": -6.230335235595703, |
| "step": 1850 |
| }, |
| { |
| "epoch": 2.968, |
| "grad_norm": 2.7016687393188477, |
| "learning_rate": 1.645935944526722e-09, |
| "logits/chosen": -1.4864146709442139, |
| "logits/rejected": -1.4250588417053223, |
| "logps/chosen": -239.615478515625, |
| "logps/rejected": -288.51947021484375, |
| "loss": 0.1743, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9490318298339844, |
| "rewards/margins": 5.670099258422852, |
| "rewards/rejected": -6.619131565093994, |
| "step": 1855 |
| }, |
| { |
| "epoch": 2.976, |
| "grad_norm": 3.5190517902374268, |
| "learning_rate": 9.555079376663556e-10, |
| "logits/chosen": -1.4785500764846802, |
| "logits/rejected": -1.411423921585083, |
| "logps/chosen": -247.8288116455078, |
| "logps/rejected": -309.9002990722656, |
| "loss": 0.1295, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.6508281230926514, |
| "rewards/margins": 5.810454368591309, |
| "rewards/rejected": -6.461281776428223, |
| "step": 1860 |
| }, |
| { |
| "epoch": 2.984, |
| "grad_norm": 3.1977896690368652, |
| "learning_rate": 4.5164197110786346e-10, |
| "logits/chosen": -1.5095391273498535, |
| "logits/rejected": -1.4798269271850586, |
| "logps/chosen": -240.112548828125, |
| "logps/rejected": -312.7066650390625, |
| "loss": 0.1405, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9202483892440796, |
| "rewards/margins": 5.801078796386719, |
| "rewards/rejected": -6.7213263511657715, |
| "step": 1865 |
| }, |
| { |
| "epoch": 2.992, |
| "grad_norm": 2.8568241596221924, |
| "learning_rate": 1.3437566013380087e-10, |
| "logits/chosen": -1.5050697326660156, |
| "logits/rejected": -1.49628746509552, |
| "logps/chosen": -222.0975341796875, |
| "logps/rejected": -315.3607482910156, |
| "loss": 0.0995, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.5805636048316956, |
| "rewards/margins": 6.518218994140625, |
| "rewards/rejected": -7.098783016204834, |
| "step": 1870 |
| }, |
| { |
| "epoch": 3.0, |
| "grad_norm": 4.118714809417725, |
| "learning_rate": 3.732689736191119e-12, |
| "logits/chosen": -1.6023108959197998, |
| "logits/rejected": -1.5270345211029053, |
| "logps/chosen": -222.2280731201172, |
| "logps/rejected": -314.62908935546875, |
| "loss": 0.1354, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7832231521606445, |
| "rewards/margins": 5.982799530029297, |
| "rewards/rejected": -6.7660231590271, |
| "step": 1875 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1875, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
