| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1407, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 3.7003118991851807, |
| "learning_rate": 4.651162790697675e-07, |
| "logits/chosen": -1.455744981765747, |
| "logits/rejected": -1.3772783279418945, |
| "logps/chosen": -222.02456665039062, |
| "logps/rejected": -294.1808776855469, |
| "loss": 0.6926, |
| "rewards/accuracies": 0.3687500059604645, |
| "rewards/chosen": -0.0002770518767647445, |
| "rewards/margins": 0.0010710380738601089, |
| "rewards/rejected": -0.001348090241663158, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 3.347348928451538, |
| "learning_rate": 1.0465116279069768e-06, |
| "logits/chosen": -1.360552191734314, |
| "logits/rejected": -1.2121436595916748, |
| "logps/chosen": -243.914306640625, |
| "logps/rejected": -300.3173828125, |
| "loss": 0.6895, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.0027127263601869345, |
| "rewards/margins": 0.007307104766368866, |
| "rewards/rejected": -0.010019831359386444, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 6.566436290740967, |
| "learning_rate": 1.6279069767441862e-06, |
| "logits/chosen": -1.3644325733184814, |
| "logits/rejected": -1.2686518430709839, |
| "logps/chosen": -259.2509460449219, |
| "logps/rejected": -282.755615234375, |
| "loss": 0.6851, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.00618976354598999, |
| "rewards/margins": 0.01635187305510044, |
| "rewards/rejected": -0.02254163660109043, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 3.0041937828063965, |
| "learning_rate": 2.2093023255813954e-06, |
| "logits/chosen": -1.3592755794525146, |
| "logits/rejected": -1.2850176095962524, |
| "logps/chosen": -255.9427947998047, |
| "logps/rejected": -290.93572998046875, |
| "loss": 0.6765, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.008222861215472221, |
| "rewards/margins": 0.034090589731931686, |
| "rewards/rejected": -0.04231344908475876, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 3.1462841033935547, |
| "learning_rate": 2.790697674418605e-06, |
| "logits/chosen": -1.4638659954071045, |
| "logits/rejected": -1.2584081888198853, |
| "logps/chosen": -242.3787384033203, |
| "logps/rejected": -286.44012451171875, |
| "loss": 0.6654, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.015348866581916809, |
| "rewards/margins": 0.057769518345594406, |
| "rewards/rejected": -0.07311838120222092, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 3.3971853256225586, |
| "learning_rate": 3.372093023255814e-06, |
| "logits/chosen": -1.4179127216339111, |
| "logits/rejected": -1.2499792575836182, |
| "logps/chosen": -258.70672607421875, |
| "logps/rejected": -291.22808837890625, |
| "loss": 0.6444, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -0.02679530717432499, |
| "rewards/margins": 0.10471153259277344, |
| "rewards/rejected": -0.13150686025619507, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 3.0295073986053467, |
| "learning_rate": 3.953488372093024e-06, |
| "logits/chosen": -1.508451223373413, |
| "logits/rejected": -1.3404954671859741, |
| "logps/chosen": -245.9871368408203, |
| "logps/rejected": -296.29498291015625, |
| "loss": 0.6181, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.03217829018831253, |
| "rewards/margins": 0.16509227454662323, |
| "rewards/rejected": -0.19727057218551636, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 2.999821901321411, |
| "learning_rate": 4.5348837209302326e-06, |
| "logits/chosen": -1.4148328304290771, |
| "logits/rejected": -1.3169008493423462, |
| "logps/chosen": -231.54904174804688, |
| "logps/rejected": -299.49896240234375, |
| "loss": 0.5936, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.0469132661819458, |
| "rewards/margins": 0.22682125866413116, |
| "rewards/rejected": -0.27373450994491577, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 3.179332733154297, |
| "learning_rate": 4.99999336897255e-06, |
| "logits/chosen": -1.4154040813446045, |
| "logits/rejected": -1.2963824272155762, |
| "logps/chosen": -250.27114868164062, |
| "logps/rejected": -313.90899658203125, |
| "loss": 0.5755, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.12327556312084198, |
| "rewards/margins": 0.2791060507297516, |
| "rewards/rejected": -0.4023815989494324, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 2.8430349826812744, |
| "learning_rate": 4.999761286705285e-06, |
| "logits/chosen": -1.5061628818511963, |
| "logits/rejected": -1.407204270362854, |
| "logps/chosen": -268.7781677246094, |
| "logps/rejected": -301.4381408691406, |
| "loss": 0.5144, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.1702246367931366, |
| "rewards/margins": 0.46079692244529724, |
| "rewards/rejected": -0.6310215592384338, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 2.7913994789123535, |
| "learning_rate": 4.999197688241076e-06, |
| "logits/chosen": -1.4658788442611694, |
| "logits/rejected": -1.3267757892608643, |
| "logps/chosen": -233.3777313232422, |
| "logps/rejected": -285.8099670410156, |
| "loss": 0.5025, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.16926373541355133, |
| "rewards/margins": 0.5302549600601196, |
| "rewards/rejected": -0.6995187401771545, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 2.3396801948547363, |
| "learning_rate": 4.9983026483238685e-06, |
| "logits/chosen": -1.5207116603851318, |
| "logits/rejected": -1.4026588201522827, |
| "logps/chosen": -243.4673309326172, |
| "logps/rejected": -300.88372802734375, |
| "loss": 0.4772, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.31207606196403503, |
| "rewards/margins": 0.6151299476623535, |
| "rewards/rejected": -0.9272059202194214, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 2.555133819580078, |
| "learning_rate": 4.997076285653087e-06, |
| "logits/chosen": -1.5522791147232056, |
| "logits/rejected": -1.3812358379364014, |
| "logps/chosen": -273.4674987792969, |
| "logps/rejected": -318.3159484863281, |
| "loss": 0.423, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.3257208466529846, |
| "rewards/margins": 0.8692599534988403, |
| "rewards/rejected": -1.1949808597564697, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 3.6287903785705566, |
| "learning_rate": 4.995518762867897e-06, |
| "logits/chosen": -1.6529529094696045, |
| "logits/rejected": -1.3509896993637085, |
| "logps/chosen": -231.0438232421875, |
| "logps/rejected": -295.55517578125, |
| "loss": 0.4277, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.40942874550819397, |
| "rewards/margins": 0.8625381588935852, |
| "rewards/rejected": -1.2719669342041016, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 3.1857149600982666, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -1.5783509016036987, |
| "logits/rejected": -1.4265633821487427, |
| "logps/chosen": -243.6620635986328, |
| "logps/rejected": -316.5206604003906, |
| "loss": 0.3824, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.5047813653945923, |
| "rewards/margins": 1.0900554656982422, |
| "rewards/rejected": -1.594836950302124, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 3.0437979698181152, |
| "learning_rate": 4.99141110707441e-06, |
| "logits/chosen": -1.6324541568756104, |
| "logits/rejected": -1.4887969493865967, |
| "logps/chosen": -243.0408172607422, |
| "logps/rejected": -290.43109130859375, |
| "loss": 0.4195, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.4802798628807068, |
| "rewards/margins": 0.9819287061691284, |
| "rewards/rejected": -1.4622085094451904, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 2.822033166885376, |
| "learning_rate": 4.9888615188199e-06, |
| "logits/chosen": -1.6993844509124756, |
| "logits/rejected": -1.4935144186019897, |
| "logps/chosen": -262.30218505859375, |
| "logps/rejected": -311.645263671875, |
| "loss": 0.3555, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.5132590532302856, |
| "rewards/margins": 1.3637869358062744, |
| "rewards/rejected": -1.8770462274551392, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 2.7532804012298584, |
| "learning_rate": 4.9859818598863104e-06, |
| "logits/chosen": -1.6364755630493164, |
| "logits/rejected": -1.4478092193603516, |
| "logps/chosen": -241.4347686767578, |
| "logps/rejected": -297.4327087402344, |
| "loss": 0.4077, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.6576055288314819, |
| "rewards/margins": 1.298345685005188, |
| "rewards/rejected": -1.9559510946273804, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 2.444952964782715, |
| "learning_rate": 4.982772512171538e-06, |
| "logits/chosen": -1.601289987564087, |
| "logits/rejected": -1.5096111297607422, |
| "logps/chosen": -233.38088989257812, |
| "logps/rejected": -311.9754333496094, |
| "loss": 0.3462, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.654367208480835, |
| "rewards/margins": 1.4337469339370728, |
| "rewards/rejected": -2.0881142616271973, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 3.0096845626831055, |
| "learning_rate": 4.979233901296523e-06, |
| "logits/chosen": -1.6930478811264038, |
| "logits/rejected": -1.5883949995040894, |
| "logps/chosen": -241.365966796875, |
| "logps/rejected": -299.60540771484375, |
| "loss": 0.2976, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.5582109689712524, |
| "rewards/margins": 1.7530122995376587, |
| "rewards/rejected": -2.311223030090332, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 2.8001527786254883, |
| "learning_rate": 4.975366496548804e-06, |
| "logits/chosen": -1.6890506744384766, |
| "logits/rejected": -1.476367712020874, |
| "logps/chosen": -253.2794647216797, |
| "logps/rejected": -304.4421691894531, |
| "loss": 0.3065, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.6882883906364441, |
| "rewards/margins": 1.7531687021255493, |
| "rewards/rejected": -2.4414572715759277, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 2.144157648086548, |
| "learning_rate": 4.971170810820279e-06, |
| "logits/chosen": -1.6170241832733154, |
| "logits/rejected": -1.5461595058441162, |
| "logps/chosen": -248.46035766601562, |
| "logps/rejected": -327.1305236816406, |
| "loss": 0.2433, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.5820830464363098, |
| "rewards/margins": 2.0067741870880127, |
| "rewards/rejected": -2.5888569355010986, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 3.336944103240967, |
| "learning_rate": 4.96664740053919e-06, |
| "logits/chosen": -1.6426379680633545, |
| "logits/rejected": -1.5436229705810547, |
| "logps/chosen": -260.5669860839844, |
| "logps/rejected": -313.0763854980469, |
| "loss": 0.3159, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.7669323682785034, |
| "rewards/margins": 1.7966525554656982, |
| "rewards/rejected": -2.563584804534912, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 2.8522286415100098, |
| "learning_rate": 4.961796865596328e-06, |
| "logits/chosen": -1.6780723333358765, |
| "logits/rejected": -1.5482378005981445, |
| "logps/chosen": -230.629150390625, |
| "logps/rejected": -312.26068115234375, |
| "loss": 0.329, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.5399895310401917, |
| "rewards/margins": 1.8903239965438843, |
| "rewards/rejected": -2.4303135871887207, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 2.774472236633301, |
| "learning_rate": 4.956619849265473e-06, |
| "logits/chosen": -1.7566455602645874, |
| "logits/rejected": -1.535705327987671, |
| "logps/chosen": -238.42520141601562, |
| "logps/rejected": -296.03424072265625, |
| "loss": 0.2868, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.4130798280239105, |
| "rewards/margins": 2.085447072982788, |
| "rewards/rejected": -2.4985270500183105, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 2.236811876296997, |
| "learning_rate": 4.951117038118091e-06, |
| "logits/chosen": -1.7088298797607422, |
| "logits/rejected": -1.5699352025985718, |
| "logps/chosen": -240.57534790039062, |
| "logps/rejected": -314.7195739746094, |
| "loss": 0.2552, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.4095377027988434, |
| "rewards/margins": 2.295804977416992, |
| "rewards/rejected": -2.7053427696228027, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 2.135939121246338, |
| "learning_rate": 4.945289161932273e-06, |
| "logits/chosen": -1.747797966003418, |
| "logits/rejected": -1.6278316974639893, |
| "logps/chosen": -245.04721069335938, |
| "logps/rejected": -334.56060791015625, |
| "loss": 0.1957, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.33136236667633057, |
| "rewards/margins": 2.5914673805236816, |
| "rewards/rejected": -2.9228296279907227, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 4.112620830535889, |
| "learning_rate": 4.93913699359596e-06, |
| "logits/chosen": -1.7526079416275024, |
| "logits/rejected": -1.58306086063385, |
| "logps/chosen": -238.26718139648438, |
| "logps/rejected": -328.44317626953125, |
| "loss": 0.2467, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.6260155439376831, |
| "rewards/margins": 2.4649341106414795, |
| "rewards/rejected": -3.090949535369873, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 3.598618507385254, |
| "learning_rate": 4.9326613490044364e-06, |
| "logits/chosen": -1.7870292663574219, |
| "logits/rejected": -1.6332266330718994, |
| "logps/chosen": -234.255126953125, |
| "logps/rejected": -304.899658203125, |
| "loss": 0.3035, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.6351826786994934, |
| "rewards/margins": 2.2837603092193604, |
| "rewards/rejected": -2.918942928314209, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 2.3492329120635986, |
| "learning_rate": 4.925863086952132e-06, |
| "logits/chosen": -1.805809736251831, |
| "logits/rejected": -1.6427209377288818, |
| "logps/chosen": -250.2255401611328, |
| "logps/rejected": -336.9315490722656, |
| "loss": 0.2122, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.4687698483467102, |
| "rewards/margins": 2.872528076171875, |
| "rewards/rejected": -3.3412978649139404, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 3.2796857357025146, |
| "learning_rate": 4.918743109018727e-06, |
| "logits/chosen": -1.870831847190857, |
| "logits/rejected": -1.696987509727478, |
| "logps/chosen": -249.03280639648438, |
| "logps/rejected": -338.06231689453125, |
| "loss": 0.2102, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.7773758769035339, |
| "rewards/margins": 2.9179739952087402, |
| "rewards/rejected": -3.695349931716919, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 3.265937089920044, |
| "learning_rate": 4.911302359449585e-06, |
| "logits/chosen": -1.862051010131836, |
| "logits/rejected": -1.6175568103790283, |
| "logps/chosen": -245.9069061279297, |
| "logps/rejected": -347.184326171875, |
| "loss": 0.1894, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.8534000515937805, |
| "rewards/margins": 3.094893455505371, |
| "rewards/rejected": -3.948293685913086, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 4.633449077606201, |
| "learning_rate": 4.9035418250305314e-06, |
| "logits/chosen": -1.7832374572753906, |
| "logits/rejected": -1.5434472560882568, |
| "logps/chosen": -257.94879150390625, |
| "logps/rejected": -334.8266296386719, |
| "loss": 0.2446, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.8249349594116211, |
| "rewards/margins": 2.9752113819122314, |
| "rewards/rejected": -3.8001461029052734, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 2.3466925621032715, |
| "learning_rate": 4.895462534956981e-06, |
| "logits/chosen": -1.7460002899169922, |
| "logits/rejected": -1.6494872570037842, |
| "logps/chosen": -239.1138153076172, |
| "logps/rejected": -327.669921875, |
| "loss": 0.2073, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.773691713809967, |
| "rewards/margins": 3.2550785541534424, |
| "rewards/rejected": -4.028769493103027, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 3.3231382369995117, |
| "learning_rate": 4.887065560697449e-06, |
| "logits/chosen": -1.7860323190689087, |
| "logits/rejected": -1.664729356765747, |
| "logps/chosen": -262.41534423828125, |
| "logps/rejected": -338.5781555175781, |
| "loss": 0.2815, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.7041072249412537, |
| "rewards/margins": 2.8621432781219482, |
| "rewards/rejected": -3.5662505626678467, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 1.6720951795578003, |
| "learning_rate": 4.878352015851459e-06, |
| "logits/chosen": -1.7511796951293945, |
| "logits/rejected": -1.5772809982299805, |
| "logps/chosen": -259.29962158203125, |
| "logps/rejected": -330.5465087890625, |
| "loss": 0.163, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5976866483688354, |
| "rewards/margins": 3.216883420944214, |
| "rewards/rejected": -3.814570188522339, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 4.071167469024658, |
| "learning_rate": 4.869323056001852e-06, |
| "logits/chosen": -1.7100389003753662, |
| "logits/rejected": -1.5902552604675293, |
| "logps/chosen": -238.5122833251953, |
| "logps/rejected": -319.55145263671875, |
| "loss": 0.2501, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.7488561868667603, |
| "rewards/margins": 2.900449514389038, |
| "rewards/rejected": -3.649306058883667, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 2.2532527446746826, |
| "learning_rate": 4.859979878561536e-06, |
| "logits/chosen": -1.741315245628357, |
| "logits/rejected": -1.579534888267517, |
| "logps/chosen": -246.63308715820312, |
| "logps/rejected": -334.5790710449219, |
| "loss": 0.1882, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.4792085289955139, |
| "rewards/margins": 3.239213466644287, |
| "rewards/rejected": -3.7184219360351562, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 5.985015869140625, |
| "learning_rate": 4.850323722614685e-06, |
| "logits/chosen": -1.751989722251892, |
| "logits/rejected": -1.5852595567703247, |
| "logps/chosen": -254.85635375976562, |
| "logps/rejected": -345.94757080078125, |
| "loss": 0.2467, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.5594536066055298, |
| "rewards/margins": 3.1026852130889893, |
| "rewards/rejected": -3.6621384620666504, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 4.524343967437744, |
| "learning_rate": 4.840355868752416e-06, |
| "logits/chosen": -1.688010811805725, |
| "logits/rejected": -1.5757395029067993, |
| "logps/chosen": -293.441162109375, |
| "logps/rejected": -335.3061828613281, |
| "loss": 0.285, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.47422847151756287, |
| "rewards/margins": 2.8011474609375, |
| "rewards/rejected": -3.2753758430480957, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 1.6691768169403076, |
| "learning_rate": 4.830077638902955e-06, |
| "logits/chosen": -1.8881086111068726, |
| "logits/rejected": -1.6681621074676514, |
| "logps/chosen": -265.43914794921875, |
| "logps/rejected": -306.125, |
| "loss": 0.2008, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.5452998876571655, |
| "rewards/margins": 3.2147326469421387, |
| "rewards/rejected": -3.7600326538085938, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 3.2968130111694336, |
| "learning_rate": 4.8194903961563255e-06, |
| "logits/chosen": -1.8698129653930664, |
| "logits/rejected": -1.733386754989624, |
| "logps/chosen": -236.70834350585938, |
| "logps/rejected": -325.7504577636719, |
| "loss": 0.2263, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.6137224435806274, |
| "rewards/margins": 3.213418960571289, |
| "rewards/rejected": -3.827141284942627, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 1.8006588220596313, |
| "learning_rate": 4.808595544583578e-06, |
| "logits/chosen": -1.7925153970718384, |
| "logits/rejected": -1.62285578250885, |
| "logps/chosen": -244.73388671875, |
| "logps/rejected": -338.7201232910156, |
| "loss": 0.1987, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.5871314406394958, |
| "rewards/margins": 3.5609264373779297, |
| "rewards/rejected": -4.14805793762207, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 2.2668755054473877, |
| "learning_rate": 4.797394529050577e-06, |
| "logits/chosen": -1.8083196878433228, |
| "logits/rejected": -1.6195793151855469, |
| "logps/chosen": -254.16452026367188, |
| "logps/rejected": -340.5189514160156, |
| "loss": 0.1648, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.307282954454422, |
| "rewards/margins": 3.5338077545166016, |
| "rewards/rejected": -3.841090440750122, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 2.5740833282470703, |
| "learning_rate": 4.785888835026393e-06, |
| "logits/chosen": -1.737634301185608, |
| "logits/rejected": -1.5834619998931885, |
| "logps/chosen": -234.80917358398438, |
| "logps/rejected": -308.63641357421875, |
| "loss": 0.2302, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.40825945138931274, |
| "rewards/margins": 3.2277324199676514, |
| "rewards/rejected": -3.6359925270080566, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 2.8291969299316406, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -1.7801179885864258, |
| "logits/rejected": -1.6636883020401, |
| "logps/chosen": -237.48291015625, |
| "logps/rejected": -349.30694580078125, |
| "loss": 0.1692, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.503801167011261, |
| "rewards/margins": 3.6341793537139893, |
| "rewards/rejected": -4.1379804611206055, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 1.6978325843811035, |
| "learning_rate": 4.761969555209394e-06, |
| "logits/chosen": -1.79836905002594, |
| "logits/rejected": -1.7076466083526611, |
| "logps/chosen": -229.66592407226562, |
| "logps/rejected": -336.6689758300781, |
| "loss": 0.1825, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.48729997873306274, |
| "rewards/margins": 3.8662896156311035, |
| "rewards/rejected": -4.353589057922363, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 2.3878140449523926, |
| "learning_rate": 4.7495591415709435e-06, |
| "logits/chosen": -1.8353424072265625, |
| "logits/rejected": -1.6892439126968384, |
| "logps/chosen": -265.48736572265625, |
| "logps/rejected": -356.034912109375, |
| "loss": 0.1745, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.8327113389968872, |
| "rewards/margins": 3.900333881378174, |
| "rewards/rejected": -4.73304557800293, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 2.2849137783050537, |
| "learning_rate": 4.736850393329353e-06, |
| "logits/chosen": -1.8372564315795898, |
| "logits/rejected": -1.6298954486846924, |
| "logps/chosen": -255.60238647460938, |
| "logps/rejected": -361.90948486328125, |
| "loss": 0.1689, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.9175726771354675, |
| "rewards/margins": 4.157658576965332, |
| "rewards/rejected": -5.075230598449707, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 3.430100917816162, |
| "learning_rate": 4.7238449959079095e-06, |
| "logits/chosen": -1.8621412515640259, |
| "logits/rejected": -1.753686547279358, |
| "logps/chosen": -237.799072265625, |
| "logps/rejected": -339.875732421875, |
| "loss": 0.1646, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.8456540107727051, |
| "rewards/margins": 4.005865573883057, |
| "rewards/rejected": -4.851519584655762, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 3.102550506591797, |
| "learning_rate": 4.710544674071259e-06, |
| "logits/chosen": -1.8992693424224854, |
| "logits/rejected": -1.7382304668426514, |
| "logps/chosen": -244.0569305419922, |
| "logps/rejected": -320.05047607421875, |
| "loss": 0.181, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7346665263175964, |
| "rewards/margins": 4.076887607574463, |
| "rewards/rejected": -4.811553955078125, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 4.898008346557617, |
| "learning_rate": 4.696951191696673e-06, |
| "logits/chosen": -1.8656402826309204, |
| "logits/rejected": -1.6837927103042603, |
| "logps/chosen": -246.1433868408203, |
| "logps/rejected": -354.2840881347656, |
| "loss": 0.2213, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.977743923664093, |
| "rewards/margins": 4.141406536102295, |
| "rewards/rejected": -5.119150638580322, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 2.6836533546447754, |
| "learning_rate": 4.6830663515401196e-06, |
| "logits/chosen": -1.8147302865982056, |
| "logits/rejected": -1.6920936107635498, |
| "logps/chosen": -246.29776000976562, |
| "logps/rejected": -344.7865295410156, |
| "loss": 0.2436, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.7966432571411133, |
| "rewards/margins": 3.7020270824432373, |
| "rewards/rejected": -4.4986701011657715, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 4.104752063751221, |
| "learning_rate": 4.6688919949971855e-06, |
| "logits/chosen": -1.8584072589874268, |
| "logits/rejected": -1.6018040180206299, |
| "logps/chosen": -246.09945678710938, |
| "logps/rejected": -320.53314208984375, |
| "loss": 0.2292, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.820715606212616, |
| "rewards/margins": 3.767307996749878, |
| "rewards/rejected": -4.588023662567139, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 2.240436553955078, |
| "learning_rate": 4.654430001858874e-06, |
| "logits/chosen": -1.672431230545044, |
| "logits/rejected": -1.5979852676391602, |
| "logps/chosen": -249.657470703125, |
| "logps/rejected": -345.25433349609375, |
| "loss": 0.199, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5963641405105591, |
| "rewards/margins": 3.945819139480591, |
| "rewards/rejected": -4.5421833992004395, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 3.5626108646392822, |
| "learning_rate": 4.6396822900623075e-06, |
| "logits/chosen": -1.8363211154937744, |
| "logits/rejected": -1.6771224737167358, |
| "logps/chosen": -258.85400390625, |
| "logps/rejected": -342.59814453125, |
| "loss": 0.1781, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.793243408203125, |
| "rewards/margins": 4.015771865844727, |
| "rewards/rejected": -4.809015274047852, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 2.846989870071411, |
| "learning_rate": 4.624650815436368e-06, |
| "logits/chosen": -1.8874149322509766, |
| "logits/rejected": -1.7275638580322266, |
| "logps/chosen": -266.51983642578125, |
| "logps/rejected": -327.6214599609375, |
| "loss": 0.2161, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.5533876419067383, |
| "rewards/margins": 3.9066085815429688, |
| "rewards/rejected": -4.459996223449707, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 2.3967525959014893, |
| "learning_rate": 4.609337571442324e-06, |
| "logits/chosen": -1.8907324075698853, |
| "logits/rejected": -1.7417322397232056, |
| "logps/chosen": -232.366943359375, |
| "logps/rejected": -336.410400390625, |
| "loss": 0.1725, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7537919878959656, |
| "rewards/margins": 3.960203170776367, |
| "rewards/rejected": -4.713995456695557, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 2.485433578491211, |
| "learning_rate": 4.5937445889094535e-06, |
| "logits/chosen": -1.7409560680389404, |
| "logits/rejected": -1.6088342666625977, |
| "logps/chosen": -268.9187927246094, |
| "logps/rejected": -331.84161376953125, |
| "loss": 0.1863, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.6697766780853271, |
| "rewards/margins": 4.0562920570373535, |
| "rewards/rejected": -4.72606897354126, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 3.9823858737945557, |
| "learning_rate": 4.577873935765722e-06, |
| "logits/chosen": -1.893109917640686, |
| "logits/rejected": -1.7313368320465088, |
| "logps/chosen": -240.6698455810547, |
| "logps/rejected": -339.091552734375, |
| "loss": 0.1527, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.630599856376648, |
| "rewards/margins": 4.312032222747803, |
| "rewards/rejected": -4.942632675170898, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 3.2793948650360107, |
| "learning_rate": 4.561727716763532e-06, |
| "logits/chosen": -1.7743825912475586, |
| "logits/rejected": -1.70479416847229, |
| "logps/chosen": -263.1169738769531, |
| "logps/rejected": -332.9526062011719, |
| "loss": 0.1769, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.5394365787506104, |
| "rewards/margins": 4.297283172607422, |
| "rewards/rejected": -4.836719036102295, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 5.001033306121826, |
| "learning_rate": 4.545308073200591e-06, |
| "logits/chosen": -1.7639503479003906, |
| "logits/rejected": -1.6067688465118408, |
| "logps/chosen": -255.49417114257812, |
| "logps/rejected": -350.12554931640625, |
| "loss": 0.1838, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.6203233003616333, |
| "rewards/margins": 4.100062370300293, |
| "rewards/rejected": -4.720385551452637, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 2.9403929710388184, |
| "learning_rate": 4.528617182635943e-06, |
| "logits/chosen": -1.8818168640136719, |
| "logits/rejected": -1.7447856664657593, |
| "logps/chosen": -251.169921875, |
| "logps/rejected": -358.80645751953125, |
| "loss": 0.1575, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9520936012268066, |
| "rewards/margins": 4.413204193115234, |
| "rewards/rejected": -5.365297317504883, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 3.748673439025879, |
| "learning_rate": 4.511657258601171e-06, |
| "logits/chosen": -1.8479007482528687, |
| "logits/rejected": -1.6725494861602783, |
| "logps/chosen": -257.38983154296875, |
| "logps/rejected": -335.9569396972656, |
| "loss": 0.1584, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9364475011825562, |
| "rewards/margins": 4.432442665100098, |
| "rewards/rejected": -5.368889808654785, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 3.777651309967041, |
| "learning_rate": 4.494430550306848e-06, |
| "logits/chosen": -1.757094383239746, |
| "logits/rejected": -1.6897532939910889, |
| "logps/chosen": -263.840087890625, |
| "logps/rejected": -342.526611328125, |
| "loss": 0.1966, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5220140218734741, |
| "rewards/margins": 4.274970054626465, |
| "rewards/rejected": -4.796984672546387, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 2.858718156814575, |
| "learning_rate": 4.476939342344246e-06, |
| "logits/chosen": -1.8669666051864624, |
| "logits/rejected": -1.7618488073349, |
| "logps/chosen": -231.40560913085938, |
| "logps/rejected": -309.1484375, |
| "loss": 0.191, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.6370632648468018, |
| "rewards/margins": 3.9954731464385986, |
| "rewards/rejected": -4.6325364112854, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 1.4209452867507935, |
| "learning_rate": 4.45918595438236e-06, |
| "logits/chosen": -1.7774779796600342, |
| "logits/rejected": -1.7063461542129517, |
| "logps/chosen": -253.4152069091797, |
| "logps/rejected": -339.8118896484375, |
| "loss": 0.2015, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.38608410954475403, |
| "rewards/margins": 3.99464750289917, |
| "rewards/rejected": -4.380731105804443, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 1.5503495931625366, |
| "learning_rate": 4.441172740860267e-06, |
| "logits/chosen": -1.767425298690796, |
| "logits/rejected": -1.6601545810699463, |
| "logps/chosen": -263.5538024902344, |
| "logps/rejected": -362.27203369140625, |
| "loss": 0.1889, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.3973921835422516, |
| "rewards/margins": 4.149497032165527, |
| "rewards/rejected": -4.546889781951904, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 2.47979998588562, |
| "learning_rate": 4.422902090674886e-06, |
| "logits/chosen": -1.7320607900619507, |
| "logits/rejected": -1.5309160947799683, |
| "logps/chosen": -260.5031433105469, |
| "logps/rejected": -330.44012451171875, |
| "loss": 0.1594, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.32683077454566956, |
| "rewards/margins": 3.9723198413848877, |
| "rewards/rejected": -4.299150466918945, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 2.5703437328338623, |
| "learning_rate": 4.4043764268641705e-06, |
| "logits/chosen": -1.7610089778900146, |
| "logits/rejected": -1.6328073740005493, |
| "logps/chosen": -271.28472900390625, |
| "logps/rejected": -344.73504638671875, |
| "loss": 0.2003, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.8157901763916016, |
| "rewards/margins": 4.135244846343994, |
| "rewards/rejected": -4.9510345458984375, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 3.8566060066223145, |
| "learning_rate": 4.385598206285758e-06, |
| "logits/chosen": -1.927005410194397, |
| "logits/rejected": -1.8474489450454712, |
| "logps/chosen": -237.0289306640625, |
| "logps/rejected": -356.33978271484375, |
| "loss": 0.1774, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.3049725592136383, |
| "rewards/margins": 4.615744590759277, |
| "rewards/rejected": -4.920717716217041, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 3.5054588317871094, |
| "learning_rate": 4.3665699192911495e-06, |
| "logits/chosen": -1.807989478111267, |
| "logits/rejected": -1.707664132118225, |
| "logps/chosen": -243.15560913085938, |
| "logps/rejected": -323.3631896972656, |
| "loss": 0.1703, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5694507956504822, |
| "rewards/margins": 3.8626322746276855, |
| "rewards/rejected": -4.4320831298828125, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 2.2553398609161377, |
| "learning_rate": 4.347294089395438e-06, |
| "logits/chosen": -1.8030427694320679, |
| "logits/rejected": -1.7392162084579468, |
| "logps/chosen": -276.47113037109375, |
| "logps/rejected": -363.3248291015625, |
| "loss": 0.1399, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.549434244632721, |
| "rewards/margins": 4.955690383911133, |
| "rewards/rejected": -5.505124092102051, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 3.9264116287231445, |
| "learning_rate": 4.327773272942647e-06, |
| "logits/chosen": -1.831888198852539, |
| "logits/rejected": -1.679851770401001, |
| "logps/chosen": -241.8992156982422, |
| "logps/rejected": -336.3548278808594, |
| "loss": 0.2017, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.47854313254356384, |
| "rewards/margins": 4.543624401092529, |
| "rewards/rejected": -5.022168159484863, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 3.0306379795074463, |
| "learning_rate": 4.308010058766703e-06, |
| "logits/chosen": -1.8648799657821655, |
| "logits/rejected": -1.7884979248046875, |
| "logps/chosen": -247.47683715820312, |
| "logps/rejected": -332.400146484375, |
| "loss": 0.1547, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6630781292915344, |
| "rewards/margins": 4.304313659667969, |
| "rewards/rejected": -4.967392444610596, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 4.7305145263671875, |
| "learning_rate": 4.2880070678481135e-06, |
| "logits/chosen": -1.8011143207550049, |
| "logits/rejected": -1.725319266319275, |
| "logps/chosen": -253.3076171875, |
| "logps/rejected": -360.08831787109375, |
| "loss": 0.1978, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -1.0798097848892212, |
| "rewards/margins": 4.478891849517822, |
| "rewards/rejected": -5.558701515197754, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 3.582191228866577, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.6858046054840088, |
| "logits/rejected": -1.5931075811386108, |
| "logps/chosen": -261.5023498535156, |
| "logps/rejected": -340.2249450683594, |
| "loss": 0.1993, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9488990902900696, |
| "rewards/margins": 4.209158897399902, |
| "rewards/rejected": -5.158058166503906, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 2.397202730178833, |
| "learning_rate": 4.24729239834814e-06, |
| "logits/chosen": -1.8469274044036865, |
| "logits/rejected": -1.6774814128875732, |
| "logps/chosen": -242.9910888671875, |
| "logps/rejected": -323.50433349609375, |
| "loss": 0.19, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.6384455561637878, |
| "rewards/margins": 4.31686544418335, |
| "rewards/rejected": -4.9553117752075195, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 2.013122081756592, |
| "learning_rate": 4.226586119311291e-06, |
| "logits/chosen": -1.8408855199813843, |
| "logits/rejected": -1.7532079219818115, |
| "logps/chosen": -255.99557495117188, |
| "logps/rejected": -339.9315185546875, |
| "loss": 0.1723, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5134051442146301, |
| "rewards/margins": 4.4296488761901855, |
| "rewards/rejected": -4.94305419921875, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 2.0674853324890137, |
| "learning_rate": 4.205650861904782e-06, |
| "logits/chosen": -1.8111393451690674, |
| "logits/rejected": -1.7262649536132812, |
| "logps/chosen": -218.2287139892578, |
| "logps/rejected": -331.74493408203125, |
| "loss": 0.2054, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.650185763835907, |
| "rewards/margins": 4.551926612854004, |
| "rewards/rejected": -5.202112674713135, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 3.9873046875, |
| "learning_rate": 4.184489402544486e-06, |
| "logits/chosen": -1.906991720199585, |
| "logits/rejected": -1.7338151931762695, |
| "logps/chosen": -248.287109375, |
| "logps/rejected": -324.5978088378906, |
| "loss": 0.2191, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.7245001196861267, |
| "rewards/margins": 4.4610748291015625, |
| "rewards/rejected": -5.185575485229492, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 3.678515672683716, |
| "learning_rate": 4.1631045476449865e-06, |
| "logits/chosen": -1.8973890542984009, |
| "logits/rejected": -1.7549943923950195, |
| "logps/chosen": -232.38192749023438, |
| "logps/rejected": -343.0477294921875, |
| "loss": 0.1735, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.3311890959739685, |
| "rewards/margins": 4.941264629364014, |
| "rewards/rejected": -5.272453308105469, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 3.474243402481079, |
| "learning_rate": 4.141499133247391e-06, |
| "logits/chosen": -1.747066855430603, |
| "logits/rejected": -1.6317088603973389, |
| "logps/chosen": -238.07205200195312, |
| "logps/rejected": -329.851318359375, |
| "loss": 0.1708, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.11001911014318466, |
| "rewards/margins": 4.652035713195801, |
| "rewards/rejected": -4.762054920196533, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 3.526038885116577, |
| "learning_rate": 4.119676024643217e-06, |
| "logits/chosen": -1.7861535549163818, |
| "logits/rejected": -1.6475368738174438, |
| "logps/chosen": -242.25662231445312, |
| "logps/rejected": -335.34637451171875, |
| "loss": 0.1491, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.10151362419128418, |
| "rewards/margins": 4.927655220031738, |
| "rewards/rejected": -5.029168128967285, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 4.218128681182861, |
| "learning_rate": 4.0976381159944055e-06, |
| "logits/chosen": -1.8683736324310303, |
| "logits/rejected": -1.7600526809692383, |
| "logps/chosen": -248.19558715820312, |
| "logps/rejected": -363.2998046875, |
| "loss": 0.1956, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.1459302008152008, |
| "rewards/margins": 4.895400047302246, |
| "rewards/rejected": -5.041329383850098, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 2.3034024238586426, |
| "learning_rate": 4.07538832994949e-06, |
| "logits/chosen": -1.8572527170181274, |
| "logits/rejected": -1.6589491367340088, |
| "logps/chosen": -243.2412109375, |
| "logps/rejected": -366.75897216796875, |
| "loss": 0.1447, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.39440393447875977, |
| "rewards/margins": 5.091270446777344, |
| "rewards/rejected": -5.4856743812561035, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 3.796588897705078, |
| "learning_rate": 4.0529296172560055e-06, |
| "logits/chosen": -1.8980152606964111, |
| "logits/rejected": -1.6751664876937866, |
| "logps/chosen": -242.3210906982422, |
| "logps/rejected": -351.9512634277344, |
| "loss": 0.1234, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.45586246252059937, |
| "rewards/margins": 5.15492582321167, |
| "rewards/rejected": -5.6107892990112305, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 3.219355344772339, |
| "learning_rate": 4.030264956369158e-06, |
| "logits/chosen": -1.819602608680725, |
| "logits/rejected": -1.7250854969024658, |
| "logps/chosen": -240.2069549560547, |
| "logps/rejected": -333.0482482910156, |
| "loss": 0.1515, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6956228017807007, |
| "rewards/margins": 4.75858211517334, |
| "rewards/rejected": -5.454204559326172, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 3.2204010486602783, |
| "learning_rate": 4.0073973530568274e-06, |
| "logits/chosen": -1.8022804260253906, |
| "logits/rejected": -1.6240564584732056, |
| "logps/chosen": -263.3968811035156, |
| "logps/rejected": -343.99139404296875, |
| "loss": 0.1268, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5587592720985413, |
| "rewards/margins": 5.209686279296875, |
| "rewards/rejected": -5.7684454917907715, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 2.604736089706421, |
| "learning_rate": 3.9843298400009435e-06, |
| "logits/chosen": -1.856161117553711, |
| "logits/rejected": -1.790837049484253, |
| "logps/chosen": -248.9732666015625, |
| "logps/rejected": -353.9434814453125, |
| "loss": 0.1207, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.5389277935028076, |
| "rewards/margins": 5.065027713775635, |
| "rewards/rejected": -5.603955268859863, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 3.975954532623291, |
| "learning_rate": 3.961065476395296e-06, |
| "logits/chosen": -1.830162763595581, |
| "logits/rejected": -1.6826388835906982, |
| "logps/chosen": -268.0751953125, |
| "logps/rejected": -363.59967041015625, |
| "loss": 0.1978, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.8818696737289429, |
| "rewards/margins": 5.387181758880615, |
| "rewards/rejected": -6.269051551818848, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 4.976337432861328, |
| "learning_rate": 3.937607347539823e-06, |
| "logits/chosen": -1.814556360244751, |
| "logits/rejected": -1.7047951221466064, |
| "logps/chosen": -282.71575927734375, |
| "logps/rejected": -367.3832092285156, |
| "loss": 0.148, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.1829640865325928, |
| "rewards/margins": 4.825137138366699, |
| "rewards/rejected": -6.008100986480713, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 2.259762763977051, |
| "learning_rate": 3.913958564431447e-06, |
| "logits/chosen": -1.8797647953033447, |
| "logits/rejected": -1.752611756324768, |
| "logps/chosen": -252.342041015625, |
| "logps/rejected": -379.76861572265625, |
| "loss": 0.1244, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.859488844871521, |
| "rewards/margins": 6.077948570251465, |
| "rewards/rejected": -6.937437534332275, |
| "step": 465 |
| }, |
| { |
| "epoch": 1.0021333333333333, |
| "grad_norm": 1.6464236974716187, |
| "learning_rate": 3.89012226335149e-06, |
| "logits/chosen": -1.7027655839920044, |
| "logits/rejected": -1.6129655838012695, |
| "logps/chosen": -256.5233459472656, |
| "logps/rejected": -357.3712158203125, |
| "loss": 0.0864, |
| "rewards/accuracies": 0.9736841917037964, |
| "rewards/chosen": -0.9277030825614929, |
| "rewards/margins": 5.774387836456299, |
| "rewards/rejected": -6.702090740203857, |
| "step": 470 |
| }, |
| { |
| "epoch": 1.0128, |
| "grad_norm": 1.6793190240859985, |
| "learning_rate": 3.86610160544975e-06, |
| "logits/chosen": -1.8415664434432983, |
| "logits/rejected": -1.6897361278533936, |
| "logps/chosen": -247.79385375976562, |
| "logps/rejected": -364.9015197753906, |
| "loss": 0.0841, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -0.8070721626281738, |
| "rewards/margins": 6.2195940017700195, |
| "rewards/rejected": -7.026666164398193, |
| "step": 475 |
| }, |
| { |
| "epoch": 1.0234666666666667, |
| "grad_norm": 2.209942579269409, |
| "learning_rate": 3.841899776325267e-06, |
| "logits/chosen": -1.8175064325332642, |
| "logits/rejected": -1.6627798080444336, |
| "logps/chosen": -276.42974853515625, |
| "logps/rejected": -348.6553955078125, |
| "loss": 0.1084, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.6729779243469238, |
| "rewards/margins": 5.814656734466553, |
| "rewards/rejected": -6.487635135650635, |
| "step": 480 |
| }, |
| { |
| "epoch": 1.0341333333333333, |
| "grad_norm": 5.6867594718933105, |
| "learning_rate": 3.817519985603855e-06, |
| "logits/chosen": -1.950702428817749, |
| "logits/rejected": -1.8375511169433594, |
| "logps/chosen": -237.49368286132812, |
| "logps/rejected": -373.88739013671875, |
| "loss": 0.122, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.1298967599868774, |
| "rewards/margins": 6.234266757965088, |
| "rewards/rejected": -7.364163398742676, |
| "step": 485 |
| }, |
| { |
| "epoch": 1.0448, |
| "grad_norm": 3.8766770362854004, |
| "learning_rate": 3.792965466512437e-06, |
| "logits/chosen": -1.8295539617538452, |
| "logits/rejected": -1.6258299350738525, |
| "logps/chosen": -284.9169616699219, |
| "logps/rejected": -360.6707763671875, |
| "loss": 0.1105, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3740360736846924, |
| "rewards/margins": 6.0007429122924805, |
| "rewards/rejected": -7.37477970123291, |
| "step": 490 |
| }, |
| { |
| "epoch": 1.0554666666666668, |
| "grad_norm": 1.0155181884765625, |
| "learning_rate": 3.7682394754502687e-06, |
| "logits/chosen": -1.8263795375823975, |
| "logits/rejected": -1.7287553548812866, |
| "logps/chosen": -253.3537139892578, |
| "logps/rejected": -351.697265625, |
| "loss": 0.1145, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.1872411966323853, |
| "rewards/margins": 6.306341648101807, |
| "rewards/rejected": -7.493583679199219, |
| "step": 495 |
| }, |
| { |
| "epoch": 1.0661333333333334, |
| "grad_norm": 2.8762567043304443, |
| "learning_rate": 3.7433452915570685e-06, |
| "logits/chosen": -1.8811756372451782, |
| "logits/rejected": -1.8007991313934326, |
| "logps/chosen": -265.2466735839844, |
| "logps/rejected": -367.4648132324219, |
| "loss": 0.1216, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.0806666612625122, |
| "rewards/margins": 6.194316864013672, |
| "rewards/rejected": -7.274983882904053, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0768, |
| "grad_norm": 2.1295065879821777, |
| "learning_rate": 3.718286216278143e-06, |
| "logits/chosen": -1.882246732711792, |
| "logits/rejected": -1.761561393737793, |
| "logps/chosen": -302.8157958984375, |
| "logps/rejected": -370.31353759765625, |
| "loss": 0.1144, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.217259407043457, |
| "rewards/margins": 6.504574775695801, |
| "rewards/rejected": -7.7218337059021, |
| "step": 505 |
| }, |
| { |
| "epoch": 1.0874666666666666, |
| "grad_norm": 4.344186782836914, |
| "learning_rate": 3.6930655729265557e-06, |
| "logits/chosen": -1.8296124935150146, |
| "logits/rejected": -1.650610327720642, |
| "logps/chosen": -282.95611572265625, |
| "logps/rejected": -352.1752014160156, |
| "loss": 0.1132, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.343949317932129, |
| "rewards/margins": 6.264041900634766, |
| "rewards/rejected": -7.6079912185668945, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.0981333333333334, |
| "grad_norm": 2.932526111602783, |
| "learning_rate": 3.667686706242386e-06, |
| "logits/chosen": -1.7797019481658936, |
| "logits/rejected": -1.6947786808013916, |
| "logps/chosen": -257.07012939453125, |
| "logps/rejected": -363.6566467285156, |
| "loss": 0.1203, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.1053550243377686, |
| "rewards/margins": 6.050717830657959, |
| "rewards/rejected": -7.156073093414307, |
| "step": 515 |
| }, |
| { |
| "epoch": 1.1088, |
| "grad_norm": 4.214327812194824, |
| "learning_rate": 3.6421529819491576e-06, |
| "logits/chosen": -1.9010741710662842, |
| "logits/rejected": -1.8004162311553955, |
| "logps/chosen": -292.2718811035156, |
| "logps/rejected": -375.58453369140625, |
| "loss": 0.1276, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.4249348640441895, |
| "rewards/margins": 5.854917049407959, |
| "rewards/rejected": -7.279852390289307, |
| "step": 520 |
| }, |
| { |
| "epoch": 1.1194666666666666, |
| "grad_norm": 2.3777670860290527, |
| "learning_rate": 3.6164677863074783e-06, |
| "logits/chosen": -1.8030182123184204, |
| "logits/rejected": -1.6846275329589844, |
| "logps/chosen": -272.50238037109375, |
| "logps/rejected": -366.3551330566406, |
| "loss": 0.1121, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.5246198177337646, |
| "rewards/margins": 6.520844459533691, |
| "rewards/rejected": -8.045464515686035, |
| "step": 525 |
| }, |
| { |
| "epoch": 1.1301333333333332, |
| "grad_norm": 4.471464157104492, |
| "learning_rate": 3.590634525665955e-06, |
| "logits/chosen": -1.7948955297470093, |
| "logits/rejected": -1.6809746026992798, |
| "logps/chosen": -262.9297180175781, |
| "logps/rejected": -381.04901123046875, |
| "loss": 0.0936, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.556490182876587, |
| "rewards/margins": 6.692111015319824, |
| "rewards/rejected": -8.248600959777832, |
| "step": 530 |
| }, |
| { |
| "epoch": 1.1408, |
| "grad_norm": 3.911820888519287, |
| "learning_rate": 3.564656626009446e-06, |
| "logits/chosen": -1.8893358707427979, |
| "logits/rejected": -1.7485568523406982, |
| "logps/chosen": -268.07177734375, |
| "logps/rejected": -390.9849853515625, |
| "loss": 0.1555, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.9269845485687256, |
| "rewards/margins": 6.015888690948486, |
| "rewards/rejected": -7.942873477935791, |
| "step": 535 |
| }, |
| { |
| "epoch": 1.1514666666666666, |
| "grad_norm": 3.294266700744629, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.8690032958984375, |
| "logits/rejected": -1.7063426971435547, |
| "logps/chosen": -267.24578857421875, |
| "logps/rejected": -384.7957458496094, |
| "loss": 0.1166, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.6315892934799194, |
| "rewards/margins": 6.775505065917969, |
| "rewards/rejected": -8.407094955444336, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.1621333333333332, |
| "grad_norm": 4.33017110824585, |
| "learning_rate": 3.512280709043534e-06, |
| "logits/chosen": -1.851406455039978, |
| "logits/rejected": -1.6782381534576416, |
| "logps/chosen": -262.625, |
| "logps/rejected": -372.85479736328125, |
| "loss": 0.1376, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.9270426034927368, |
| "rewards/margins": 6.284356594085693, |
| "rewards/rejected": -7.211399078369141, |
| "step": 545 |
| }, |
| { |
| "epoch": 1.1728, |
| "grad_norm": 3.753199815750122, |
| "learning_rate": 3.4858896377832966e-06, |
| "logits/chosen": -1.8479499816894531, |
| "logits/rejected": -1.7503430843353271, |
| "logps/chosen": -239.7978057861328, |
| "logps/rejected": -369.9670104980469, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9708150625228882, |
| "rewards/margins": 6.7487077713012695, |
| "rewards/rejected": -7.719522953033447, |
| "step": 550 |
| }, |
| { |
| "epoch": 1.1834666666666667, |
| "grad_norm": 3.7350525856018066, |
| "learning_rate": 3.459367818685231e-06, |
| "logits/chosen": -1.8815902471542358, |
| "logits/rejected": -1.7400764226913452, |
| "logps/chosen": -243.23226928710938, |
| "logps/rejected": -369.14349365234375, |
| "loss": 0.1243, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9012893438339233, |
| "rewards/margins": 6.555959224700928, |
| "rewards/rejected": -7.457249641418457, |
| "step": 555 |
| }, |
| { |
| "epoch": 1.1941333333333333, |
| "grad_norm": 3.1958982944488525, |
| "learning_rate": 3.432718769050228e-06, |
| "logits/chosen": -1.764378547668457, |
| "logits/rejected": -1.6634269952774048, |
| "logps/chosen": -248.8746337890625, |
| "logps/rejected": -346.23834228515625, |
| "loss": 0.1185, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.120922565460205, |
| "rewards/margins": 5.680094242095947, |
| "rewards/rejected": -6.801016807556152, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.2048, |
| "grad_norm": 1.215667486190796, |
| "learning_rate": 3.405946023052386e-06, |
| "logits/chosen": -1.8708391189575195, |
| "logits/rejected": -1.7209943532943726, |
| "logps/chosen": -268.7168884277344, |
| "logps/rejected": -383.05194091796875, |
| "loss": 0.1217, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.5790942907333374, |
| "rewards/margins": 6.199840068817139, |
| "rewards/rejected": -7.778934478759766, |
| "step": 565 |
| }, |
| { |
| "epoch": 1.2154666666666667, |
| "grad_norm": 4.288374900817871, |
| "learning_rate": 3.3790531312703055e-06, |
| "logits/chosen": -1.7612943649291992, |
| "logits/rejected": -1.6972544193267822, |
| "logps/chosen": -243.02767944335938, |
| "logps/rejected": -364.6146545410156, |
| "loss": 0.1226, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.6340267658233643, |
| "rewards/margins": 5.973841667175293, |
| "rewards/rejected": -7.6078691482543945, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.2261333333333333, |
| "grad_norm": 2.426377058029175, |
| "learning_rate": 3.35204366021622e-06, |
| "logits/chosen": -1.701094388961792, |
| "logits/rejected": -1.6422580480575562, |
| "logps/chosen": -262.212646484375, |
| "logps/rejected": -371.54376220703125, |
| "loss": 0.1251, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.1702404022216797, |
| "rewards/margins": 6.165692329406738, |
| "rewards/rejected": -7.33593225479126, |
| "step": 575 |
| }, |
| { |
| "epoch": 1.2368000000000001, |
| "grad_norm": 4.914599895477295, |
| "learning_rate": 3.3249211918630055e-06, |
| "logits/chosen": -1.8257882595062256, |
| "logits/rejected": -1.7253891229629517, |
| "logps/chosen": -248.6946563720703, |
| "logps/rejected": -377.42041015625, |
| "loss": 0.1123, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.544409155845642, |
| "rewards/margins": 6.707772731781006, |
| "rewards/rejected": -8.252182006835938, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.2474666666666667, |
| "grad_norm": 3.5044515132904053, |
| "learning_rate": 3.2976893231691424e-06, |
| "logits/chosen": -1.656166672706604, |
| "logits/rejected": -1.6489145755767822, |
| "logps/chosen": -270.85125732421875, |
| "logps/rejected": -362.0047302246094, |
| "loss": 0.1183, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.5041134357452393, |
| "rewards/margins": 6.453446865081787, |
| "rewards/rejected": -7.957561492919922, |
| "step": 585 |
| }, |
| { |
| "epoch": 1.2581333333333333, |
| "grad_norm": 2.836451292037964, |
| "learning_rate": 3.270351665601691e-06, |
| "logits/chosen": -1.7868537902832031, |
| "logits/rejected": -1.6743824481964111, |
| "logps/chosen": -255.75405883789062, |
| "logps/rejected": -383.26068115234375, |
| "loss": 0.088, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5279412269592285, |
| "rewards/margins": 7.09237003326416, |
| "rewards/rejected": -8.62031078338623, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.2688, |
| "grad_norm": 5.035298824310303, |
| "learning_rate": 3.2429118446573417e-06, |
| "logits/chosen": -1.7914812564849854, |
| "logits/rejected": -1.6376540660858154, |
| "logps/chosen": -252.78878784179688, |
| "logps/rejected": -383.7055969238281, |
| "loss": 0.1018, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.580556869506836, |
| "rewards/margins": 6.985851287841797, |
| "rewards/rejected": -8.566408157348633, |
| "step": 595 |
| }, |
| { |
| "epoch": 1.2794666666666665, |
| "grad_norm": 4.154091835021973, |
| "learning_rate": 3.215373499381602e-06, |
| "logits/chosen": -1.7791035175323486, |
| "logits/rejected": -1.624894142150879, |
| "logps/chosen": -251.47994995117188, |
| "logps/rejected": -339.7424621582031, |
| "loss": 0.1252, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.2945369482040405, |
| "rewards/margins": 5.928402900695801, |
| "rewards/rejected": -7.222939968109131, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.2901333333333334, |
| "grad_norm": 1.5163425207138062, |
| "learning_rate": 3.1877402818861954e-06, |
| "logits/chosen": -1.8631162643432617, |
| "logits/rejected": -1.7690770626068115, |
| "logps/chosen": -246.5840606689453, |
| "logps/rejected": -398.3299255371094, |
| "loss": 0.0679, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.8003944158554077, |
| "rewards/margins": 7.669053554534912, |
| "rewards/rejected": -9.46944808959961, |
| "step": 605 |
| }, |
| { |
| "epoch": 1.3008, |
| "grad_norm": 1.9956570863723755, |
| "learning_rate": 3.1600158568647145e-06, |
| "logits/chosen": -1.7449201345443726, |
| "logits/rejected": -1.6165313720703125, |
| "logps/chosen": -282.90399169921875, |
| "logps/rejected": -367.9050598144531, |
| "loss": 0.1269, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6516921520233154, |
| "rewards/margins": 6.830259799957275, |
| "rewards/rejected": -8.481951713562012, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.3114666666666666, |
| "grad_norm": 1.5468382835388184, |
| "learning_rate": 3.1322039011066172e-06, |
| "logits/chosen": -1.780417799949646, |
| "logits/rejected": -1.7057693004608154, |
| "logps/chosen": -257.28753662109375, |
| "logps/rejected": -378.251708984375, |
| "loss": 0.0976, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1922212839126587, |
| "rewards/margins": 7.120926856994629, |
| "rewards/rejected": -8.313148498535156, |
| "step": 615 |
| }, |
| { |
| "epoch": 1.3221333333333334, |
| "grad_norm": 2.0375499725341797, |
| "learning_rate": 3.1043081030096125e-06, |
| "logits/chosen": -1.7060388326644897, |
| "logits/rejected": -1.641692876815796, |
| "logps/chosen": -254.7395782470703, |
| "logps/rejected": -357.2270202636719, |
| "loss": 0.1211, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.581194281578064, |
| "rewards/margins": 6.09962272644043, |
| "rewards/rejected": -7.680817604064941, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.3328, |
| "grad_norm": 3.9574337005615234, |
| "learning_rate": 3.0763321620905072e-06, |
| "logits/chosen": -1.736730933189392, |
| "logits/rejected": -1.6614354848861694, |
| "logps/chosen": -255.10400390625, |
| "logps/rejected": -376.260498046875, |
| "loss": 0.1029, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.3230445384979248, |
| "rewards/margins": 6.803250312805176, |
| "rewards/rejected": -8.12629508972168, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.3434666666666666, |
| "grad_norm": 2.847869634628296, |
| "learning_rate": 3.048279788494583e-06, |
| "logits/chosen": -1.808293342590332, |
| "logits/rejected": -1.6874719858169556, |
| "logps/chosen": -241.78811645507812, |
| "logps/rejected": -371.5953674316406, |
| "loss": 0.104, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4912164211273193, |
| "rewards/margins": 6.666079521179199, |
| "rewards/rejected": -8.157295227050781, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.3541333333333334, |
| "grad_norm": 3.1436240673065186, |
| "learning_rate": 3.0201547025035553e-06, |
| "logits/chosen": -1.786981225013733, |
| "logits/rejected": -1.7256486415863037, |
| "logps/chosen": -251.4448699951172, |
| "logps/rejected": -379.3356628417969, |
| "loss": 0.0956, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5715595483779907, |
| "rewards/margins": 6.413365840911865, |
| "rewards/rejected": -7.98492431640625, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.3648, |
| "grad_norm": 3.5846307277679443, |
| "learning_rate": 2.9919606340421963e-06, |
| "logits/chosen": -1.6872005462646484, |
| "logits/rejected": -1.5978786945343018, |
| "logps/chosen": -247.89767456054688, |
| "logps/rejected": -378.5216369628906, |
| "loss": 0.0886, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7142194509506226, |
| "rewards/margins": 6.703309535980225, |
| "rewards/rejected": -8.417529106140137, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.3754666666666666, |
| "grad_norm": 3.123239278793335, |
| "learning_rate": 2.9637013221836723e-06, |
| "logits/chosen": -1.7665446996688843, |
| "logits/rejected": -1.66081964969635, |
| "logps/chosen": -263.37200927734375, |
| "logps/rejected": -377.9620666503906, |
| "loss": 0.1191, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.6348273754119873, |
| "rewards/margins": 6.643343925476074, |
| "rewards/rejected": -8.278172492980957, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.3861333333333334, |
| "grad_norm": 5.442415714263916, |
| "learning_rate": 2.9353805146536762e-06, |
| "logits/chosen": -1.7114347219467163, |
| "logits/rejected": -1.5927168130874634, |
| "logps/chosen": -258.96282958984375, |
| "logps/rejected": -360.3382873535156, |
| "loss": 0.0935, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6617997884750366, |
| "rewards/margins": 6.874170780181885, |
| "rewards/rejected": -8.535970687866211, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.3968, |
| "grad_norm": 4.683470726013184, |
| "learning_rate": 2.907001967333401e-06, |
| "logits/chosen": -1.748187780380249, |
| "logits/rejected": -1.7318449020385742, |
| "logps/chosen": -271.8290710449219, |
| "logps/rejected": -402.3279724121094, |
| "loss": 0.0957, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.7431981563568115, |
| "rewards/margins": 7.089310646057129, |
| "rewards/rejected": -8.83250904083252, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.4074666666666666, |
| "grad_norm": 2.5879671573638916, |
| "learning_rate": 2.878569443761442e-06, |
| "logits/chosen": -1.695862054824829, |
| "logits/rejected": -1.6406151056289673, |
| "logps/chosen": -258.796875, |
| "logps/rejected": -352.96966552734375, |
| "loss": 0.154, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.9192339181900024, |
| "rewards/margins": 6.030402660369873, |
| "rewards/rejected": -7.949636936187744, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.4181333333333335, |
| "grad_norm": 3.4974851608276367, |
| "learning_rate": 2.85008671463468e-06, |
| "logits/chosen": -1.7247854471206665, |
| "logits/rejected": -1.613422155380249, |
| "logps/chosen": -246.9770965576172, |
| "logps/rejected": -371.6976623535156, |
| "loss": 0.0889, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.660942792892456, |
| "rewards/margins": 7.5227766036987305, |
| "rewards/rejected": -9.18371868133545, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.4288, |
| "grad_norm": 6.057355880737305, |
| "learning_rate": 2.8215575573082127e-06, |
| "logits/chosen": -1.791477918624878, |
| "logits/rejected": -1.7090383768081665, |
| "logps/chosen": -240.56173706054688, |
| "logps/rejected": -391.3064880371094, |
| "loss": 0.1342, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.9196621179580688, |
| "rewards/margins": 7.553940773010254, |
| "rewards/rejected": -9.473602294921875, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.4394666666666667, |
| "grad_norm": 3.2657501697540283, |
| "learning_rate": 2.792985755294404e-06, |
| "logits/chosen": -1.745305061340332, |
| "logits/rejected": -1.6773016452789307, |
| "logps/chosen": -256.55780029296875, |
| "logps/rejected": -385.68304443359375, |
| "loss": 0.1057, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1826651096343994, |
| "rewards/margins": 6.950257301330566, |
| "rewards/rejected": -9.132922172546387, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.4501333333333333, |
| "grad_norm": 1.1689438819885254, |
| "learning_rate": 2.764375097761122e-06, |
| "logits/chosen": -1.6467416286468506, |
| "logits/rejected": -1.5601441860198975, |
| "logps/chosen": -241.9093475341797, |
| "logps/rejected": -382.58197021484375, |
| "loss": 0.1232, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4370450973510742, |
| "rewards/margins": 6.910675048828125, |
| "rewards/rejected": -8.3477201461792, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.4607999999999999, |
| "grad_norm": 2.4693634510040283, |
| "learning_rate": 2.73572937902922e-06, |
| "logits/chosen": -1.7208999395370483, |
| "logits/rejected": -1.5330959558486938, |
| "logps/chosen": -260.446044921875, |
| "logps/rejected": -351.178466796875, |
| "loss": 0.0977, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.2097041606903076, |
| "rewards/margins": 7.18826961517334, |
| "rewards/rejected": -8.397974014282227, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.4714666666666667, |
| "grad_norm": 0.7693854570388794, |
| "learning_rate": 2.7070523980693362e-06, |
| "logits/chosen": -1.5774312019348145, |
| "logits/rejected": -1.5146809816360474, |
| "logps/chosen": -255.57992553710938, |
| "logps/rejected": -353.2482604980469, |
| "loss": 0.1025, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.336712121963501, |
| "rewards/margins": 6.85360860824585, |
| "rewards/rejected": -8.19032096862793, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.4821333333333333, |
| "grad_norm": 1.9660614728927612, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.7294142246246338, |
| "logits/rejected": -1.6881201267242432, |
| "logps/chosen": -254.1973876953125, |
| "logps/rejected": -371.61785888671875, |
| "loss": 0.1127, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.7192710638046265, |
| "rewards/margins": 7.066683769226074, |
| "rewards/rejected": -8.785955429077148, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.4928, |
| "grad_norm": 3.160243511199951, |
| "learning_rate": 2.649619865573666e-06, |
| "logits/chosen": -1.8685362339019775, |
| "logits/rejected": -1.6781858205795288, |
| "logps/chosen": -250.9998016357422, |
| "logps/rejected": -389.24212646484375, |
| "loss": 0.0503, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.2655198574066162, |
| "rewards/margins": 8.069536209106445, |
| "rewards/rejected": -9.335058212280273, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.5034666666666667, |
| "grad_norm": 2.4785704612731934, |
| "learning_rate": 2.620871930691058e-06, |
| "logits/chosen": -1.7562789916992188, |
| "logits/rejected": -1.6890990734100342, |
| "logps/chosen": -257.45574951171875, |
| "logps/rejected": -376.5826110839844, |
| "loss": 0.1266, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.5733845233917236, |
| "rewards/margins": 6.933058261871338, |
| "rewards/rejected": -8.506443977355957, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.5141333333333333, |
| "grad_norm": 2.3427517414093018, |
| "learning_rate": 2.5921079658767152e-06, |
| "logits/chosen": -1.7983629703521729, |
| "logits/rejected": -1.6955991983413696, |
| "logps/chosen": -265.74200439453125, |
| "logps/rejected": -360.1547546386719, |
| "loss": 0.144, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.7915146350860596, |
| "rewards/margins": 6.563783168792725, |
| "rewards/rejected": -8.355298042297363, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.5248, |
| "grad_norm": 3.470958948135376, |
| "learning_rate": 2.56333178578297e-06, |
| "logits/chosen": -1.779236078262329, |
| "logits/rejected": -1.6289777755737305, |
| "logps/chosen": -253.560546875, |
| "logps/rejected": -373.86187744140625, |
| "loss": 0.1072, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.766411542892456, |
| "rewards/margins": 7.2796630859375, |
| "rewards/rejected": -9.046075820922852, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.5354666666666668, |
| "grad_norm": 6.400219440460205, |
| "learning_rate": 2.534547206682136e-06, |
| "logits/chosen": -1.7732305526733398, |
| "logits/rejected": -1.6621010303497314, |
| "logps/chosen": -248.0487060546875, |
| "logps/rejected": -365.96014404296875, |
| "loss": 0.1459, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.613825798034668, |
| "rewards/margins": 7.0502424240112305, |
| "rewards/rejected": -8.664068222045898, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.5461333333333334, |
| "grad_norm": 2.4263100624084473, |
| "learning_rate": 2.5057580459603947e-06, |
| "logits/chosen": -1.8636562824249268, |
| "logits/rejected": -1.7396777868270874, |
| "logps/chosen": -259.3055419921875, |
| "logps/rejected": -364.75115966796875, |
| "loss": 0.084, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.641998529434204, |
| "rewards/margins": 7.215865135192871, |
| "rewards/rejected": -8.857863426208496, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.5568, |
| "grad_norm": 2.6077945232391357, |
| "learning_rate": 2.4769681216115384e-06, |
| "logits/chosen": -1.8251991271972656, |
| "logits/rejected": -1.752423882484436, |
| "logps/chosen": -272.037841796875, |
| "logps/rejected": -376.82769775390625, |
| "loss": 0.106, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.859642744064331, |
| "rewards/margins": 6.8842644691467285, |
| "rewards/rejected": -8.74390697479248, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.5674666666666668, |
| "grad_norm": 2.8333311080932617, |
| "learning_rate": 2.448181251730632e-06, |
| "logits/chosen": -1.7708256244659424, |
| "logits/rejected": -1.621558427810669, |
| "logps/chosen": -251.92745971679688, |
| "logps/rejected": -365.28289794921875, |
| "loss": 0.1074, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.313032627105713, |
| "rewards/margins": 6.848665714263916, |
| "rewards/rejected": -8.161698341369629, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.5781333333333334, |
| "grad_norm": 3.1226139068603516, |
| "learning_rate": 2.419401254007658e-06, |
| "logits/chosen": -1.7166410684585571, |
| "logits/rejected": -1.6209323406219482, |
| "logps/chosen": -248.4964599609375, |
| "logps/rejected": -359.0329895019531, |
| "loss": 0.083, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.2056635618209839, |
| "rewards/margins": 7.018212795257568, |
| "rewards/rejected": -8.223875999450684, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.5888, |
| "grad_norm": 4.352267742156982, |
| "learning_rate": 2.390631945221221e-06, |
| "logits/chosen": -1.7971668243408203, |
| "logits/rejected": -1.6970182657241821, |
| "logps/chosen": -256.62530517578125, |
| "logps/rejected": -390.9517517089844, |
| "loss": 0.1439, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.6711896657943726, |
| "rewards/margins": 6.6160407066345215, |
| "rewards/rejected": -8.287230491638184, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.5994666666666668, |
| "grad_norm": 0.7765277028083801, |
| "learning_rate": 2.361877140732366e-06, |
| "logits/chosen": -1.834469199180603, |
| "logits/rejected": -1.6532119512557983, |
| "logps/chosen": -249.960693359375, |
| "logps/rejected": -363.2095642089844, |
| "loss": 0.063, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.540730357170105, |
| "rewards/margins": 7.1370530128479, |
| "rewards/rejected": -8.67778205871582, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.6101333333333332, |
| "grad_norm": 3.875185489654541, |
| "learning_rate": 2.333140653978592e-06, |
| "logits/chosen": -1.7678000926971436, |
| "logits/rejected": -1.6562210321426392, |
| "logps/chosen": -254.3054962158203, |
| "logps/rejected": -370.5506286621094, |
| "loss": 0.1142, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.564232587814331, |
| "rewards/margins": 7.189335823059082, |
| "rewards/rejected": -8.753567695617676, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.6208, |
| "grad_norm": 1.7597862482070923, |
| "learning_rate": 2.3044262959681156e-06, |
| "logits/chosen": -1.7677427530288696, |
| "logits/rejected": -1.6805483102798462, |
| "logps/chosen": -238.32080078125, |
| "logps/rejected": -368.8369140625, |
| "loss": 0.0859, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3052327632904053, |
| "rewards/margins": 6.70794677734375, |
| "rewards/rejected": -8.013179779052734, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.6314666666666666, |
| "grad_norm": 3.3619678020477295, |
| "learning_rate": 2.2757378747744575e-06, |
| "logits/chosen": -1.8301753997802734, |
| "logits/rejected": -1.6926501989364624, |
| "logps/chosen": -242.58230590820312, |
| "logps/rejected": -367.21893310546875, |
| "loss": 0.0896, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6891601085662842, |
| "rewards/margins": 7.6020050048828125, |
| "rewards/rejected": -9.291166305541992, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.6421333333333332, |
| "grad_norm": 12.008077621459961, |
| "learning_rate": 2.24707919503142e-06, |
| "logits/chosen": -1.776662826538086, |
| "logits/rejected": -1.733825922012329, |
| "logps/chosen": -255.0137939453125, |
| "logps/rejected": -397.6699523925781, |
| "loss": 0.1343, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.5180081129074097, |
| "rewards/margins": 7.4235711097717285, |
| "rewards/rejected": -8.941579818725586, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.6528, |
| "grad_norm": 2.1331827640533447, |
| "learning_rate": 2.2184540574285213e-06, |
| "logits/chosen": -1.7663053274154663, |
| "logits/rejected": -1.6731045246124268, |
| "logps/chosen": -277.5382995605469, |
| "logps/rejected": -404.40435791015625, |
| "loss": 0.0919, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.5203996896743774, |
| "rewards/margins": 7.110515594482422, |
| "rewards/rejected": -8.630915641784668, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.6634666666666666, |
| "grad_norm": 4.992430210113525, |
| "learning_rate": 2.1898662582069486e-06, |
| "logits/chosen": -1.7861846685409546, |
| "logits/rejected": -1.666831612586975, |
| "logps/chosen": -245.0485076904297, |
| "logps/rejected": -373.1742858886719, |
| "loss": 0.0901, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7309143543243408, |
| "rewards/margins": 7.291200160980225, |
| "rewards/rejected": -9.022114753723145, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.6741333333333333, |
| "grad_norm": 1.1797524690628052, |
| "learning_rate": 2.1613195886561044e-06, |
| "logits/chosen": -1.7284221649169922, |
| "logits/rejected": -1.6566905975341797, |
| "logps/chosen": -259.3564453125, |
| "logps/rejected": -381.2314758300781, |
| "loss": 0.1172, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.002119302749634, |
| "rewards/margins": 7.27401876449585, |
| "rewards/rejected": -9.276138305664062, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.6848, |
| "grad_norm": 4.15262508392334, |
| "learning_rate": 2.132817834610811e-06, |
| "logits/chosen": -1.7999082803726196, |
| "logits/rejected": -1.6664059162139893, |
| "logps/chosen": -256.9659118652344, |
| "logps/rejected": -378.0249938964844, |
| "loss": 0.1458, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0210254192352295, |
| "rewards/margins": 7.168332099914551, |
| "rewards/rejected": -9.18935775756836, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.6954666666666667, |
| "grad_norm": 1.3209813833236694, |
| "learning_rate": 2.1043647759492338e-06, |
| "logits/chosen": -1.6264126300811768, |
| "logits/rejected": -1.6061350107192993, |
| "logps/chosen": -264.5497131347656, |
| "logps/rejected": -382.03759765625, |
| "loss": 0.1261, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.736513376235962, |
| "rewards/margins": 6.7935075759887695, |
| "rewards/rejected": -8.530019760131836, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.7061333333333333, |
| "grad_norm": 4.297145366668701, |
| "learning_rate": 2.075964186091599e-06, |
| "logits/chosen": -1.7351667881011963, |
| "logits/rejected": -1.7021973133087158, |
| "logps/chosen": -252.9200897216797, |
| "logps/rejected": -394.8159484863281, |
| "loss": 0.1009, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8241618871688843, |
| "rewards/margins": 7.219931125640869, |
| "rewards/rejected": -9.044093132019043, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.7168, |
| "grad_norm": 1.3593531847000122, |
| "learning_rate": 2.047619831499765e-06, |
| "logits/chosen": -1.6939481496810913, |
| "logits/rejected": -1.6624157428741455, |
| "logps/chosen": -256.53021240234375, |
| "logps/rejected": -387.21490478515625, |
| "loss": 0.1098, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6262582540512085, |
| "rewards/margins": 7.0810866355896, |
| "rewards/rejected": -8.707345962524414, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.7274666666666667, |
| "grad_norm": 2.174813747406006, |
| "learning_rate": 2.0193354711777155e-06, |
| "logits/chosen": -1.8526443243026733, |
| "logits/rejected": -1.7142467498779297, |
| "logps/chosen": -262.6146240234375, |
| "logps/rejected": -372.4858093261719, |
| "loss": 0.0899, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.491632103919983, |
| "rewards/margins": 7.115251064300537, |
| "rewards/rejected": -8.606884956359863, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.7381333333333333, |
| "grad_norm": 2.587124824523926, |
| "learning_rate": 1.991114856173048e-06, |
| "logits/chosen": -1.8616607189178467, |
| "logits/rejected": -1.7974239587783813, |
| "logps/chosen": -251.6106414794922, |
| "logps/rejected": -384.2275390625, |
| "loss": 0.0772, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.016730308532715, |
| "rewards/margins": 7.789799690246582, |
| "rewards/rejected": -9.806529998779297, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.7488000000000001, |
| "grad_norm": 1.9651814699172974, |
| "learning_rate": 1.9629617290795115e-06, |
| "logits/chosen": -1.8186252117156982, |
| "logits/rejected": -1.7092021703720093, |
| "logps/chosen": -244.533447265625, |
| "logps/rejected": -407.6358642578125, |
| "loss": 0.0777, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.0436220169067383, |
| "rewards/margins": 7.583238124847412, |
| "rewards/rejected": -9.626859664916992, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.7594666666666665, |
| "grad_norm": 7.782098770141602, |
| "learning_rate": 1.934879823540663e-06, |
| "logits/chosen": -1.8047682046890259, |
| "logits/rejected": -1.6448373794555664, |
| "logps/chosen": -243.6357879638672, |
| "logps/rejected": -371.1468811035156, |
| "loss": 0.1504, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.050574779510498, |
| "rewards/margins": 7.807431697845459, |
| "rewards/rejected": -9.85800552368164, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.7701333333333333, |
| "grad_norm": 5.324383735656738, |
| "learning_rate": 1.9068728637547195e-06, |
| "logits/chosen": -1.7081286907196045, |
| "logits/rejected": -1.611659049987793, |
| "logps/chosen": -254.0673370361328, |
| "logps/rejected": -399.0706481933594, |
| "loss": 0.1823, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.228820323944092, |
| "rewards/margins": 7.0853753089904785, |
| "rewards/rejected": -9.31419563293457, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.7808000000000002, |
| "grad_norm": 4.415861129760742, |
| "learning_rate": 1.878944563980658e-06, |
| "logits/chosen": -1.754969596862793, |
| "logits/rejected": -1.5842684507369995, |
| "logps/chosen": -267.50372314453125, |
| "logps/rejected": -385.69244384765625, |
| "loss": 0.117, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.268184185028076, |
| "rewards/margins": 7.348923683166504, |
| "rewards/rejected": -9.617107391357422, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.7914666666666665, |
| "grad_norm": 4.7538299560546875, |
| "learning_rate": 1.85109862804563e-06, |
| "logits/chosen": -1.7412681579589844, |
| "logits/rejected": -1.6989322900772095, |
| "logps/chosen": -267.0843811035156, |
| "logps/rejected": -400.9412536621094, |
| "loss": 0.0858, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5771751403808594, |
| "rewards/margins": 8.0297212600708, |
| "rewards/rejected": -9.60689640045166, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.8021333333333334, |
| "grad_norm": 2.8075060844421387, |
| "learning_rate": 1.823338748853766e-06, |
| "logits/chosen": -1.7989814281463623, |
| "logits/rejected": -1.695749282836914, |
| "logps/chosen": -267.8409118652344, |
| "logps/rejected": -406.75860595703125, |
| "loss": 0.1053, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9596891403198242, |
| "rewards/margins": 7.2939252853393555, |
| "rewards/rejected": -9.25361442565918, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.8128, |
| "grad_norm": 2.450321912765503, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.8007376194000244, |
| "logits/rejected": -1.696488618850708, |
| "logps/chosen": -268.1695251464844, |
| "logps/rejected": -390.95697021484375, |
| "loss": 0.0814, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.6625934839248657, |
| "rewards/margins": 7.488409519195557, |
| "rewards/rejected": -9.151002883911133, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.8234666666666666, |
| "grad_norm": 4.618243217468262, |
| "learning_rate": 1.7680918747639608e-06, |
| "logits/chosen": -1.7959873676300049, |
| "logits/rejected": -1.7379106283187866, |
| "logps/chosen": -280.7197265625, |
| "logps/rejected": -414.0787048339844, |
| "loss": 0.0978, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.2764201164245605, |
| "rewards/margins": 7.298665523529053, |
| "rewards/rejected": -9.57508659362793, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.8341333333333334, |
| "grad_norm": 4.439633369445801, |
| "learning_rate": 1.740612206659057e-06, |
| "logits/chosen": -1.786678671836853, |
| "logits/rejected": -1.6649478673934937, |
| "logps/chosen": -255.8995819091797, |
| "logps/rejected": -389.19049072265625, |
| "loss": 0.1015, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0884041786193848, |
| "rewards/margins": 7.275248050689697, |
| "rewards/rejected": -9.363652229309082, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.8448, |
| "grad_norm": 3.4084367752075195, |
| "learning_rate": 1.7132332479117204e-06, |
| "logits/chosen": -1.7992115020751953, |
| "logits/rejected": -1.6410239934921265, |
| "logps/chosen": -272.89288330078125, |
| "logps/rejected": -402.8624572753906, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0090274810791016, |
| "rewards/margins": 7.744027614593506, |
| "rewards/rejected": -9.75305461883545, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.8554666666666666, |
| "grad_norm": 3.4071297645568848, |
| "learning_rate": 1.6859586294959666e-06, |
| "logits/chosen": -1.8045650720596313, |
| "logits/rejected": -1.6810834407806396, |
| "logps/chosen": -263.1116638183594, |
| "logps/rejected": -397.77880859375, |
| "loss": 0.0751, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.172431468963623, |
| "rewards/margins": 8.27795124053955, |
| "rewards/rejected": -10.450382232666016, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.8661333333333334, |
| "grad_norm": 6.4309983253479, |
| "learning_rate": 1.6587919685482884e-06, |
| "logits/chosen": -1.8300575017929077, |
| "logits/rejected": -1.7437467575073242, |
| "logps/chosen": -253.6339874267578, |
| "logps/rejected": -413.36407470703125, |
| "loss": 0.1226, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.1895852088928223, |
| "rewards/margins": 8.25765323638916, |
| "rewards/rejected": -10.44723892211914, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.8768, |
| "grad_norm": 4.335732460021973, |
| "learning_rate": 1.6317368678879497e-06, |
| "logits/chosen": -1.7067629098892212, |
| "logits/rejected": -1.6405712366104126, |
| "logps/chosen": -249.4864501953125, |
| "logps/rejected": -369.77691650390625, |
| "loss": 0.1302, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.5186713933944702, |
| "rewards/margins": 6.943307399749756, |
| "rewards/rejected": -8.4619779586792, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.8874666666666666, |
| "grad_norm": 6.803175926208496, |
| "learning_rate": 1.6047969155391856e-06, |
| "logits/chosen": -1.7294635772705078, |
| "logits/rejected": -1.6601709127426147, |
| "logps/chosen": -276.130859375, |
| "logps/rejected": -373.820556640625, |
| "loss": 0.1253, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.0530483722686768, |
| "rewards/margins": 6.646698951721191, |
| "rewards/rejected": -8.699747085571289, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.8981333333333335, |
| "grad_norm": 4.3072590827941895, |
| "learning_rate": 1.5779756842553617e-06, |
| "logits/chosen": -1.7643864154815674, |
| "logits/rejected": -1.7279653549194336, |
| "logps/chosen": -278.6283874511719, |
| "logps/rejected": -399.0856018066406, |
| "loss": 0.115, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.922265648841858, |
| "rewards/margins": 7.855748176574707, |
| "rewards/rejected": -9.778013229370117, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.9088, |
| "grad_norm": 2.2582664489746094, |
| "learning_rate": 1.551276731045157e-06, |
| "logits/chosen": -1.7211596965789795, |
| "logits/rejected": -1.6444860696792603, |
| "logps/chosen": -290.8036804199219, |
| "logps/rejected": -395.46099853515625, |
| "loss": 0.1081, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.7245696783065796, |
| "rewards/margins": 7.342646598815918, |
| "rewards/rejected": -9.067216873168945, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.9194666666666667, |
| "grad_norm": 2.4317190647125244, |
| "learning_rate": 1.5247035967008369e-06, |
| "logits/chosen": -1.6402801275253296, |
| "logits/rejected": -1.6369024515151978, |
| "logps/chosen": -248.6401824951172, |
| "logps/rejected": -388.771484375, |
| "loss": 0.1485, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.689244270324707, |
| "rewards/margins": 7.255883693695068, |
| "rewards/rejected": -8.945128440856934, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9301333333333335, |
| "grad_norm": 1.7278972864151, |
| "learning_rate": 1.498259805328678e-06, |
| "logits/chosen": -1.799878716468811, |
| "logits/rejected": -1.6546272039413452, |
| "logps/chosen": -253.69113159179688, |
| "logps/rejected": -377.49237060546875, |
| "loss": 0.1227, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6651760339736938, |
| "rewards/margins": 7.9041266441345215, |
| "rewards/rejected": -9.569303512573242, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.9407999999999999, |
| "grad_norm": 4.5444016456604, |
| "learning_rate": 1.471948863881601e-06, |
| "logits/chosen": -1.7796027660369873, |
| "logits/rejected": -1.633963942527771, |
| "logps/chosen": -284.4118347167969, |
| "logps/rejected": -395.56317138671875, |
| "loss": 0.0978, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.3895740509033203, |
| "rewards/margins": 7.175274848937988, |
| "rewards/rejected": -8.564848899841309, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.9514666666666667, |
| "grad_norm": 3.3118534088134766, |
| "learning_rate": 1.4457742616940864e-06, |
| "logits/chosen": -1.6823219060897827, |
| "logits/rejected": -1.6117699146270752, |
| "logps/chosen": -263.32958984375, |
| "logps/rejected": -377.5818786621094, |
| "loss": 0.1023, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.4369984865188599, |
| "rewards/margins": 7.370998382568359, |
| "rewards/rejected": -8.80799674987793, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.9621333333333333, |
| "grad_norm": 3.7233338356018066, |
| "learning_rate": 1.4197394700194165e-06, |
| "logits/chosen": -1.7843736410140991, |
| "logits/rejected": -1.6962296962738037, |
| "logps/chosen": -260.7303161621094, |
| "logps/rejected": -370.4283142089844, |
| "loss": 0.0852, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.220878005027771, |
| "rewards/margins": 7.065885066986084, |
| "rewards/rejected": -8.286763191223145, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.9727999999999999, |
| "grad_norm": 3.0080010890960693, |
| "learning_rate": 1.3938479415693244e-06, |
| "logits/chosen": -1.8749253749847412, |
| "logits/rejected": -1.7267560958862305, |
| "logps/chosen": -239.6318359375, |
| "logps/rejected": -353.81134033203125, |
| "loss": 0.0997, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.0718796253204346, |
| "rewards/margins": 6.565346717834473, |
| "rewards/rejected": -7.637226104736328, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.9834666666666667, |
| "grad_norm": 3.3940622806549072, |
| "learning_rate": 1.368103110056096e-06, |
| "logits/chosen": -1.7393932342529297, |
| "logits/rejected": -1.6677579879760742, |
| "logps/chosen": -267.26727294921875, |
| "logps/rejected": -375.4582824707031, |
| "loss": 0.0921, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6906468868255615, |
| "rewards/margins": 7.26543664932251, |
| "rewards/rejected": -8.956083297729492, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.9941333333333333, |
| "grad_norm": 1.297263503074646, |
| "learning_rate": 1.3425083897371983e-06, |
| "logits/chosen": -1.652087926864624, |
| "logits/rejected": -1.6215932369232178, |
| "logps/chosen": -275.397216796875, |
| "logps/rejected": -385.1170349121094, |
| "loss": 0.099, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3324906826019287, |
| "rewards/margins": 7.011690616607666, |
| "rewards/rejected": -8.344181060791016, |
| "step": 935 |
| }, |
| { |
| "epoch": 2.0042666666666666, |
| "grad_norm": 2.640092372894287, |
| "learning_rate": 1.317067174962478e-06, |
| "logits/chosen": -1.716870903968811, |
| "logits/rejected": -1.6423840522766113, |
| "logps/chosen": -280.1388244628906, |
| "logps/rejected": -405.33685302734375, |
| "loss": 0.0804, |
| "rewards/accuracies": 0.9868420958518982, |
| "rewards/chosen": -1.5010406970977783, |
| "rewards/margins": 7.210644245147705, |
| "rewards/rejected": -8.711686134338379, |
| "step": 940 |
| }, |
| { |
| "epoch": 2.0149333333333335, |
| "grad_norm": 0.6266006231307983, |
| "learning_rate": 1.291782839724009e-06, |
| "logits/chosen": -1.8108909130096436, |
| "logits/rejected": -1.6904157400131226, |
| "logps/chosen": -283.57354736328125, |
| "logps/rejected": -391.6214294433594, |
| "loss": 0.0755, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4528306722640991, |
| "rewards/margins": 7.529816627502441, |
| "rewards/rejected": -8.982647895812988, |
| "step": 945 |
| }, |
| { |
| "epoch": 2.0256, |
| "grad_norm": 1.9588918685913086, |
| "learning_rate": 1.2666587372086352e-06, |
| "logits/chosen": -1.6926864385604858, |
| "logits/rejected": -1.6244741678237915, |
| "logps/chosen": -250.9656524658203, |
| "logps/rejected": -378.7623596191406, |
| "loss": 0.085, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.433120608329773, |
| "rewards/margins": 6.887550354003906, |
| "rewards/rejected": -8.320670127868652, |
| "step": 950 |
| }, |
| { |
| "epoch": 2.0362666666666667, |
| "grad_norm": 2.0722038745880127, |
| "learning_rate": 1.241698199353275e-06, |
| "logits/chosen": -1.8123573064804077, |
| "logits/rejected": -1.7618974447250366, |
| "logps/chosen": -253.0042266845703, |
| "logps/rejected": -365.7357482910156, |
| "loss": 0.0788, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.5714794397354126, |
| "rewards/margins": 6.544591426849365, |
| "rewards/rejected": -8.116071701049805, |
| "step": 955 |
| }, |
| { |
| "epoch": 2.0469333333333335, |
| "grad_norm": 1.5139915943145752, |
| "learning_rate": 1.2169045364030404e-06, |
| "logits/chosen": -1.8418400287628174, |
| "logits/rejected": -1.7378206253051758, |
| "logps/chosen": -247.4741973876953, |
| "logps/rejected": -366.4799499511719, |
| "loss": 0.072, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.658636450767517, |
| "rewards/margins": 7.499236106872559, |
| "rewards/rejected": -9.157873153686523, |
| "step": 960 |
| }, |
| { |
| "epoch": 2.0576, |
| "grad_norm": 6.052288055419922, |
| "learning_rate": 1.1922810364722402e-06, |
| "logits/chosen": -1.7987949848175049, |
| "logits/rejected": -1.6627559661865234, |
| "logps/chosen": -270.3972473144531, |
| "logps/rejected": -379.8498840332031, |
| "loss": 0.084, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3009984493255615, |
| "rewards/margins": 6.970292568206787, |
| "rewards/rejected": -8.271291732788086, |
| "step": 965 |
| }, |
| { |
| "epoch": 2.0682666666666667, |
| "grad_norm": 1.6238378286361694, |
| "learning_rate": 1.1678309651083055e-06, |
| "logits/chosen": -1.8293536901474, |
| "logits/rejected": -1.7384620904922485, |
| "logps/chosen": -265.1159973144531, |
| "logps/rejected": -396.7872314453125, |
| "loss": 0.0514, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7698032855987549, |
| "rewards/margins": 7.758294582366943, |
| "rewards/rejected": -9.528097152709961, |
| "step": 970 |
| }, |
| { |
| "epoch": 2.0789333333333335, |
| "grad_norm": 1.211134910583496, |
| "learning_rate": 1.1435575648587224e-06, |
| "logits/chosen": -1.6530580520629883, |
| "logits/rejected": -1.649542212486267, |
| "logps/chosen": -239.8114776611328, |
| "logps/rejected": -357.4482116699219, |
| "loss": 0.0854, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.353614091873169, |
| "rewards/margins": 7.633919715881348, |
| "rewards/rejected": -8.987533569335938, |
| "step": 975 |
| }, |
| { |
| "epoch": 2.0896, |
| "grad_norm": 3.313692331314087, |
| "learning_rate": 1.1194640548410039e-06, |
| "logits/chosen": -1.7463849782943726, |
| "logits/rejected": -1.6572465896606445, |
| "logps/chosen": -272.10955810546875, |
| "logps/rejected": -365.974609375, |
| "loss": 0.1135, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.6518981456756592, |
| "rewards/margins": 7.179516792297363, |
| "rewards/rejected": -8.831415176391602, |
| "step": 980 |
| }, |
| { |
| "epoch": 2.1002666666666667, |
| "grad_norm": 2.2295451164245605, |
| "learning_rate": 1.095553630315774e-06, |
| "logits/chosen": -1.735437035560608, |
| "logits/rejected": -1.6388800144195557, |
| "logps/chosen": -255.56564331054688, |
| "logps/rejected": -370.2838439941406, |
| "loss": 0.0804, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7196842432022095, |
| "rewards/margins": 7.343177795410156, |
| "rewards/rejected": -9.06286334991455, |
| "step": 985 |
| }, |
| { |
| "epoch": 2.1109333333333336, |
| "grad_norm": 1.403108835220337, |
| "learning_rate": 1.0718294622630188e-06, |
| "logits/chosen": -1.8256213665008545, |
| "logits/rejected": -1.7383081912994385, |
| "logps/chosen": -246.55419921875, |
| "logps/rejected": -404.0504150390625, |
| "loss": 0.0593, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.499764084815979, |
| "rewards/margins": 7.549830436706543, |
| "rewards/rejected": -9.049592971801758, |
| "step": 990 |
| }, |
| { |
| "epoch": 2.1216, |
| "grad_norm": 3.562825918197632, |
| "learning_rate": 1.0482946969615495e-06, |
| "logits/chosen": -1.859043836593628, |
| "logits/rejected": -1.6865688562393188, |
| "logps/chosen": -258.09228515625, |
| "logps/rejected": -418.1453552246094, |
| "loss": 0.0666, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.222257614135742, |
| "rewards/margins": 8.348030090332031, |
| "rewards/rejected": -10.570287704467773, |
| "step": 995 |
| }, |
| { |
| "epoch": 2.1322666666666668, |
| "grad_norm": 0.6572063565254211, |
| "learning_rate": 1.0249524555717461e-06, |
| "logits/chosen": -1.804359793663025, |
| "logits/rejected": -1.5746873617172241, |
| "logps/chosen": -233.9191436767578, |
| "logps/rejected": -369.0513000488281, |
| "loss": 0.0785, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3012584447860718, |
| "rewards/margins": 7.778643608093262, |
| "rewards/rejected": -9.079902648925781, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.142933333333333, |
| "grad_norm": 2.282438278198242, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.6505435705184937, |
| "logits/rejected": -1.6292928457260132, |
| "logps/chosen": -265.0, |
| "logps/rejected": -377.8451232910156, |
| "loss": 0.0635, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.9961382150650024, |
| "rewards/margins": 7.445557594299316, |
| "rewards/rejected": -9.441695213317871, |
| "step": 1005 |
| }, |
| { |
| "epoch": 2.1536, |
| "grad_norm": 3.273324966430664, |
| "learning_rate": 9.788579010963387e-07, |
| "logits/chosen": -1.7554762363433838, |
| "logits/rejected": -1.622506856918335, |
| "logps/chosen": -276.3446960449219, |
| "logps/rejected": -387.22259521484375, |
| "loss": 0.0746, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0051329135894775, |
| "rewards/margins": 8.116923332214355, |
| "rewards/rejected": -10.122055053710938, |
| "step": 1010 |
| }, |
| { |
| "epoch": 2.164266666666667, |
| "grad_norm": 2.056083917617798, |
| "learning_rate": 9.56111701030998e-07, |
| "logits/chosen": -1.777017593383789, |
| "logits/rejected": -1.634634256362915, |
| "logps/chosen": -257.826171875, |
| "logps/rejected": -375.39349365234375, |
| "loss": 0.0912, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8016389608383179, |
| "rewards/margins": 7.549478054046631, |
| "rewards/rejected": -9.351117134094238, |
| "step": 1015 |
| }, |
| { |
| "epoch": 2.174933333333333, |
| "grad_norm": 2.441992998123169, |
| "learning_rate": 9.335702501071459e-07, |
| "logits/chosen": -1.7044284343719482, |
| "logits/rejected": -1.6332372426986694, |
| "logps/chosen": -265.9286193847656, |
| "logps/rejected": -397.9439697265625, |
| "loss": 0.0764, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.5722075700759888, |
| "rewards/margins": 7.702206611633301, |
| "rewards/rejected": -9.274413108825684, |
| "step": 1020 |
| }, |
| { |
| "epoch": 2.1856, |
| "grad_norm": 3.191629409790039, |
| "learning_rate": 9.112365377526636e-07, |
| "logits/chosen": -1.8316318988800049, |
| "logits/rejected": -1.7240879535675049, |
| "logps/chosen": -234.20425415039062, |
| "logps/rejected": -386.95831298828125, |
| "loss": 0.0628, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1653170585632324, |
| "rewards/margins": 8.376472473144531, |
| "rewards/rejected": -10.541789054870605, |
| "step": 1025 |
| }, |
| { |
| "epoch": 2.196266666666667, |
| "grad_norm": 0.986142635345459, |
| "learning_rate": 8.891135258453195e-07, |
| "logits/chosen": -1.6836477518081665, |
| "logits/rejected": -1.6255674362182617, |
| "logps/chosen": -262.45831298828125, |
| "logps/rejected": -398.49090576171875, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9247242212295532, |
| "rewards/margins": 8.067194938659668, |
| "rewards/rejected": -9.99191951751709, |
| "step": 1030 |
| }, |
| { |
| "epoch": 2.206933333333333, |
| "grad_norm": 3.276202917098999, |
| "learning_rate": 8.672041483199709e-07, |
| "logits/chosen": -1.7350730895996094, |
| "logits/rejected": -1.6049894094467163, |
| "logps/chosen": -275.9724426269531, |
| "logps/rejected": -408.84161376953125, |
| "loss": 0.1033, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.006324291229248, |
| "rewards/margins": 7.502695560455322, |
| "rewards/rejected": -9.50901985168457, |
| "step": 1035 |
| }, |
| { |
| "epoch": 2.2176, |
| "grad_norm": 1.4491831064224243, |
| "learning_rate": 8.455113107794652e-07, |
| "logits/chosen": -1.7363901138305664, |
| "logits/rejected": -1.6504329442977905, |
| "logps/chosen": -250.92138671875, |
| "logps/rejected": -378.4014587402344, |
| "loss": 0.053, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.7011101245880127, |
| "rewards/margins": 8.367371559143066, |
| "rewards/rejected": -10.068480491638184, |
| "step": 1040 |
| }, |
| { |
| "epoch": 2.228266666666667, |
| "grad_norm": 2.495983362197876, |
| "learning_rate": 8.240378901093035e-07, |
| "logits/chosen": -1.7328039407730103, |
| "logits/rejected": -1.6397279500961304, |
| "logps/chosen": -277.71942138671875, |
| "logps/rejected": -401.9482116699219, |
| "loss": 0.0764, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0400569438934326, |
| "rewards/margins": 8.178812980651855, |
| "rewards/rejected": -10.218870162963867, |
| "step": 1045 |
| }, |
| { |
| "epoch": 2.238933333333333, |
| "grad_norm": 0.955857515335083, |
| "learning_rate": 8.027867340961113e-07, |
| "logits/chosen": -1.6360746622085571, |
| "logits/rejected": -1.591421365737915, |
| "logps/chosen": -275.9684143066406, |
| "logps/rejected": -408.9102478027344, |
| "loss": 0.0648, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0029473304748535, |
| "rewards/margins": 8.104959487915039, |
| "rewards/rejected": -10.107905387878418, |
| "step": 1050 |
| }, |
| { |
| "epoch": 2.2496, |
| "grad_norm": 3.9517617225646973, |
| "learning_rate": 7.817606610499639e-07, |
| "logits/chosen": -1.7221180200576782, |
| "logits/rejected": -1.6445471048355103, |
| "logps/chosen": -281.46209716796875, |
| "logps/rejected": -402.68359375, |
| "loss": 0.0544, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2298858165740967, |
| "rewards/margins": 8.064582824707031, |
| "rewards/rejected": -10.294468879699707, |
| "step": 1055 |
| }, |
| { |
| "epoch": 2.2602666666666664, |
| "grad_norm": 0.5854415893554688, |
| "learning_rate": 7.609624594306278e-07, |
| "logits/chosen": -1.8364423513412476, |
| "logits/rejected": -1.717710256576538, |
| "logps/chosen": -272.9776306152344, |
| "logps/rejected": -412.51751708984375, |
| "loss": 0.0728, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.3371310234069824, |
| "rewards/margins": 8.450685501098633, |
| "rewards/rejected": -10.787816047668457, |
| "step": 1060 |
| }, |
| { |
| "epoch": 2.2709333333333332, |
| "grad_norm": 1.2468198537826538, |
| "learning_rate": 7.403948874777556e-07, |
| "logits/chosen": -1.7665348052978516, |
| "logits/rejected": -1.694647192955017, |
| "logps/chosen": -252.20999145507812, |
| "logps/rejected": -381.0401916503906, |
| "loss": 0.0656, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9755653142929077, |
| "rewards/margins": 8.150426864624023, |
| "rewards/rejected": -10.125993728637695, |
| "step": 1065 |
| }, |
| { |
| "epoch": 2.2816, |
| "grad_norm": 0.46930360794067383, |
| "learning_rate": 7.20060672845091e-07, |
| "logits/chosen": -1.7220458984375, |
| "logits/rejected": -1.6614139080047607, |
| "logps/chosen": -274.89935302734375, |
| "logps/rejected": -415.0008239746094, |
| "loss": 0.0554, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.264709949493408, |
| "rewards/margins": 8.847898483276367, |
| "rewards/rejected": -11.112607955932617, |
| "step": 1070 |
| }, |
| { |
| "epoch": 2.2922666666666665, |
| "grad_norm": 0.8241863250732422, |
| "learning_rate": 6.99962512238733e-07, |
| "logits/chosen": -1.6326186656951904, |
| "logits/rejected": -1.541085958480835, |
| "logps/chosen": -257.67852783203125, |
| "logps/rejected": -377.48297119140625, |
| "loss": 0.0646, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0227603912353516, |
| "rewards/margins": 7.802756309509277, |
| "rewards/rejected": -9.825517654418945, |
| "step": 1075 |
| }, |
| { |
| "epoch": 2.3029333333333333, |
| "grad_norm": 2.2578377723693848, |
| "learning_rate": 6.801030710594949e-07, |
| "logits/chosen": -1.6156307458877563, |
| "logits/rejected": -1.5336544513702393, |
| "logps/chosen": -271.2424011230469, |
| "logps/rejected": -407.70916748046875, |
| "loss": 0.0802, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3306591510772705, |
| "rewards/margins": 8.529387474060059, |
| "rewards/rejected": -10.860045433044434, |
| "step": 1080 |
| }, |
| { |
| "epoch": 2.3136, |
| "grad_norm": 5.054652214050293, |
| "learning_rate": 6.604849830494264e-07, |
| "logits/chosen": -1.7466853857040405, |
| "logits/rejected": -1.686531662940979, |
| "logps/chosen": -259.6241760253906, |
| "logps/rejected": -385.6200256347656, |
| "loss": 0.0746, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.436797618865967, |
| "rewards/margins": 8.553865432739258, |
| "rewards/rejected": -10.990663528442383, |
| "step": 1085 |
| }, |
| { |
| "epoch": 2.3242666666666665, |
| "grad_norm": 0.7833718061447144, |
| "learning_rate": 6.411108499425253e-07, |
| "logits/chosen": -1.7405407428741455, |
| "logits/rejected": -1.6250250339508057, |
| "logps/chosen": -270.8252258300781, |
| "logps/rejected": -394.97406005859375, |
| "loss": 0.065, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.308990001678467, |
| "rewards/margins": 8.283408164978027, |
| "rewards/rejected": -10.592397689819336, |
| "step": 1090 |
| }, |
| { |
| "epoch": 2.3349333333333333, |
| "grad_norm": 3.1654927730560303, |
| "learning_rate": 6.219832411197013e-07, |
| "logits/chosen": -1.6691102981567383, |
| "logits/rejected": -1.56028151512146, |
| "logps/chosen": -267.05072021484375, |
| "logps/rejected": -386.01934814453125, |
| "loss": 0.0988, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.091330051422119, |
| "rewards/margins": 8.160810470581055, |
| "rewards/rejected": -10.2521390914917, |
| "step": 1095 |
| }, |
| { |
| "epoch": 2.3456, |
| "grad_norm": 0.874729335308075, |
| "learning_rate": 6.031046932680229e-07, |
| "logits/chosen": -1.7366691827774048, |
| "logits/rejected": -1.633628249168396, |
| "logps/chosen": -263.6493225097656, |
| "logps/rejected": -395.6670227050781, |
| "loss": 0.061, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.301298141479492, |
| "rewards/margins": 8.136720657348633, |
| "rewards/rejected": -10.438018798828125, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.3562666666666665, |
| "grad_norm": 2.866297483444214, |
| "learning_rate": 5.844777100443091e-07, |
| "logits/chosen": -1.7248958349227905, |
| "logits/rejected": -1.6350904703140259, |
| "logps/chosen": -286.45098876953125, |
| "logps/rejected": -419.8631286621094, |
| "loss": 0.0935, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.488067388534546, |
| "rewards/margins": 8.331172943115234, |
| "rewards/rejected": -10.81924057006836, |
| "step": 1105 |
| }, |
| { |
| "epoch": 2.3669333333333333, |
| "grad_norm": 1.4425195455551147, |
| "learning_rate": 5.66104761743092e-07, |
| "logits/chosen": -1.8552663326263428, |
| "logits/rejected": -1.7886766195297241, |
| "logps/chosen": -256.6435241699219, |
| "logps/rejected": -435.21502685546875, |
| "loss": 0.0306, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.3535618782043457, |
| "rewards/margins": 9.712784767150879, |
| "rewards/rejected": -12.066347122192383, |
| "step": 1110 |
| }, |
| { |
| "epoch": 2.3776, |
| "grad_norm": 5.315279960632324, |
| "learning_rate": 5.479882849690102e-07, |
| "logits/chosen": -1.6617796421051025, |
| "logits/rejected": -1.6377900838851929, |
| "logps/chosen": -238.333740234375, |
| "logps/rejected": -358.6871032714844, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.9535621404647827, |
| "rewards/margins": 7.863764762878418, |
| "rewards/rejected": -9.817326545715332, |
| "step": 1115 |
| }, |
| { |
| "epoch": 2.3882666666666665, |
| "grad_norm": 4.480486869812012, |
| "learning_rate": 5.301306823136687e-07, |
| "logits/chosen": -1.693786382675171, |
| "logits/rejected": -1.5776554346084595, |
| "logps/chosen": -259.76934814453125, |
| "logps/rejected": -386.46551513671875, |
| "loss": 0.0925, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.3971920013427734, |
| "rewards/margins": 7.947735786437988, |
| "rewards/rejected": -10.344927787780762, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.3989333333333334, |
| "grad_norm": 2.4435911178588867, |
| "learning_rate": 5.125343220370088e-07, |
| "logits/chosen": -1.7341457605361938, |
| "logits/rejected": -1.6198333501815796, |
| "logps/chosen": -264.7253112792969, |
| "logps/rejected": -387.65765380859375, |
| "loss": 0.0634, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.605968952178955, |
| "rewards/margins": 8.447062492370605, |
| "rewards/rejected": -11.053030967712402, |
| "step": 1125 |
| }, |
| { |
| "epoch": 2.4096, |
| "grad_norm": 0.4779101014137268, |
| "learning_rate": 4.952015377532329e-07, |
| "logits/chosen": -1.763282060623169, |
| "logits/rejected": -1.6295171976089478, |
| "logps/chosen": -272.1696472167969, |
| "logps/rejected": -430.5567932128906, |
| "loss": 0.0397, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.594783306121826, |
| "rewards/margins": 8.7913818359375, |
| "rewards/rejected": -11.386165618896484, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.4202666666666666, |
| "grad_norm": 3.4868571758270264, |
| "learning_rate": 4.781346281213204e-07, |
| "logits/chosen": -1.7784852981567383, |
| "logits/rejected": -1.7182724475860596, |
| "logps/chosen": -253.816650390625, |
| "logps/rejected": -415.3417053222656, |
| "loss": 0.0583, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5662384033203125, |
| "rewards/margins": 9.169561386108398, |
| "rewards/rejected": -11.735798835754395, |
| "step": 1135 |
| }, |
| { |
| "epoch": 2.4309333333333334, |
| "grad_norm": 1.0860471725463867, |
| "learning_rate": 4.613358565401818e-07, |
| "logits/chosen": -1.735467553138733, |
| "logits/rejected": -1.5868717432022095, |
| "logps/chosen": -281.07281494140625, |
| "logps/rejected": -403.81378173828125, |
| "loss": 0.0974, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.6185271739959717, |
| "rewards/margins": 8.223875045776367, |
| "rewards/rejected": -10.84240436553955, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.4416, |
| "grad_norm": 2.8024306297302246, |
| "learning_rate": 4.448074508484906e-07, |
| "logits/chosen": -1.7682955265045166, |
| "logits/rejected": -1.626478910446167, |
| "logps/chosen": -276.26904296875, |
| "logps/rejected": -389.8146057128906, |
| "loss": 0.0643, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2558016777038574, |
| "rewards/margins": 7.577065467834473, |
| "rewards/rejected": -9.832866668701172, |
| "step": 1145 |
| }, |
| { |
| "epoch": 2.4522666666666666, |
| "grad_norm": 4.152536869049072, |
| "learning_rate": 4.2855160302922777e-07, |
| "logits/chosen": -1.692818284034729, |
| "logits/rejected": -1.6406266689300537, |
| "logps/chosen": -266.7679138183594, |
| "logps/rejected": -426.93035888671875, |
| "loss": 0.0647, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.1690187454223633, |
| "rewards/margins": 9.473871231079102, |
| "rewards/rejected": -11.642890930175781, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.4629333333333334, |
| "grad_norm": 2.232576608657837, |
| "learning_rate": 4.125704689189819e-07, |
| "logits/chosen": -1.71976637840271, |
| "logits/rejected": -1.531394362449646, |
| "logps/chosen": -311.14019775390625, |
| "logps/rejected": -393.14801025390625, |
| "loss": 0.0587, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.7771999835968018, |
| "rewards/margins": 8.577513694763184, |
| "rewards/rejected": -10.354714393615723, |
| "step": 1155 |
| }, |
| { |
| "epoch": 2.4736000000000002, |
| "grad_norm": 0.38305553793907166, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.7162336111068726, |
| "logits/rejected": -1.6649755239486694, |
| "logps/chosen": -254.1268310546875, |
| "logps/rejected": -379.88067626953125, |
| "loss": 0.0335, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.962572693824768, |
| "rewards/margins": 8.603181838989258, |
| "rewards/rejected": -10.565755844116211, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.4842666666666666, |
| "grad_norm": 1.2464972734451294, |
| "learning_rate": 3.8144078272934676e-07, |
| "logits/chosen": -1.740289330482483, |
| "logits/rejected": -1.6422460079193115, |
| "logps/chosen": -276.12921142578125, |
| "logps/rejected": -404.734130859375, |
| "loss": 0.0628, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.208197832107544, |
| "rewards/margins": 8.343481063842773, |
| "rewards/rejected": -10.551678657531738, |
| "step": 1165 |
| }, |
| { |
| "epoch": 2.4949333333333334, |
| "grad_norm": 1.7205370664596558, |
| "learning_rate": 3.6629635904222943e-07, |
| "logits/chosen": -1.7128251791000366, |
| "logits/rejected": -1.6331779956817627, |
| "logps/chosen": -248.5833282470703, |
| "logps/rejected": -392.1952819824219, |
| "loss": 0.0736, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.1602909564971924, |
| "rewards/margins": 8.458908081054688, |
| "rewards/rejected": -10.619199752807617, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.5056000000000003, |
| "grad_norm": 2.6853506565093994, |
| "learning_rate": 3.514349053011698e-07, |
| "logits/chosen": -1.6488935947418213, |
| "logits/rejected": -1.5202906131744385, |
| "logps/chosen": -284.69122314453125, |
| "logps/rejected": -407.3353576660156, |
| "loss": 0.1129, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.177077531814575, |
| "rewards/margins": 8.398216247558594, |
| "rewards/rejected": -10.575294494628906, |
| "step": 1175 |
| }, |
| { |
| "epoch": 2.5162666666666667, |
| "grad_norm": 3.8878862857818604, |
| "learning_rate": 3.3685839241941133e-07, |
| "logits/chosen": -1.6800882816314697, |
| "logits/rejected": -1.6675255298614502, |
| "logps/chosen": -240.10922241210938, |
| "logps/rejected": -375.624267578125, |
| "loss": 0.0799, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.040280818939209, |
| "rewards/margins": 8.62706184387207, |
| "rewards/rejected": -10.667341232299805, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.5269333333333335, |
| "grad_norm": 1.2384514808654785, |
| "learning_rate": 3.2256875352158435e-07, |
| "logits/chosen": -1.7015600204467773, |
| "logits/rejected": -1.6122848987579346, |
| "logps/chosen": -279.6901550292969, |
| "logps/rejected": -418.06622314453125, |
| "loss": 0.0642, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.619711399078369, |
| "rewards/margins": 8.104384422302246, |
| "rewards/rejected": -10.72409439086914, |
| "step": 1185 |
| }, |
| { |
| "epoch": 2.5376, |
| "grad_norm": 1.836496114730835, |
| "learning_rate": 3.085678836873399e-07, |
| "logits/chosen": -1.7416712045669556, |
| "logits/rejected": -1.635202169418335, |
| "logps/chosen": -263.14776611328125, |
| "logps/rejected": -392.9107971191406, |
| "loss": 0.0629, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4201416969299316, |
| "rewards/margins": 8.608360290527344, |
| "rewards/rejected": -11.028502464294434, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.5482666666666667, |
| "grad_norm": 2.9101390838623047, |
| "learning_rate": 2.9485763970002106e-07, |
| "logits/chosen": -1.7004315853118896, |
| "logits/rejected": -1.5977933406829834, |
| "logps/chosen": -273.3360595703125, |
| "logps/rejected": -400.28741455078125, |
| "loss": 0.0519, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.7801859378814697, |
| "rewards/margins": 8.464853286743164, |
| "rewards/rejected": -11.245038986206055, |
| "step": 1195 |
| }, |
| { |
| "epoch": 2.558933333333333, |
| "grad_norm": 2.5379421710968018, |
| "learning_rate": 2.8143983980042174e-07, |
| "logits/chosen": -1.78177011013031, |
| "logits/rejected": -1.6268895864486694, |
| "logps/chosen": -261.5065002441406, |
| "logps/rejected": -413.2015686035156, |
| "loss": 0.0682, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.6103978157043457, |
| "rewards/margins": 8.457944869995117, |
| "rewards/rejected": -11.068342208862305, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.5696, |
| "grad_norm": 1.3991801738739014, |
| "learning_rate": 2.683162634456515e-07, |
| "logits/chosen": -1.585166335105896, |
| "logits/rejected": -1.547821283340454, |
| "logps/chosen": -265.8369445800781, |
| "logps/rejected": -381.36395263671875, |
| "loss": 0.058, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.0374863147735596, |
| "rewards/margins": 8.10401725769043, |
| "rewards/rejected": -10.141504287719727, |
| "step": 1205 |
| }, |
| { |
| "epoch": 2.5802666666666667, |
| "grad_norm": 1.468181848526001, |
| "learning_rate": 2.5548865107314606e-07, |
| "logits/chosen": -1.6954666376113892, |
| "logits/rejected": -1.5682141780853271, |
| "logps/chosen": -300.32867431640625, |
| "logps/rejected": -409.02191162109375, |
| "loss": 0.0738, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4429965019226074, |
| "rewards/margins": 8.085245132446289, |
| "rewards/rejected": -10.528241157531738, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.590933333333333, |
| "grad_norm": 1.9603363275527954, |
| "learning_rate": 2.429587038698511e-07, |
| "logits/chosen": -1.6946890354156494, |
| "logits/rejected": -1.6248438358306885, |
| "logps/chosen": -250.2937774658203, |
| "logps/rejected": -393.65191650390625, |
| "loss": 0.0931, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.27023983001709, |
| "rewards/margins": 8.228813171386719, |
| "rewards/rejected": -10.499053001403809, |
| "step": 1215 |
| }, |
| { |
| "epoch": 2.6016, |
| "grad_norm": 2.1774659156799316, |
| "learning_rate": 2.307280835466144e-07, |
| "logits/chosen": -1.6663267612457275, |
| "logits/rejected": -1.6113440990447998, |
| "logps/chosen": -252.66366577148438, |
| "logps/rejected": -393.9212646484375, |
| "loss": 0.0488, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.4190704822540283, |
| "rewards/margins": 8.709858894348145, |
| "rewards/rejected": -11.128928184509277, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.6122666666666667, |
| "grad_norm": 2.0370330810546875, |
| "learning_rate": 2.1879841211780856e-07, |
| "logits/chosen": -1.6466172933578491, |
| "logits/rejected": -1.5969603061676025, |
| "logps/chosen": -271.2088623046875, |
| "logps/rejected": -395.32940673828125, |
| "loss": 0.1036, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.6539084911346436, |
| "rewards/margins": 8.313491821289062, |
| "rewards/rejected": -10.967400550842285, |
| "step": 1225 |
| }, |
| { |
| "epoch": 2.622933333333333, |
| "grad_norm": 1.2966225147247314, |
| "learning_rate": 2.0717127168622202e-07, |
| "logits/chosen": -1.7209373712539673, |
| "logits/rejected": -1.6275761127471924, |
| "logps/chosen": -269.34234619140625, |
| "logps/rejected": -388.9476623535156, |
| "loss": 0.0974, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.696427822113037, |
| "rewards/margins": 8.036722183227539, |
| "rewards/rejected": -10.733149528503418, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.6336, |
| "grad_norm": 2.5588059425354004, |
| "learning_rate": 1.958482042332424e-07, |
| "logits/chosen": -1.6694538593292236, |
| "logits/rejected": -1.5491987466812134, |
| "logps/chosen": -286.1966552734375, |
| "logps/rejected": -389.8074951171875, |
| "loss": 0.1192, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.7213234901428223, |
| "rewards/margins": 8.268573760986328, |
| "rewards/rejected": -10.989896774291992, |
| "step": 1235 |
| }, |
| { |
| "epoch": 2.6442666666666668, |
| "grad_norm": 3.664574384689331, |
| "learning_rate": 1.8483071141435937e-07, |
| "logits/chosen": -1.6609315872192383, |
| "logits/rejected": -1.603724718093872, |
| "logps/chosen": -282.61090087890625, |
| "logps/rejected": -388.0448913574219, |
| "loss": 0.0811, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.6346335411071777, |
| "rewards/margins": 7.658135890960693, |
| "rewards/rejected": -10.292769432067871, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.654933333333333, |
| "grad_norm": 6.937290191650391, |
| "learning_rate": 1.741202543600176e-07, |
| "logits/chosen": -1.7843129634857178, |
| "logits/rejected": -1.6339008808135986, |
| "logps/chosen": -261.72698974609375, |
| "logps/rejected": -393.80853271484375, |
| "loss": 0.1399, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.6746315956115723, |
| "rewards/margins": 7.8194074630737305, |
| "rewards/rejected": -10.494039535522461, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.6656, |
| "grad_norm": 1.8684526681900024, |
| "learning_rate": 1.6371825348184155e-07, |
| "logits/chosen": -1.6750285625457764, |
| "logits/rejected": -1.6143300533294678, |
| "logps/chosen": -280.13720703125, |
| "logps/rejected": -439.83770751953125, |
| "loss": 0.0874, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.390502452850342, |
| "rewards/margins": 9.048379898071289, |
| "rewards/rejected": -11.438881874084473, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.676266666666667, |
| "grad_norm": 3.4125890731811523, |
| "learning_rate": 1.5362608828426246e-07, |
| "logits/chosen": -1.692704200744629, |
| "logits/rejected": -1.614297866821289, |
| "logps/chosen": -273.0475158691406, |
| "logps/rejected": -390.3589172363281, |
| "loss": 0.105, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.143181800842285, |
| "rewards/margins": 7.783265113830566, |
| "rewards/rejected": -9.926446914672852, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.686933333333333, |
| "grad_norm": 6.792511940002441, |
| "learning_rate": 1.438450971815686e-07, |
| "logits/chosen": -1.777464509010315, |
| "logits/rejected": -1.7350318431854248, |
| "logps/chosen": -249.89877319335938, |
| "logps/rejected": -405.6546630859375, |
| "loss": 0.0653, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.4070210456848145, |
| "rewards/margins": 8.65117359161377, |
| "rewards/rejected": -11.058195114135742, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.6976, |
| "grad_norm": 4.641789436340332, |
| "learning_rate": 1.3437657732040783e-07, |
| "logits/chosen": -1.6777312755584717, |
| "logits/rejected": -1.5757746696472168, |
| "logps/chosen": -279.8283996582031, |
| "logps/rejected": -402.06903076171875, |
| "loss": 0.0696, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8644824028015137, |
| "rewards/margins": 8.306800842285156, |
| "rewards/rejected": -11.171284675598145, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.708266666666667, |
| "grad_norm": 3.1361374855041504, |
| "learning_rate": 1.2522178440776022e-07, |
| "logits/chosen": -1.72237229347229, |
| "logits/rejected": -1.6854991912841797, |
| "logps/chosen": -258.05682373046875, |
| "logps/rejected": -367.5188903808594, |
| "loss": 0.0931, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.406632661819458, |
| "rewards/margins": 7.722708225250244, |
| "rewards/rejected": -10.129342079162598, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.718933333333333, |
| "grad_norm": 0.9646216630935669, |
| "learning_rate": 1.1638193254440655e-07, |
| "logits/chosen": -1.7326161861419678, |
| "logits/rejected": -1.64824640750885, |
| "logps/chosen": -266.62567138671875, |
| "logps/rejected": -391.803466796875, |
| "loss": 0.072, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.726926803588867, |
| "rewards/margins": 7.8639631271362305, |
| "rewards/rejected": -10.590889930725098, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.7296, |
| "grad_norm": 1.7434144020080566, |
| "learning_rate": 1.0785819406391562e-07, |
| "logits/chosen": -1.652780532836914, |
| "logits/rejected": -1.623931646347046, |
| "logps/chosen": -256.96063232421875, |
| "logps/rejected": -414.2125549316406, |
| "loss": 0.0834, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.3380179405212402, |
| "rewards/margins": 9.076375007629395, |
| "rewards/rejected": -11.414393424987793, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.740266666666667, |
| "grad_norm": 0.6346848607063293, |
| "learning_rate": 9.965169937717095e-08, |
| "logits/chosen": -1.7070586681365967, |
| "logits/rejected": -1.6493428945541382, |
| "logps/chosen": -265.99652099609375, |
| "logps/rejected": -411.8456115722656, |
| "loss": 0.0846, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.9297215938568115, |
| "rewards/margins": 8.40210247039795, |
| "rewards/rejected": -11.331823348999023, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.7509333333333332, |
| "grad_norm": 3.758061647415161, |
| "learning_rate": 9.176353682245676e-08, |
| "logits/chosen": -1.7081968784332275, |
| "logits/rejected": -1.6483770608901978, |
| "logps/chosen": -249.968994140625, |
| "logps/rejected": -401.0195617675781, |
| "loss": 0.0668, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.510201930999756, |
| "rewards/margins": 8.664570808410645, |
| "rewards/rejected": -11.174773216247559, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.7616, |
| "grad_norm": 2.5584027767181396, |
| "learning_rate": 8.41947525211223e-08, |
| "logits/chosen": -1.6588901281356812, |
| "logits/rejected": -1.6179072856903076, |
| "logps/chosen": -270.0908203125, |
| "logps/rejected": -404.5058898925781, |
| "loss": 0.0701, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3527674674987793, |
| "rewards/margins": 8.176429748535156, |
| "rewards/rejected": -10.529196739196777, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.772266666666667, |
| "grad_norm": 0.7949896454811096, |
| "learning_rate": 7.694635023884789e-08, |
| "logits/chosen": -1.6635520458221436, |
| "logits/rejected": -1.633556604385376, |
| "logps/chosen": -273.2143249511719, |
| "logps/rejected": -411.2735900878906, |
| "loss": 0.0689, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.589845895767212, |
| "rewards/margins": 8.327569007873535, |
| "rewards/rejected": -10.917413711547852, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7829333333333333, |
| "grad_norm": 1.5558366775512695, |
| "learning_rate": 7.00192912525241e-08, |
| "logits/chosen": -1.7068121433258057, |
| "logits/rejected": -1.60091233253479, |
| "logps/chosen": -275.8150939941406, |
| "logps/rejected": -400.9371643066406, |
| "loss": 0.0792, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.610804557800293, |
| "rewards/margins": 8.357553482055664, |
| "rewards/rejected": -10.968358993530273, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.7936, |
| "grad_norm": 3.2928590774536133, |
| "learning_rate": 6.341449422277047e-08, |
| "logits/chosen": -1.673627495765686, |
| "logits/rejected": -1.665616750717163, |
| "logps/chosen": -286.53314208984375, |
| "logps/rejected": -410.33099365234375, |
| "loss": 0.0881, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.5637083053588867, |
| "rewards/margins": 7.853028774261475, |
| "rewards/rejected": -10.416738510131836, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.804266666666667, |
| "grad_norm": 2.260819911956787, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.6774076223373413, |
| "logits/rejected": -1.676124930381775, |
| "logps/chosen": -273.211669921875, |
| "logps/rejected": -410.07977294921875, |
| "loss": 0.058, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.765542507171631, |
| "rewards/margins": 8.562185287475586, |
| "rewards/rejected": -11.327728271484375, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.8149333333333333, |
| "grad_norm": 3.5438122749328613, |
| "learning_rate": 5.117514686876379e-08, |
| "logits/chosen": -1.6422332525253296, |
| "logits/rejected": -1.5864102840423584, |
| "logps/chosen": -244.44949340820312, |
| "logps/rejected": -404.03802490234375, |
| "loss": 0.0853, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.003594160079956, |
| "rewards/margins": 8.986794471740723, |
| "rewards/rejected": -10.990389823913574, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.8256, |
| "grad_norm": 2.5405335426330566, |
| "learning_rate": 4.5542219716255274e-08, |
| "logits/chosen": -1.7319309711456299, |
| "logits/rejected": -1.6355736255645752, |
| "logps/chosen": -290.3944396972656, |
| "logps/rejected": -438.39697265625, |
| "loss": 0.0431, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.727210760116577, |
| "rewards/margins": 9.366616249084473, |
| "rewards/rejected": -12.093827247619629, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.836266666666667, |
| "grad_norm": 0.5261633992195129, |
| "learning_rate": 4.023480064854174e-08, |
| "logits/chosen": -1.783346176147461, |
| "logits/rejected": -1.6483442783355713, |
| "logps/chosen": -250.863525390625, |
| "logps/rejected": -378.8450927734375, |
| "loss": 0.0649, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.479001522064209, |
| "rewards/margins": 8.179192543029785, |
| "rewards/rejected": -10.658193588256836, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.8469333333333333, |
| "grad_norm": 0.4898112714290619, |
| "learning_rate": 3.525359353098584e-08, |
| "logits/chosen": -1.7240718603134155, |
| "logits/rejected": -1.593396544456482, |
| "logps/chosen": -263.9686584472656, |
| "logps/rejected": -387.5354309082031, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9066507816314697, |
| "rewards/margins": 8.4071044921875, |
| "rewards/rejected": -10.31375503540039, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.8576, |
| "grad_norm": 5.789946556091309, |
| "learning_rate": 3.059925896700094e-08, |
| "logits/chosen": -1.6261345148086548, |
| "logits/rejected": -1.5245052576065063, |
| "logps/chosen": -279.90728759765625, |
| "logps/rejected": -391.10687255859375, |
| "loss": 0.1376, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.2622745037078857, |
| "rewards/margins": 8.004220962524414, |
| "rewards/rejected": -10.266495704650879, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.8682666666666665, |
| "grad_norm": 4.288834571838379, |
| "learning_rate": 2.6272414210443964e-08, |
| "logits/chosen": -1.6331889629364014, |
| "logits/rejected": -1.5616645812988281, |
| "logps/chosen": -285.0086669921875, |
| "logps/rejected": -380.7625427246094, |
| "loss": 0.0822, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.7788119316101074, |
| "rewards/margins": 8.204727172851562, |
| "rewards/rejected": -10.983540534973145, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.8789333333333333, |
| "grad_norm": 2.1750404834747314, |
| "learning_rate": 2.2273633083753632e-08, |
| "logits/chosen": -1.6630783081054688, |
| "logits/rejected": -1.647499680519104, |
| "logps/chosen": -246.02957153320312, |
| "logps/rejected": -382.4586486816406, |
| "loss": 0.0665, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2752842903137207, |
| "rewards/margins": 8.254133224487305, |
| "rewards/rejected": -10.5294189453125, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.8895999999999997, |
| "grad_norm": 1.8180460929870605, |
| "learning_rate": 1.8603445901851647e-08, |
| "logits/chosen": -1.7276430130004883, |
| "logits/rejected": -1.6383469104766846, |
| "logps/chosen": -262.91595458984375, |
| "logps/rejected": -414.97955322265625, |
| "loss": 0.0761, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.588278293609619, |
| "rewards/margins": 8.917512893676758, |
| "rewards/rejected": -11.505790710449219, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.9002666666666665, |
| "grad_norm": 3.9006402492523193, |
| "learning_rate": 1.5262339401813375e-08, |
| "logits/chosen": -1.7000020742416382, |
| "logits/rejected": -1.6428985595703125, |
| "logps/chosen": -253.78414916992188, |
| "logps/rejected": -418.7207946777344, |
| "loss": 0.0732, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5373544692993164, |
| "rewards/margins": 8.902445793151855, |
| "rewards/rejected": -11.439800262451172, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.9109333333333334, |
| "grad_norm": 4.6902689933776855, |
| "learning_rate": 1.2250756678315601e-08, |
| "logits/chosen": -1.743619680404663, |
| "logits/rejected": -1.6624910831451416, |
| "logps/chosen": -287.44415283203125, |
| "logps/rejected": -420.8153381347656, |
| "loss": 0.0475, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5243899822235107, |
| "rewards/margins": 8.417469024658203, |
| "rewards/rejected": -10.941858291625977, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.9215999999999998, |
| "grad_norm": 3.476588726043701, |
| "learning_rate": 9.569097124875193e-09, |
| "logits/chosen": -1.6620032787322998, |
| "logits/rejected": -1.669245958328247, |
| "logps/chosen": -270.68609619140625, |
| "logps/rejected": -412.59857177734375, |
| "loss": 0.105, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.7364907264709473, |
| "rewards/margins": 8.751927375793457, |
| "rewards/rejected": -11.488417625427246, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.9322666666666666, |
| "grad_norm": 2.2454278469085693, |
| "learning_rate": 7.217716380881479e-09, |
| "logits/chosen": -1.6574100255966187, |
| "logits/rejected": -1.654077172279358, |
| "logps/chosen": -282.48870849609375, |
| "logps/rejected": -396.88031005859375, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.570164203643799, |
| "rewards/margins": 8.327180862426758, |
| "rewards/rejected": -10.897344589233398, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.9429333333333334, |
| "grad_norm": 1.074591040611267, |
| "learning_rate": 5.196926284430359e-09, |
| "logits/chosen": -1.6404346227645874, |
| "logits/rejected": -1.5578677654266357, |
| "logps/chosen": -256.85791015625, |
| "logps/rejected": -386.71575927734375, |
| "loss": 0.0563, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.627209424972534, |
| "rewards/margins": 7.910440921783447, |
| "rewards/rejected": -10.537651062011719, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.9536, |
| "grad_norm": 1.59853994846344, |
| "learning_rate": 3.5069948309712775e-09, |
| "logits/chosen": -1.651651382446289, |
| "logits/rejected": -1.5908164978027344, |
| "logps/chosen": -271.5132141113281, |
| "logps/rejected": -393.08148193359375, |
| "loss": 0.0829, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2630515098571777, |
| "rewards/margins": 7.777309417724609, |
| "rewards/rejected": -10.040361404418945, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.9642666666666666, |
| "grad_norm": 1.6438560485839844, |
| "learning_rate": 2.1481461377634294e-09, |
| "logits/chosen": -1.6835439205169678, |
| "logits/rejected": -1.664738416671753, |
| "logps/chosen": -273.68212890625, |
| "logps/rejected": -414.2811584472656, |
| "loss": 0.0402, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.379215955734253, |
| "rewards/margins": 9.452581405639648, |
| "rewards/rejected": -11.831796646118164, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.9749333333333334, |
| "grad_norm": 2.484712600708008, |
| "learning_rate": 1.120560414154259e-09, |
| "logits/chosen": -1.7078514099121094, |
| "logits/rejected": -1.516570806503296, |
| "logps/chosen": -251.81283569335938, |
| "logps/rejected": -385.6812744140625, |
| "loss": 0.0671, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3888099193573, |
| "rewards/margins": 8.810364723205566, |
| "rewards/rejected": -11.199174880981445, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.9856, |
| "grad_norm": 1.5737336874008179, |
| "learning_rate": 4.2437393768079983e-10, |
| "logits/chosen": -1.609192132949829, |
| "logits/rejected": -1.5943609476089478, |
| "logps/chosen": -266.55340576171875, |
| "logps/rejected": -393.8858947753906, |
| "loss": 0.0634, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4015676975250244, |
| "rewards/margins": 8.4376859664917, |
| "rewards/rejected": -10.839254379272461, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.9962666666666666, |
| "grad_norm": 2.8753819465637207, |
| "learning_rate": 5.967903599635171e-11, |
| "logits/chosen": -1.6768028736114502, |
| "logits/rejected": -1.5742844343185425, |
| "logps/chosen": -267.65020751953125, |
| "logps/rejected": -438.10845947265625, |
| "loss": 0.0667, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.351564884185791, |
| "rewards/margins": 9.339567184448242, |
| "rewards/rejected": -11.691132545471191, |
| "step": 1405 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1407, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
