| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1407, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 3.040982484817505, |
| "learning_rate": 4.651162790697675e-07, |
| "logits/chosen": -1.4189865589141846, |
| "logits/rejected": -1.2718557119369507, |
| "logps/chosen": -232.1872100830078, |
| "logps/rejected": -285.8193359375, |
| "loss": 0.693, |
| "rewards/accuracies": 0.3187499940395355, |
| "rewards/chosen": -0.0001921796938404441, |
| "rewards/margins": 0.00023017881903797388, |
| "rewards/rejected": -0.0004223585710860789, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 2.945732355117798, |
| "learning_rate": 1.0465116279069768e-06, |
| "logits/chosen": -1.5402610301971436, |
| "logits/rejected": -1.3184393644332886, |
| "logps/chosen": -230.44815063476562, |
| "logps/rejected": -273.22674560546875, |
| "loss": 0.69, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.00048116204561665654, |
| "rewards/margins": 0.006424086634069681, |
| "rewards/rejected": -0.0069052488543093204, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 3.0436723232269287, |
| "learning_rate": 1.6279069767441862e-06, |
| "logits/chosen": -1.4266681671142578, |
| "logits/rejected": -1.2699027061462402, |
| "logps/chosen": -214.630859375, |
| "logps/rejected": -266.53839111328125, |
| "loss": 0.684, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.002722587902098894, |
| "rewards/margins": 0.018564948812127113, |
| "rewards/rejected": -0.02128753624856472, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 3.2988665103912354, |
| "learning_rate": 2.2093023255813954e-06, |
| "logits/chosen": -1.3134950399398804, |
| "logits/rejected": -1.1439664363861084, |
| "logps/chosen": -224.536376953125, |
| "logps/rejected": -263.8288269042969, |
| "loss": 0.6772, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.005389872007071972, |
| "rewards/margins": 0.03269362449645996, |
| "rewards/rejected": -0.03808349743485451, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 2.985452890396118, |
| "learning_rate": 2.790697674418605e-06, |
| "logits/chosen": -1.4989473819732666, |
| "logits/rejected": -1.2572524547576904, |
| "logps/chosen": -206.94247436523438, |
| "logps/rejected": -274.64508056640625, |
| "loss": 0.6598, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.002713642315939069, |
| "rewards/margins": 0.0694461390376091, |
| "rewards/rejected": -0.0721597820520401, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 3.0096566677093506, |
| "learning_rate": 3.372093023255814e-06, |
| "logits/chosen": -1.3679146766662598, |
| "logits/rejected": -1.1917188167572021, |
| "logps/chosen": -244.6914825439453, |
| "logps/rejected": -277.0018310546875, |
| "loss": 0.6441, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.0334165021777153, |
| "rewards/margins": 0.10440608114004135, |
| "rewards/rejected": -0.13782256841659546, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 3.8459115028381348, |
| "learning_rate": 3.953488372093024e-06, |
| "logits/chosen": -1.4499976634979248, |
| "logits/rejected": -1.2488772869110107, |
| "logps/chosen": -236.5290069580078, |
| "logps/rejected": -287.55169677734375, |
| "loss": 0.6063, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.033719420433044434, |
| "rewards/margins": 0.19089257717132568, |
| "rewards/rejected": -0.2246120274066925, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 2.8168575763702393, |
| "learning_rate": 4.5348837209302326e-06, |
| "logits/chosen": -1.4405505657196045, |
| "logits/rejected": -1.2370808124542236, |
| "logps/chosen": -234.57730102539062, |
| "logps/rejected": -293.7424621582031, |
| "loss": 0.594, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.06413023173809052, |
| "rewards/margins": 0.22726896405220032, |
| "rewards/rejected": -0.291399210691452, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 3.0759992599487305, |
| "learning_rate": 4.99999336897255e-06, |
| "logits/chosen": -1.4353830814361572, |
| "logits/rejected": -1.2781504392623901, |
| "logps/chosen": -232.90866088867188, |
| "logps/rejected": -282.9027404785156, |
| "loss": 0.5602, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.07559499889612198, |
| "rewards/margins": 0.31765851378440857, |
| "rewards/rejected": -0.39325350522994995, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 3.0262410640716553, |
| "learning_rate": 4.999761286705285e-06, |
| "logits/chosen": -1.5332512855529785, |
| "logits/rejected": -1.4040536880493164, |
| "logps/chosen": -221.7960662841797, |
| "logps/rejected": -294.1759033203125, |
| "loss": 0.4985, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.04457804188132286, |
| "rewards/margins": 0.4884178638458252, |
| "rewards/rejected": -0.5329959392547607, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 3.2148373126983643, |
| "learning_rate": 4.999197688241076e-06, |
| "logits/chosen": -1.4515631198883057, |
| "logits/rejected": -1.2591352462768555, |
| "logps/chosen": -216.49111938476562, |
| "logps/rejected": -275.7051086425781, |
| "loss": 0.4696, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.11984169483184814, |
| "rewards/margins": 0.6132544279098511, |
| "rewards/rejected": -0.7330960035324097, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 2.543278694152832, |
| "learning_rate": 4.9983026483238685e-06, |
| "logits/chosen": -1.5313522815704346, |
| "logits/rejected": -1.3514273166656494, |
| "logps/chosen": -221.7779998779297, |
| "logps/rejected": -289.889404296875, |
| "loss": 0.4535, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.18022173643112183, |
| "rewards/margins": 0.6861358880996704, |
| "rewards/rejected": -0.8663576245307922, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 2.6283793449401855, |
| "learning_rate": 4.997076285653087e-06, |
| "logits/chosen": -1.5744426250457764, |
| "logits/rejected": -1.3084551095962524, |
| "logps/chosen": -235.96768188476562, |
| "logps/rejected": -293.5603942871094, |
| "loss": 0.4044, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.23008236289024353, |
| "rewards/margins": 0.9606491923332214, |
| "rewards/rejected": -1.1907316446304321, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 2.195939064025879, |
| "learning_rate": 4.995518762867897e-06, |
| "logits/chosen": -1.60664963722229, |
| "logits/rejected": -1.3430724143981934, |
| "logps/chosen": -218.2381591796875, |
| "logps/rejected": -294.97222900390625, |
| "loss": 0.359, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.27671200037002563, |
| "rewards/margins": 1.1187210083007812, |
| "rewards/rejected": -1.395432949066162, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 3.54109263420105, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -1.5840482711791992, |
| "logits/rejected": -1.3805081844329834, |
| "logps/chosen": -232.5129852294922, |
| "logps/rejected": -295.33099365234375, |
| "loss": 0.3903, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -0.47087717056274414, |
| "rewards/margins": 1.1671949625015259, |
| "rewards/rejected": -1.6380720138549805, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 1.889794945716858, |
| "learning_rate": 4.99141110707441e-06, |
| "logits/chosen": -1.570359230041504, |
| "logits/rejected": -1.364939570426941, |
| "logps/chosen": -207.35525512695312, |
| "logps/rejected": -281.3659973144531, |
| "loss": 0.3623, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.21106550097465515, |
| "rewards/margins": 1.3174269199371338, |
| "rewards/rejected": -1.5284923315048218, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 3.4203381538391113, |
| "learning_rate": 4.9888615188199e-06, |
| "logits/chosen": -1.640369176864624, |
| "logits/rejected": -1.4251915216445923, |
| "logps/chosen": -222.8439178466797, |
| "logps/rejected": -288.3473815917969, |
| "loss": 0.3774, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.3367365002632141, |
| "rewards/margins": 1.3324146270751953, |
| "rewards/rejected": -1.6691510677337646, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 3.1954681873321533, |
| "learning_rate": 4.9859818598863104e-06, |
| "logits/chosen": -1.55571711063385, |
| "logits/rejected": -1.3716849088668823, |
| "logps/chosen": -226.4812774658203, |
| "logps/rejected": -290.22906494140625, |
| "loss": 0.3714, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.3206635117530823, |
| "rewards/margins": 1.3784153461456299, |
| "rewards/rejected": -1.6990787982940674, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 2.557746648788452, |
| "learning_rate": 4.982772512171538e-06, |
| "logits/chosen": -1.6502959728240967, |
| "logits/rejected": -1.453385353088379, |
| "logps/chosen": -233.37594604492188, |
| "logps/rejected": -294.97259521484375, |
| "loss": 0.3449, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.4198358654975891, |
| "rewards/margins": 1.5545928478240967, |
| "rewards/rejected": -1.9744285345077515, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 2.5644826889038086, |
| "learning_rate": 4.979233901296523e-06, |
| "logits/chosen": -1.6967132091522217, |
| "logits/rejected": -1.5241398811340332, |
| "logps/chosen": -242.16000366210938, |
| "logps/rejected": -305.0459289550781, |
| "loss": 0.3319, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -0.45428210496902466, |
| "rewards/margins": 1.6128562688827515, |
| "rewards/rejected": -2.067138433456421, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 3.155392646789551, |
| "learning_rate": 4.975366496548804e-06, |
| "logits/chosen": -1.6549123525619507, |
| "logits/rejected": -1.411078691482544, |
| "logps/chosen": -240.0164794921875, |
| "logps/rejected": -305.4057922363281, |
| "loss": 0.2697, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.26041409373283386, |
| "rewards/margins": 1.9732940196990967, |
| "rewards/rejected": -2.233708143234253, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 2.429647445678711, |
| "learning_rate": 4.971170810820279e-06, |
| "logits/chosen": -1.6650527715682983, |
| "logits/rejected": -1.4914207458496094, |
| "logps/chosen": -222.6490478515625, |
| "logps/rejected": -295.0611877441406, |
| "loss": 0.3103, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.23167634010314941, |
| "rewards/margins": 1.8154058456420898, |
| "rewards/rejected": -2.0470821857452393, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 3.5919618606567383, |
| "learning_rate": 4.96664740053919e-06, |
| "logits/chosen": -1.6617170572280884, |
| "logits/rejected": -1.5017478466033936, |
| "logps/chosen": -220.0016326904297, |
| "logps/rejected": -293.1516418457031, |
| "loss": 0.3044, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.28731054067611694, |
| "rewards/margins": 1.8548088073730469, |
| "rewards/rejected": -2.1421194076538086, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 2.7757253646850586, |
| "learning_rate": 4.961796865596328e-06, |
| "logits/chosen": -1.6444618701934814, |
| "logits/rejected": -1.3935641050338745, |
| "logps/chosen": -221.69973754882812, |
| "logps/rejected": -286.38458251953125, |
| "loss": 0.2808, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.2497580498456955, |
| "rewards/margins": 1.8792965412139893, |
| "rewards/rejected": -2.129054546356201, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 1.7543736696243286, |
| "learning_rate": 4.956619849265473e-06, |
| "logits/chosen": -1.7066230773925781, |
| "logits/rejected": -1.5209218263626099, |
| "logps/chosen": -214.9864044189453, |
| "logps/rejected": -305.9405212402344, |
| "loss": 0.2312, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.12859247624874115, |
| "rewards/margins": 2.200359344482422, |
| "rewards/rejected": -2.3289520740509033, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 2.8612709045410156, |
| "learning_rate": 4.951117038118091e-06, |
| "logits/chosen": -1.7071641683578491, |
| "logits/rejected": -1.5247122049331665, |
| "logps/chosen": -246.58291625976562, |
| "logps/rejected": -303.3542785644531, |
| "loss": 0.2443, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.3323816657066345, |
| "rewards/margins": 2.2684006690979004, |
| "rewards/rejected": -2.6007821559906006, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 3.071027994155884, |
| "learning_rate": 4.945289161932273e-06, |
| "logits/chosen": -1.7539116144180298, |
| "logits/rejected": -1.6009010076522827, |
| "logps/chosen": -239.4345245361328, |
| "logps/rejected": -295.87200927734375, |
| "loss": 0.2486, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -0.12532970309257507, |
| "rewards/margins": 2.3635432720184326, |
| "rewards/rejected": -2.48887300491333, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 2.4682374000549316, |
| "learning_rate": 4.93913699359596e-06, |
| "logits/chosen": -1.7155869007110596, |
| "logits/rejected": -1.4812595844268799, |
| "logps/chosen": -227.5341796875, |
| "logps/rejected": -289.00030517578125, |
| "loss": 0.2207, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.25302743911743164, |
| "rewards/margins": 2.457817554473877, |
| "rewards/rejected": -2.7108449935913086, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 3.2257938385009766, |
| "learning_rate": 4.9326613490044364e-06, |
| "logits/chosen": -1.6698166131973267, |
| "logits/rejected": -1.486440896987915, |
| "logps/chosen": -238.375, |
| "logps/rejected": -297.738525390625, |
| "loss": 0.2757, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.17056556046009064, |
| "rewards/margins": 2.2673239707946777, |
| "rewards/rejected": -2.437889575958252, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 2.9405577182769775, |
| "learning_rate": 4.925863086952132e-06, |
| "logits/chosen": -1.7178294658660889, |
| "logits/rejected": -1.480749249458313, |
| "logps/chosen": -223.380615234375, |
| "logps/rejected": -290.5135803222656, |
| "loss": 0.2558, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.057668160647153854, |
| "rewards/margins": 2.5294711589813232, |
| "rewards/rejected": -2.587139129638672, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 1.0869413614273071, |
| "learning_rate": 4.918743109018727e-06, |
| "logits/chosen": -1.8033220767974854, |
| "logits/rejected": -1.534440040588379, |
| "logps/chosen": -211.77084350585938, |
| "logps/rejected": -310.13201904296875, |
| "loss": 0.1938, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.08430004119873047, |
| "rewards/margins": 2.9461827278137207, |
| "rewards/rejected": -3.030482530593872, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 1.6788111925125122, |
| "learning_rate": 4.911302359449585e-06, |
| "logits/chosen": -1.8236114978790283, |
| "logits/rejected": -1.563982367515564, |
| "logps/chosen": -213.33139038085938, |
| "logps/rejected": -300.1676940917969, |
| "loss": 0.1788, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.054857976734638214, |
| "rewards/margins": 2.9758267402648926, |
| "rewards/rejected": -3.03068470954895, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 2.340954303741455, |
| "learning_rate": 4.9035418250305314e-06, |
| "logits/chosen": -1.6831834316253662, |
| "logits/rejected": -1.4661271572113037, |
| "logps/chosen": -228.6976318359375, |
| "logps/rejected": -309.2475891113281, |
| "loss": 0.2267, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.21103031933307648, |
| "rewards/margins": 2.8673250675201416, |
| "rewards/rejected": -3.078355312347412, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 3.3007848262786865, |
| "learning_rate": 4.895462534956981e-06, |
| "logits/chosen": -1.7243316173553467, |
| "logits/rejected": -1.5944850444793701, |
| "logps/chosen": -237.75094604492188, |
| "logps/rejected": -310.4878234863281, |
| "loss": 0.1998, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.272445410490036, |
| "rewards/margins": 3.051952362060547, |
| "rewards/rejected": -3.324397563934326, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 2.133695363998413, |
| "learning_rate": 4.887065560697449e-06, |
| "logits/chosen": -1.660601258277893, |
| "logits/rejected": -1.5193674564361572, |
| "logps/chosen": -235.02200317382812, |
| "logps/rejected": -330.9132995605469, |
| "loss": 0.2288, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.2509080469608307, |
| "rewards/margins": 3.1449391841888428, |
| "rewards/rejected": -3.3958473205566406, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 2.8581511974334717, |
| "learning_rate": 4.878352015851459e-06, |
| "logits/chosen": -1.7492008209228516, |
| "logits/rejected": -1.5329986810684204, |
| "logps/chosen": -305.83502197265625, |
| "logps/rejected": -326.94097900390625, |
| "loss": 0.2059, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5418329238891602, |
| "rewards/margins": 3.5084152221679688, |
| "rewards/rejected": -4.050248146057129, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 1.8751473426818848, |
| "learning_rate": 4.869323056001852e-06, |
| "logits/chosen": -1.639556884765625, |
| "logits/rejected": -1.5379990339279175, |
| "logps/chosen": -244.7671356201172, |
| "logps/rejected": -285.7706298828125, |
| "loss": 0.3078, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.567600429058075, |
| "rewards/margins": 2.8157169818878174, |
| "rewards/rejected": -3.383317470550537, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 2.4277961254119873, |
| "learning_rate": 4.859979878561536e-06, |
| "logits/chosen": -1.7614755630493164, |
| "logits/rejected": -1.5140868425369263, |
| "logps/chosen": -229.64779663085938, |
| "logps/rejected": -282.7133483886719, |
| "loss": 0.2575, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.49064379930496216, |
| "rewards/margins": 3.128330945968628, |
| "rewards/rejected": -3.6189746856689453, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 3.492586612701416, |
| "learning_rate": 4.850323722614685e-06, |
| "logits/chosen": -1.8276838064193726, |
| "logits/rejected": -1.6381219625473022, |
| "logps/chosen": -224.7960205078125, |
| "logps/rejected": -312.1609191894531, |
| "loss": 0.2089, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.38933640718460083, |
| "rewards/margins": 3.459027051925659, |
| "rewards/rejected": -3.8483633995056152, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 2.13981556892395, |
| "learning_rate": 4.840355868752416e-06, |
| "logits/chosen": -1.7009217739105225, |
| "logits/rejected": -1.4874917268753052, |
| "logps/chosen": -281.18817138671875, |
| "logps/rejected": -294.76593017578125, |
| "loss": 0.277, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.39215925335884094, |
| "rewards/margins": 2.851746082305908, |
| "rewards/rejected": -3.2439048290252686, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 1.7804917097091675, |
| "learning_rate": 4.830077638902955e-06, |
| "logits/chosen": -1.8614517450332642, |
| "logits/rejected": -1.6383779048919678, |
| "logps/chosen": -205.98171997070312, |
| "logps/rejected": -303.1836853027344, |
| "loss": 0.1783, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.43484029173851013, |
| "rewards/margins": 3.4785313606262207, |
| "rewards/rejected": -3.9133713245391846, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 3.0044808387756348, |
| "learning_rate": 4.8194903961563255e-06, |
| "logits/chosen": -1.9666965007781982, |
| "logits/rejected": -1.7533382177352905, |
| "logps/chosen": -202.64224243164062, |
| "logps/rejected": -304.18115234375, |
| "loss": 0.1671, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.2894894480705261, |
| "rewards/margins": 3.478058338165283, |
| "rewards/rejected": -3.767547607421875, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 2.3777949810028076, |
| "learning_rate": 4.808595544583578e-06, |
| "logits/chosen": -1.802603006362915, |
| "logits/rejected": -1.5886794328689575, |
| "logps/chosen": -240.7107391357422, |
| "logps/rejected": -315.6220397949219, |
| "loss": 0.1541, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.681304395198822, |
| "rewards/margins": 3.757567882537842, |
| "rewards/rejected": -4.438872337341309, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 2.1798269748687744, |
| "learning_rate": 4.797394529050577e-06, |
| "logits/chosen": -1.7322242259979248, |
| "logits/rejected": -1.5891096591949463, |
| "logps/chosen": -243.38955688476562, |
| "logps/rejected": -320.7882080078125, |
| "loss": 0.1923, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.6223587989807129, |
| "rewards/margins": 3.5130057334899902, |
| "rewards/rejected": -4.135365009307861, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 2.2701711654663086, |
| "learning_rate": 4.785888835026393e-06, |
| "logits/chosen": -1.8017656803131104, |
| "logits/rejected": -1.5656979084014893, |
| "logps/chosen": -228.77890014648438, |
| "logps/rejected": -307.7184143066406, |
| "loss": 0.1845, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.6147029995918274, |
| "rewards/margins": 3.9243907928466797, |
| "rewards/rejected": -4.539093971252441, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 2.7234058380126953, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -1.8135614395141602, |
| "logits/rejected": -1.704559564590454, |
| "logps/chosen": -225.6403350830078, |
| "logps/rejected": -312.3384704589844, |
| "loss": 0.1871, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.6364536881446838, |
| "rewards/margins": 3.503260374069214, |
| "rewards/rejected": -4.139714241027832, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 1.9700164794921875, |
| "learning_rate": 4.761969555209394e-06, |
| "logits/chosen": -1.844793677330017, |
| "logits/rejected": -1.7481451034545898, |
| "logps/chosen": -218.34042358398438, |
| "logps/rejected": -332.3699951171875, |
| "loss": 0.1436, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.7182143926620483, |
| "rewards/margins": 4.262960910797119, |
| "rewards/rejected": -4.981175422668457, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 2.5656299591064453, |
| "learning_rate": 4.7495591415709435e-06, |
| "logits/chosen": -1.8036201000213623, |
| "logits/rejected": -1.6583620309829712, |
| "logps/chosen": -238.70132446289062, |
| "logps/rejected": -306.24420166015625, |
| "loss": 0.2146, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.8436425924301147, |
| "rewards/margins": 3.7423672676086426, |
| "rewards/rejected": -4.586009979248047, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 1.9172959327697754, |
| "learning_rate": 4.736850393329353e-06, |
| "logits/chosen": -1.8544458150863647, |
| "logits/rejected": -1.6191198825836182, |
| "logps/chosen": -232.9375457763672, |
| "logps/rejected": -318.1675720214844, |
| "loss": 0.1511, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.5402190089225769, |
| "rewards/margins": 4.564599990844727, |
| "rewards/rejected": -5.104818344116211, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 4.023538112640381, |
| "learning_rate": 4.7238449959079095e-06, |
| "logits/chosen": -1.9062020778656006, |
| "logits/rejected": -1.7378747463226318, |
| "logps/chosen": -229.92919921875, |
| "logps/rejected": -324.6113586425781, |
| "loss": 0.1818, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.6886031031608582, |
| "rewards/margins": 4.174241542816162, |
| "rewards/rejected": -4.862845420837402, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 2.3114891052246094, |
| "learning_rate": 4.710544674071259e-06, |
| "logits/chosen": -1.920933723449707, |
| "logits/rejected": -1.7430894374847412, |
| "logps/chosen": -231.68533325195312, |
| "logps/rejected": -307.49627685546875, |
| "loss": 0.16, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.701662003993988, |
| "rewards/margins": 4.109788417816162, |
| "rewards/rejected": -4.811450481414795, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 1.7377828359603882, |
| "learning_rate": 4.696951191696673e-06, |
| "logits/chosen": -1.9457124471664429, |
| "logits/rejected": -1.76227605342865, |
| "logps/chosen": -222.2087860107422, |
| "logps/rejected": -338.64349365234375, |
| "loss": 0.1941, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.81779545545578, |
| "rewards/margins": 4.531395435333252, |
| "rewards/rejected": -5.349190711975098, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 2.8788559436798096, |
| "learning_rate": 4.6830663515401196e-06, |
| "logits/chosen": -1.9213287830352783, |
| "logits/rejected": -1.7758309841156006, |
| "logps/chosen": -231.01126098632812, |
| "logps/rejected": -329.1012268066406, |
| "loss": 0.2076, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5432181358337402, |
| "rewards/margins": 4.00408935546875, |
| "rewards/rejected": -4.54730749130249, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 2.318044900894165, |
| "learning_rate": 4.6688919949971855e-06, |
| "logits/chosen": -1.7594585418701172, |
| "logits/rejected": -1.6069118976593018, |
| "logps/chosen": -231.47994995117188, |
| "logps/rejected": -327.43133544921875, |
| "loss": 0.1632, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.7389811277389526, |
| "rewards/margins": 4.123358249664307, |
| "rewards/rejected": -4.862339973449707, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 2.450094223022461, |
| "learning_rate": 4.654430001858874e-06, |
| "logits/chosen": -1.7364698648452759, |
| "logits/rejected": -1.5289427042007446, |
| "logps/chosen": -236.38003540039062, |
| "logps/rejected": -317.5977478027344, |
| "loss": 0.1806, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.30682843923568726, |
| "rewards/margins": 4.335862159729004, |
| "rewards/rejected": -4.642690181732178, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 2.3008522987365723, |
| "learning_rate": 4.6396822900623075e-06, |
| "logits/chosen": -1.8670647144317627, |
| "logits/rejected": -1.6288022994995117, |
| "logps/chosen": -248.07656860351562, |
| "logps/rejected": -322.63616943359375, |
| "loss": 0.1814, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.26228052377700806, |
| "rewards/margins": 4.046158790588379, |
| "rewards/rejected": -4.308439254760742, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 2.12398362159729, |
| "learning_rate": 4.624650815436368e-06, |
| "logits/chosen": -1.8561731576919556, |
| "logits/rejected": -1.6781187057495117, |
| "logps/chosen": -256.2021179199219, |
| "logps/rejected": -307.13232421875, |
| "loss": 0.1669, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.36551186442375183, |
| "rewards/margins": 3.9093899726867676, |
| "rewards/rejected": -4.274901390075684, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 3.869758367538452, |
| "learning_rate": 4.609337571442324e-06, |
| "logits/chosen": -1.8233314752578735, |
| "logits/rejected": -1.7314884662628174, |
| "logps/chosen": -213.710205078125, |
| "logps/rejected": -327.7879333496094, |
| "loss": 0.1684, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.21804675459861755, |
| "rewards/margins": 4.537252902984619, |
| "rewards/rejected": -4.7552995681762695, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 3.0921249389648438, |
| "learning_rate": 4.5937445889094535e-06, |
| "logits/chosen": -1.8009964227676392, |
| "logits/rejected": -1.5761785507202148, |
| "logps/chosen": -236.89102172851562, |
| "logps/rejected": -313.84295654296875, |
| "loss": 0.1917, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.15902478992938995, |
| "rewards/margins": 4.049332141876221, |
| "rewards/rejected": -4.208357334136963, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 2.362419843673706, |
| "learning_rate": 4.577873935765722e-06, |
| "logits/chosen": -1.9038022756576538, |
| "logits/rejected": -1.6466445922851562, |
| "logps/chosen": -222.8800506591797, |
| "logps/rejected": -315.28875732421875, |
| "loss": 0.1559, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.3516892194747925, |
| "rewards/margins": 4.556002616882324, |
| "rewards/rejected": -4.90769100189209, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 2.472935438156128, |
| "learning_rate": 4.561727716763532e-06, |
| "logits/chosen": -1.805426001548767, |
| "logits/rejected": -1.6315984725952148, |
| "logps/chosen": -244.3173370361328, |
| "logps/rejected": -319.24176025390625, |
| "loss": 0.1987, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.3080672025680542, |
| "rewards/margins": 3.776719331741333, |
| "rewards/rejected": -4.084786415100098, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 1.679807186126709, |
| "learning_rate": 4.545308073200591e-06, |
| "logits/chosen": -1.7192637920379639, |
| "logits/rejected": -1.536003828048706, |
| "logps/chosen": -238.8625946044922, |
| "logps/rejected": -325.7824401855469, |
| "loss": 0.1617, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.10333986580371857, |
| "rewards/margins": 4.606169700622559, |
| "rewards/rejected": -4.709508895874023, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 1.7725813388824463, |
| "learning_rate": 4.528617182635943e-06, |
| "logits/chosen": -1.7725048065185547, |
| "logits/rejected": -1.664609670639038, |
| "logps/chosen": -244.0134735107422, |
| "logps/rejected": -344.0420837402344, |
| "loss": 0.2013, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.8228844404220581, |
| "rewards/margins": 4.634818077087402, |
| "rewards/rejected": -5.45770263671875, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 4.325881004333496, |
| "learning_rate": 4.511657258601171e-06, |
| "logits/chosen": -1.9085323810577393, |
| "logits/rejected": -1.6071668863296509, |
| "logps/chosen": -221.92404174804688, |
| "logps/rejected": -292.5198669433594, |
| "loss": 0.1981, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.2094966471195221, |
| "rewards/margins": 4.443403244018555, |
| "rewards/rejected": -4.652899742126465, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 1.9912614822387695, |
| "learning_rate": 4.494430550306848e-06, |
| "logits/chosen": -1.761791467666626, |
| "logits/rejected": -1.6368515491485596, |
| "logps/chosen": -225.29550170898438, |
| "logps/rejected": -304.720703125, |
| "loss": 0.2184, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.3698757588863373, |
| "rewards/margins": 3.930206298828125, |
| "rewards/rejected": -4.300081729888916, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 2.0521910190582275, |
| "learning_rate": 4.476939342344246e-06, |
| "logits/chosen": -1.7833703756332397, |
| "logits/rejected": -1.6641095876693726, |
| "logps/chosen": -216.5574951171875, |
| "logps/rejected": -319.1539306640625, |
| "loss": 0.1463, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6160684823989868, |
| "rewards/margins": 4.553713321685791, |
| "rewards/rejected": -5.169781684875488, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 2.2633090019226074, |
| "learning_rate": 4.45918595438236e-06, |
| "logits/chosen": -1.6932775974273682, |
| "logits/rejected": -1.6147457361221313, |
| "logps/chosen": -228.96987915039062, |
| "logps/rejected": -333.4344482421875, |
| "loss": 0.1349, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.269506573677063, |
| "rewards/margins": 4.556288719177246, |
| "rewards/rejected": -4.8257951736450195, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 1.9004288911819458, |
| "learning_rate": 4.441172740860267e-06, |
| "logits/chosen": -1.7839820384979248, |
| "logits/rejected": -1.586343765258789, |
| "logps/chosen": -231.0703582763672, |
| "logps/rejected": -334.8110046386719, |
| "loss": 0.1565, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.480475515127182, |
| "rewards/margins": 4.550753116607666, |
| "rewards/rejected": -5.031228065490723, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 1.4590946435928345, |
| "learning_rate": 4.422902090674886e-06, |
| "logits/chosen": -1.7286231517791748, |
| "logits/rejected": -1.5739670991897583, |
| "logps/chosen": -230.1336669921875, |
| "logps/rejected": -371.85638427734375, |
| "loss": 0.1483, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5395830869674683, |
| "rewards/margins": 4.902421474456787, |
| "rewards/rejected": -5.442005157470703, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 1.425997257232666, |
| "learning_rate": 4.4043764268641705e-06, |
| "logits/chosen": -1.7624887228012085, |
| "logits/rejected": -1.5530593395233154, |
| "logps/chosen": -236.2501983642578, |
| "logps/rejected": -325.4139404296875, |
| "loss": 0.1035, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.555716872215271, |
| "rewards/margins": 4.979168891906738, |
| "rewards/rejected": -5.534886360168457, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 2.0888102054595947, |
| "learning_rate": 4.385598206285758e-06, |
| "logits/chosen": -1.9180511236190796, |
| "logits/rejected": -1.7492713928222656, |
| "logps/chosen": -228.6175537109375, |
| "logps/rejected": -325.20953369140625, |
| "loss": 0.1149, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.6507972478866577, |
| "rewards/margins": 5.017643451690674, |
| "rewards/rejected": -5.668440818786621, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 1.01185142993927, |
| "learning_rate": 4.3665699192911495e-06, |
| "logits/chosen": -1.6995750665664673, |
| "logits/rejected": -1.5751515626907349, |
| "logps/chosen": -218.4319305419922, |
| "logps/rejected": -341.39703369140625, |
| "loss": 0.1579, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.7953837513923645, |
| "rewards/margins": 5.295377254486084, |
| "rewards/rejected": -6.090761661529541, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 3.163583755493164, |
| "learning_rate": 4.347294089395438e-06, |
| "logits/chosen": -1.8674875497817993, |
| "logits/rejected": -1.6801738739013672, |
| "logps/chosen": -237.85214233398438, |
| "logps/rejected": -345.7062683105469, |
| "loss": 0.1076, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.8477768898010254, |
| "rewards/margins": 6.111438751220703, |
| "rewards/rejected": -6.959216117858887, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 2.2099385261535645, |
| "learning_rate": 4.327773272942647e-06, |
| "logits/chosen": -1.7721245288848877, |
| "logits/rejected": -1.6586471796035767, |
| "logps/chosen": -240.5192413330078, |
| "logps/rejected": -312.6970520019531, |
| "loss": 0.1863, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.2079604864120483, |
| "rewards/margins": 4.806216239929199, |
| "rewards/rejected": -6.014176845550537, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 2.9571621417999268, |
| "learning_rate": 4.308010058766703e-06, |
| "logits/chosen": -1.8337215185165405, |
| "logits/rejected": -1.675649642944336, |
| "logps/chosen": -238.7671661376953, |
| "logps/rejected": -331.68511962890625, |
| "loss": 0.2111, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.2102830410003662, |
| "rewards/margins": 5.248218536376953, |
| "rewards/rejected": -6.45850133895874, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 2.9183189868927, |
| "learning_rate": 4.2880070678481135e-06, |
| "logits/chosen": -1.8363081216812134, |
| "logits/rejected": -1.6459910869598389, |
| "logps/chosen": -227.5572052001953, |
| "logps/rejected": -328.91925048828125, |
| "loss": 0.1342, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.3287819623947144, |
| "rewards/margins": 5.704113006591797, |
| "rewards/rejected": -7.032895088195801, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 2.6443653106689453, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.7400104999542236, |
| "logits/rejected": -1.6507030725479126, |
| "logps/chosen": -236.0244598388672, |
| "logps/rejected": -360.4212951660156, |
| "loss": 0.1261, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1412465572357178, |
| "rewards/margins": 5.938536643981934, |
| "rewards/rejected": -7.0797834396362305, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 2.404409408569336, |
| "learning_rate": 4.24729239834814e-06, |
| "logits/chosen": -1.7891966104507446, |
| "logits/rejected": -1.6409313678741455, |
| "logps/chosen": -242.6314697265625, |
| "logps/rejected": -348.565673828125, |
| "loss": 0.177, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.0923963785171509, |
| "rewards/margins": 5.512354373931885, |
| "rewards/rejected": -6.604750156402588, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 2.2743067741394043, |
| "learning_rate": 4.226586119311291e-06, |
| "logits/chosen": -1.8340394496917725, |
| "logits/rejected": -1.7385257482528687, |
| "logps/chosen": -238.93197631835938, |
| "logps/rejected": -331.0224304199219, |
| "loss": 0.17, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9892851710319519, |
| "rewards/margins": 4.964252471923828, |
| "rewards/rejected": -5.953537464141846, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 3.5358939170837402, |
| "learning_rate": 4.205650861904782e-06, |
| "logits/chosen": -1.7910658121109009, |
| "logits/rejected": -1.6007837057113647, |
| "logps/chosen": -212.20230102539062, |
| "logps/rejected": -335.0103759765625, |
| "loss": 0.2147, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -1.0847444534301758, |
| "rewards/margins": 5.438946723937988, |
| "rewards/rejected": -6.523691654205322, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 3.7709548473358154, |
| "learning_rate": 4.184489402544486e-06, |
| "logits/chosen": -1.8345584869384766, |
| "logits/rejected": -1.7050797939300537, |
| "logps/chosen": -233.6999969482422, |
| "logps/rejected": -334.1361389160156, |
| "loss": 0.1381, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.8633434176445007, |
| "rewards/margins": 5.313804626464844, |
| "rewards/rejected": -6.177147388458252, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 6.689227104187012, |
| "learning_rate": 4.1631045476449865e-06, |
| "logits/chosen": -1.8753330707550049, |
| "logits/rejected": -1.6922645568847656, |
| "logps/chosen": -227.6363525390625, |
| "logps/rejected": -362.1102600097656, |
| "loss": 0.1487, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.9522708058357239, |
| "rewards/margins": 5.492487907409668, |
| "rewards/rejected": -6.444759368896484, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 2.497356653213501, |
| "learning_rate": 4.141499133247391e-06, |
| "logits/chosen": -1.758281946182251, |
| "logits/rejected": -1.60012948513031, |
| "logps/chosen": -221.84115600585938, |
| "logps/rejected": -317.8287353515625, |
| "loss": 0.1434, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5248526334762573, |
| "rewards/margins": 5.158529758453369, |
| "rewards/rejected": -5.6833815574646, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 3.754014492034912, |
| "learning_rate": 4.119676024643217e-06, |
| "logits/chosen": -1.7635116577148438, |
| "logits/rejected": -1.615441918373108, |
| "logps/chosen": -227.84927368164062, |
| "logps/rejected": -329.3919372558594, |
| "loss": 0.2183, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -1.008711576461792, |
| "rewards/margins": 4.943830966949463, |
| "rewards/rejected": -5.952542781829834, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 4.455972194671631, |
| "learning_rate": 4.0976381159944055e-06, |
| "logits/chosen": -1.8479392528533936, |
| "logits/rejected": -1.6086915731430054, |
| "logps/chosen": -253.4595184326172, |
| "logps/rejected": -353.01531982421875, |
| "loss": 0.1233, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9238975644111633, |
| "rewards/margins": 5.841231346130371, |
| "rewards/rejected": -6.765129089355469, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 3.814085006713867, |
| "learning_rate": 4.07538832994949e-06, |
| "logits/chosen": -1.7635217905044556, |
| "logits/rejected": -1.5600554943084717, |
| "logps/chosen": -235.2505340576172, |
| "logps/rejected": -334.1875915527344, |
| "loss": 0.1404, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1111348867416382, |
| "rewards/margins": 5.897833347320557, |
| "rewards/rejected": -7.008968353271484, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 1.5983116626739502, |
| "learning_rate": 4.0529296172560055e-06, |
| "logits/chosen": -1.7206604480743408, |
| "logits/rejected": -1.6044199466705322, |
| "logps/chosen": -255.4562225341797, |
| "logps/rejected": -353.7682189941406, |
| "loss": 0.1347, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.611911416053772, |
| "rewards/margins": 5.771895408630371, |
| "rewards/rejected": -7.383807182312012, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 3.102590322494507, |
| "learning_rate": 4.030264956369158e-06, |
| "logits/chosen": -1.7923877239227295, |
| "logits/rejected": -1.6720997095108032, |
| "logps/chosen": -229.66604614257812, |
| "logps/rejected": -340.2394714355469, |
| "loss": 0.1827, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -1.3031582832336426, |
| "rewards/margins": 4.982819557189941, |
| "rewards/rejected": -6.285977840423584, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 2.5847177505493164, |
| "learning_rate": 4.0073973530568274e-06, |
| "logits/chosen": -1.8058351278305054, |
| "logits/rejected": -1.6101223230361938, |
| "logps/chosen": -235.02920532226562, |
| "logps/rejected": -333.5020751953125, |
| "loss": 0.2007, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -1.0441210269927979, |
| "rewards/margins": 5.034656047821045, |
| "rewards/rejected": -6.078777313232422, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 2.9892916679382324, |
| "learning_rate": 3.9843298400009435e-06, |
| "logits/chosen": -1.8426891565322876, |
| "logits/rejected": -1.7364394664764404, |
| "logps/chosen": -237.25210571289062, |
| "logps/rejected": -343.7469177246094, |
| "loss": 0.1625, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9305593371391296, |
| "rewards/margins": 5.86553955078125, |
| "rewards/rejected": -6.7960991859436035, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 2.571195363998413, |
| "learning_rate": 3.961065476395296e-06, |
| "logits/chosen": -1.8120330572128296, |
| "logits/rejected": -1.6824085712432861, |
| "logps/chosen": -235.7975616455078, |
| "logps/rejected": -355.3353576660156, |
| "loss": 0.1486, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.8333516120910645, |
| "rewards/margins": 5.53482723236084, |
| "rewards/rejected": -6.368178844451904, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 1.5515739917755127, |
| "learning_rate": 3.937607347539823e-06, |
| "logits/chosen": -1.8604084253311157, |
| "logits/rejected": -1.6165311336517334, |
| "logps/chosen": -216.8756561279297, |
| "logps/rejected": -344.33734130859375, |
| "loss": 0.0933, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.6246806383132935, |
| "rewards/margins": 5.950966835021973, |
| "rewards/rejected": -6.575646877288818, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 3.6848971843719482, |
| "learning_rate": 3.913958564431447e-06, |
| "logits/chosen": -1.8793195486068726, |
| "logits/rejected": -1.6981279850006104, |
| "logps/chosen": -242.66354370117188, |
| "logps/rejected": -337.1656188964844, |
| "loss": 0.1586, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.083747148513794, |
| "rewards/margins": 5.684298992156982, |
| "rewards/rejected": -6.7680463790893555, |
| "step": 465 |
| }, |
| { |
| "epoch": 1.0021333333333333, |
| "grad_norm": 1.5835093259811401, |
| "learning_rate": 3.89012226335149e-06, |
| "logits/chosen": -1.67910635471344, |
| "logits/rejected": -1.6301380395889282, |
| "logps/chosen": -213.78390502929688, |
| "logps/rejected": -342.0399475097656, |
| "loss": 0.1175, |
| "rewards/accuracies": 0.9473684430122375, |
| "rewards/chosen": -1.068010687828064, |
| "rewards/margins": 5.761088848114014, |
| "rewards/rejected": -6.829100131988525, |
| "step": 470 |
| }, |
| { |
| "epoch": 1.0128, |
| "grad_norm": 1.5585732460021973, |
| "learning_rate": 3.86610160544975e-06, |
| "logits/chosen": -1.7755944728851318, |
| "logits/rejected": -1.6420894861221313, |
| "logps/chosen": -241.9550018310547, |
| "logps/rejected": -339.51776123046875, |
| "loss": 0.106, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1774930953979492, |
| "rewards/margins": 5.725667476654053, |
| "rewards/rejected": -6.90316104888916, |
| "step": 475 |
| }, |
| { |
| "epoch": 1.0234666666666667, |
| "grad_norm": 2.624796152114868, |
| "learning_rate": 3.841899776325267e-06, |
| "logits/chosen": -1.8001434803009033, |
| "logits/rejected": -1.6177053451538086, |
| "logps/chosen": -245.3330078125, |
| "logps/rejected": -339.3960266113281, |
| "loss": 0.116, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.7908494472503662, |
| "rewards/margins": 6.218578338623047, |
| "rewards/rejected": -7.009427070617676, |
| "step": 480 |
| }, |
| { |
| "epoch": 1.0341333333333333, |
| "grad_norm": 1.0212681293487549, |
| "learning_rate": 3.817519985603855e-06, |
| "logits/chosen": -1.9008468389511108, |
| "logits/rejected": -1.785378098487854, |
| "logps/chosen": -208.4319610595703, |
| "logps/rejected": -339.15185546875, |
| "loss": 0.1052, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.1869536638259888, |
| "rewards/margins": 6.368255615234375, |
| "rewards/rejected": -7.555208683013916, |
| "step": 485 |
| }, |
| { |
| "epoch": 1.0448, |
| "grad_norm": 2.107454776763916, |
| "learning_rate": 3.792965466512437e-06, |
| "logits/chosen": -1.7998336553573608, |
| "logits/rejected": -1.6306270360946655, |
| "logps/chosen": -241.4481658935547, |
| "logps/rejected": -336.5618591308594, |
| "loss": 0.0921, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.0441654920578003, |
| "rewards/margins": 6.187332630157471, |
| "rewards/rejected": -7.231498718261719, |
| "step": 490 |
| }, |
| { |
| "epoch": 1.0554666666666668, |
| "grad_norm": 2.4917521476745605, |
| "learning_rate": 3.7682394754502687e-06, |
| "logits/chosen": -1.8501405715942383, |
| "logits/rejected": -1.7844288349151611, |
| "logps/chosen": -233.9818115234375, |
| "logps/rejected": -370.55322265625, |
| "loss": 0.11, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.3015844821929932, |
| "rewards/margins": 6.5641679763793945, |
| "rewards/rejected": -7.86575174331665, |
| "step": 495 |
| }, |
| { |
| "epoch": 1.0661333333333334, |
| "grad_norm": 4.674692153930664, |
| "learning_rate": 3.7433452915570685e-06, |
| "logits/chosen": -1.834409475326538, |
| "logits/rejected": -1.7216771841049194, |
| "logps/chosen": -224.1982421875, |
| "logps/rejected": -330.5555114746094, |
| "loss": 0.1307, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9702581167221069, |
| "rewards/margins": 5.728388786315918, |
| "rewards/rejected": -6.698647499084473, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0768, |
| "grad_norm": 1.135600209236145, |
| "learning_rate": 3.718286216278143e-06, |
| "logits/chosen": -1.8727703094482422, |
| "logits/rejected": -1.6628506183624268, |
| "logps/chosen": -232.24893188476562, |
| "logps/rejected": -348.3692321777344, |
| "loss": 0.0987, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2046107053756714, |
| "rewards/margins": 6.24137020111084, |
| "rewards/rejected": -7.445981025695801, |
| "step": 505 |
| }, |
| { |
| "epoch": 1.0874666666666666, |
| "grad_norm": 3.7477798461914062, |
| "learning_rate": 3.6930655729265557e-06, |
| "logits/chosen": -1.8030757904052734, |
| "logits/rejected": -1.6741348505020142, |
| "logps/chosen": -252.7802276611328, |
| "logps/rejected": -363.14422607421875, |
| "loss": 0.1047, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.1648820638656616, |
| "rewards/margins": 6.590028285980225, |
| "rewards/rejected": -7.754910469055176, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.0981333333333334, |
| "grad_norm": 1.919650673866272, |
| "learning_rate": 3.667686706242386e-06, |
| "logits/chosen": -1.7448360919952393, |
| "logits/rejected": -1.6716108322143555, |
| "logps/chosen": -255.5791473388672, |
| "logps/rejected": -367.09210205078125, |
| "loss": 0.0976, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3545987606048584, |
| "rewards/margins": 6.418627738952637, |
| "rewards/rejected": -7.773226737976074, |
| "step": 515 |
| }, |
| { |
| "epoch": 1.1088, |
| "grad_norm": 2.581798553466797, |
| "learning_rate": 3.6421529819491576e-06, |
| "logits/chosen": -1.8232694864273071, |
| "logits/rejected": -1.7192691564559937, |
| "logps/chosen": -248.962890625, |
| "logps/rejected": -359.66497802734375, |
| "loss": 0.098, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4379160404205322, |
| "rewards/margins": 6.792909145355225, |
| "rewards/rejected": -8.230825424194336, |
| "step": 520 |
| }, |
| { |
| "epoch": 1.1194666666666666, |
| "grad_norm": 1.842156171798706, |
| "learning_rate": 3.6164677863074783e-06, |
| "logits/chosen": -1.8928749561309814, |
| "logits/rejected": -1.7210674285888672, |
| "logps/chosen": -217.7656707763672, |
| "logps/rejected": -342.0232238769531, |
| "loss": 0.087, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3847095966339111, |
| "rewards/margins": 7.149325370788574, |
| "rewards/rejected": -8.534035682678223, |
| "step": 525 |
| }, |
| { |
| "epoch": 1.1301333333333332, |
| "grad_norm": 4.107654571533203, |
| "learning_rate": 3.590634525665955e-06, |
| "logits/chosen": -1.9062941074371338, |
| "logits/rejected": -1.7494275569915771, |
| "logps/chosen": -222.1142120361328, |
| "logps/rejected": -378.0558776855469, |
| "loss": 0.1408, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -1.640011191368103, |
| "rewards/margins": 7.438580513000488, |
| "rewards/rejected": -9.078592300415039, |
| "step": 530 |
| }, |
| { |
| "epoch": 1.1408, |
| "grad_norm": 4.996397495269775, |
| "learning_rate": 3.564656626009446e-06, |
| "logits/chosen": -1.9074268341064453, |
| "logits/rejected": -1.6736358404159546, |
| "logps/chosen": -305.10498046875, |
| "logps/rejected": -382.6112060546875, |
| "loss": 0.1103, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.8150031566619873, |
| "rewards/margins": 7.754342555999756, |
| "rewards/rejected": -9.569345474243164, |
| "step": 535 |
| }, |
| { |
| "epoch": 1.1514666666666666, |
| "grad_norm": 2.6492626667022705, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.8353687524795532, |
| "logits/rejected": -1.7401046752929688, |
| "logps/chosen": -234.1809539794922, |
| "logps/rejected": -361.4974060058594, |
| "loss": 0.1014, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9234129190444946, |
| "rewards/margins": 6.884763240814209, |
| "rewards/rejected": -8.80817699432373, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.1621333333333332, |
| "grad_norm": 1.1152234077453613, |
| "learning_rate": 3.512280709043534e-06, |
| "logits/chosen": -1.8246501684188843, |
| "logits/rejected": -1.6780567169189453, |
| "logps/chosen": -251.69308471679688, |
| "logps/rejected": -354.8346252441406, |
| "loss": 0.1077, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4507404565811157, |
| "rewards/margins": 6.514955043792725, |
| "rewards/rejected": -7.965695381164551, |
| "step": 545 |
| }, |
| { |
| "epoch": 1.1728, |
| "grad_norm": 4.050209999084473, |
| "learning_rate": 3.4858896377832966e-06, |
| "logits/chosen": -1.8190292119979858, |
| "logits/rejected": -1.7231298685073853, |
| "logps/chosen": -232.68844604492188, |
| "logps/rejected": -375.8724365234375, |
| "loss": 0.0755, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.4951999187469482, |
| "rewards/margins": 7.632623195648193, |
| "rewards/rejected": -9.127823829650879, |
| "step": 550 |
| }, |
| { |
| "epoch": 1.1834666666666667, |
| "grad_norm": 5.159008026123047, |
| "learning_rate": 3.459367818685231e-06, |
| "logits/chosen": -1.8263816833496094, |
| "logits/rejected": -1.6552646160125732, |
| "logps/chosen": -237.4913330078125, |
| "logps/rejected": -357.56719970703125, |
| "loss": 0.0946, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.2089338302612305, |
| "rewards/margins": 7.155152320861816, |
| "rewards/rejected": -8.36408519744873, |
| "step": 555 |
| }, |
| { |
| "epoch": 1.1941333333333333, |
| "grad_norm": 11.910748481750488, |
| "learning_rate": 3.432718769050228e-06, |
| "logits/chosen": -1.7615569829940796, |
| "logits/rejected": -1.5479238033294678, |
| "logps/chosen": -245.3880615234375, |
| "logps/rejected": -345.82427978515625, |
| "loss": 0.137, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.5121991634368896, |
| "rewards/margins": 6.261626243591309, |
| "rewards/rejected": -7.773825168609619, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.2048, |
| "grad_norm": 1.417484164237976, |
| "learning_rate": 3.405946023052386e-06, |
| "logits/chosen": -1.8306633234024048, |
| "logits/rejected": -1.7571738958358765, |
| "logps/chosen": -233.8412628173828, |
| "logps/rejected": -352.0577087402344, |
| "loss": 0.0998, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.8432447910308838, |
| "rewards/margins": 6.6724748611450195, |
| "rewards/rejected": -8.515719413757324, |
| "step": 565 |
| }, |
| { |
| "epoch": 1.2154666666666667, |
| "grad_norm": 2.3207719326019287, |
| "learning_rate": 3.3790531312703055e-06, |
| "logits/chosen": -1.888755440711975, |
| "logits/rejected": -1.743829369544983, |
| "logps/chosen": -244.34909057617188, |
| "logps/rejected": -382.1915588378906, |
| "loss": 0.087, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.826398491859436, |
| "rewards/margins": 7.315705299377441, |
| "rewards/rejected": -9.14210319519043, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.2261333333333333, |
| "grad_norm": 3.761608123779297, |
| "learning_rate": 3.35204366021622e-06, |
| "logits/chosen": -1.7861427068710327, |
| "logits/rejected": -1.6182992458343506, |
| "logps/chosen": -231.2759552001953, |
| "logps/rejected": -336.6302185058594, |
| "loss": 0.1579, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -1.3482277393341064, |
| "rewards/margins": 6.256989479064941, |
| "rewards/rejected": -7.605217933654785, |
| "step": 575 |
| }, |
| { |
| "epoch": 1.2368000000000001, |
| "grad_norm": 2.557394027709961, |
| "learning_rate": 3.3249211918630055e-06, |
| "logits/chosen": -1.8882964849472046, |
| "logits/rejected": -1.7571732997894287, |
| "logps/chosen": -220.7556610107422, |
| "logps/rejected": -365.0335998535156, |
| "loss": 0.095, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.1241406202316284, |
| "rewards/margins": 7.271350860595703, |
| "rewards/rejected": -8.395490646362305, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.2474666666666667, |
| "grad_norm": 3.2217822074890137, |
| "learning_rate": 3.2976893231691424e-06, |
| "logits/chosen": -1.7663228511810303, |
| "logits/rejected": -1.6859546899795532, |
| "logps/chosen": -249.8618927001953, |
| "logps/rejected": -372.42169189453125, |
| "loss": 0.115, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -1.5136463642120361, |
| "rewards/margins": 6.937762260437012, |
| "rewards/rejected": -8.451409339904785, |
| "step": 585 |
| }, |
| { |
| "epoch": 1.2581333333333333, |
| "grad_norm": 0.9602277874946594, |
| "learning_rate": 3.270351665601691e-06, |
| "logits/chosen": -1.794925332069397, |
| "logits/rejected": -1.7219619750976562, |
| "logps/chosen": -232.79190063476562, |
| "logps/rejected": -343.2873840332031, |
| "loss": 0.1009, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.2487173080444336, |
| "rewards/margins": 6.3407440185546875, |
| "rewards/rejected": -7.589460849761963, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.2688, |
| "grad_norm": 2.6589019298553467, |
| "learning_rate": 3.2429118446573417e-06, |
| "logits/chosen": -1.8542613983154297, |
| "logits/rejected": -1.6390371322631836, |
| "logps/chosen": -232.03125, |
| "logps/rejected": -352.66241455078125, |
| "loss": 0.1434, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.5202168226242065, |
| "rewards/margins": 6.778920650482178, |
| "rewards/rejected": -8.299138069152832, |
| "step": 595 |
| }, |
| { |
| "epoch": 1.2794666666666665, |
| "grad_norm": 1.2758371829986572, |
| "learning_rate": 3.215373499381602e-06, |
| "logits/chosen": -1.8230146169662476, |
| "logits/rejected": -1.6330006122589111, |
| "logps/chosen": -236.1676025390625, |
| "logps/rejected": -346.3940734863281, |
| "loss": 0.0952, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.3624433279037476, |
| "rewards/margins": 7.068747520446777, |
| "rewards/rejected": -8.431192398071289, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.2901333333333334, |
| "grad_norm": 1.0389660596847534, |
| "learning_rate": 3.1877402818861954e-06, |
| "logits/chosen": -1.808443307876587, |
| "logits/rejected": -1.6811857223510742, |
| "logps/chosen": -262.27532958984375, |
| "logps/rejected": -381.2078552246094, |
| "loss": 0.0504, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.750425100326538, |
| "rewards/margins": 7.616028785705566, |
| "rewards/rejected": -9.366454124450684, |
| "step": 605 |
| }, |
| { |
| "epoch": 1.3008, |
| "grad_norm": 5.7284016609191895, |
| "learning_rate": 3.1600158568647145e-06, |
| "logits/chosen": -1.7468950748443604, |
| "logits/rejected": -1.7098772525787354, |
| "logps/chosen": -232.5611114501953, |
| "logps/rejected": -350.3565673828125, |
| "loss": 0.0777, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.3892863988876343, |
| "rewards/margins": 6.9035325050354, |
| "rewards/rejected": -8.292818069458008, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.3114666666666666, |
| "grad_norm": 2.848447561264038, |
| "learning_rate": 3.1322039011066172e-06, |
| "logits/chosen": -1.850782036781311, |
| "logits/rejected": -1.6880401372909546, |
| "logps/chosen": -247.71377563476562, |
| "logps/rejected": -349.6388854980469, |
| "loss": 0.1083, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.515555500984192, |
| "rewards/margins": 6.665856838226318, |
| "rewards/rejected": -8.181412696838379, |
| "step": 615 |
| }, |
| { |
| "epoch": 1.3221333333333334, |
| "grad_norm": 2.5418522357940674, |
| "learning_rate": 3.1043081030096125e-06, |
| "logits/chosen": -1.7264868021011353, |
| "logits/rejected": -1.5793980360031128, |
| "logps/chosen": -243.20101928710938, |
| "logps/rejected": -349.4842224121094, |
| "loss": 0.1148, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.619227409362793, |
| "rewards/margins": 7.061232089996338, |
| "rewards/rejected": -8.680459022521973, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.3328, |
| "grad_norm": 1.7328300476074219, |
| "learning_rate": 3.0763321620905072e-06, |
| "logits/chosen": -1.7665011882781982, |
| "logits/rejected": -1.5880069732666016, |
| "logps/chosen": -263.8298034667969, |
| "logps/rejected": -368.64764404296875, |
| "loss": 0.1128, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.6416599750518799, |
| "rewards/margins": 6.925388336181641, |
| "rewards/rejected": -8.567049026489258, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.3434666666666666, |
| "grad_norm": 1.3818984031677246, |
| "learning_rate": 3.048279788494583e-06, |
| "logits/chosen": -1.8107048273086548, |
| "logits/rejected": -1.6657230854034424, |
| "logps/chosen": -231.5718231201172, |
| "logps/rejected": -361.8509521484375, |
| "loss": 0.0717, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.662607192993164, |
| "rewards/margins": 7.121241569519043, |
| "rewards/rejected": -8.783849716186523, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.3541333333333334, |
| "grad_norm": 1.9574915170669556, |
| "learning_rate": 3.0201547025035553e-06, |
| "logits/chosen": -1.7519423961639404, |
| "logits/rejected": -1.6680078506469727, |
| "logps/chosen": -265.30389404296875, |
| "logps/rejected": -374.69158935546875, |
| "loss": 0.1317, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.633040428161621, |
| "rewards/margins": 7.419091701507568, |
| "rewards/rejected": -9.052130699157715, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.3648, |
| "grad_norm": 4.3590898513793945, |
| "learning_rate": 2.9919606340421963e-06, |
| "logits/chosen": -1.8167431354522705, |
| "logits/rejected": -1.582132339477539, |
| "logps/chosen": -244.23715209960938, |
| "logps/rejected": -375.7297668457031, |
| "loss": 0.106, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.620737075805664, |
| "rewards/margins": 7.2550506591796875, |
| "rewards/rejected": -8.875787734985352, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.3754666666666666, |
| "grad_norm": 2.495652437210083, |
| "learning_rate": 2.9637013221836723e-06, |
| "logits/chosen": -1.8191438913345337, |
| "logits/rejected": -1.6955486536026, |
| "logps/chosen": -243.5186309814453, |
| "logps/rejected": -383.9856872558594, |
| "loss": 0.0928, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.691001534461975, |
| "rewards/margins": 7.247744560241699, |
| "rewards/rejected": -8.938745498657227, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.3861333333333334, |
| "grad_norm": 0.8766574859619141, |
| "learning_rate": 2.9353805146536762e-06, |
| "logits/chosen": -1.7703936100006104, |
| "logits/rejected": -1.5947972536087036, |
| "logps/chosen": -256.13348388671875, |
| "logps/rejected": -367.19384765625, |
| "loss": 0.1009, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.5907480716705322, |
| "rewards/margins": 7.426206111907959, |
| "rewards/rejected": -9.01695442199707, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.3968, |
| "grad_norm": 3.314081907272339, |
| "learning_rate": 2.907001967333401e-06, |
| "logits/chosen": -1.8712552785873413, |
| "logits/rejected": -1.7601633071899414, |
| "logps/chosen": -242.0836181640625, |
| "logps/rejected": -389.406982421875, |
| "loss": 0.0926, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9015403985977173, |
| "rewards/margins": 7.678501129150391, |
| "rewards/rejected": -9.58004093170166, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.4074666666666666, |
| "grad_norm": 2.852855920791626, |
| "learning_rate": 2.878569443761442e-06, |
| "logits/chosen": -1.7180980443954468, |
| "logits/rejected": -1.593620777130127, |
| "logps/chosen": -245.14639282226562, |
| "logps/rejected": -350.68536376953125, |
| "loss": 0.0907, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2707574367523193, |
| "rewards/margins": 7.63345193862915, |
| "rewards/rejected": -9.90420913696289, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.4181333333333335, |
| "grad_norm": 2.0365729331970215, |
| "learning_rate": 2.85008671463468e-06, |
| "logits/chosen": -1.8370532989501953, |
| "logits/rejected": -1.7093183994293213, |
| "logps/chosen": -242.4522247314453, |
| "logps/rejected": -368.5684814453125, |
| "loss": 0.0805, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7941030263900757, |
| "rewards/margins": 7.619574546813965, |
| "rewards/rejected": -9.413678169250488, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.4288, |
| "grad_norm": 0.7839527726173401, |
| "learning_rate": 2.8215575573082127e-06, |
| "logits/chosen": -1.819196343421936, |
| "logits/rejected": -1.6762737035751343, |
| "logps/chosen": -235.5693817138672, |
| "logps/rejected": -372.1368713378906, |
| "loss": 0.0533, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.2108426094055176, |
| "rewards/margins": 8.851494789123535, |
| "rewards/rejected": -11.062335968017578, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.4394666666666667, |
| "grad_norm": 5.5416059494018555, |
| "learning_rate": 2.792985755294404e-06, |
| "logits/chosen": -1.8234074115753174, |
| "logits/rejected": -1.6806237697601318, |
| "logps/chosen": -235.62826538085938, |
| "logps/rejected": -380.11541748046875, |
| "loss": 0.1041, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.754598617553711, |
| "rewards/margins": 8.41307258605957, |
| "rewards/rejected": -11.167672157287598, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.4501333333333333, |
| "grad_norm": 6.455689907073975, |
| "learning_rate": 2.764375097761122e-06, |
| "logits/chosen": -1.627570390701294, |
| "logits/rejected": -1.4764485359191895, |
| "logps/chosen": -251.55331420898438, |
| "logps/rejected": -352.8821105957031, |
| "loss": 0.1401, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.669222593307495, |
| "rewards/margins": 7.199682712554932, |
| "rewards/rejected": -9.868905067443848, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.4607999999999999, |
| "grad_norm": 4.647602558135986, |
| "learning_rate": 2.73572937902922e-06, |
| "logits/chosen": -1.671311616897583, |
| "logits/rejected": -1.548492431640625, |
| "logps/chosen": -276.5353088378906, |
| "logps/rejected": -366.04937744140625, |
| "loss": 0.1593, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.8360071182250977, |
| "rewards/margins": 7.552770137786865, |
| "rewards/rejected": -10.388776779174805, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.4714666666666667, |
| "grad_norm": 0.9273281693458557, |
| "learning_rate": 2.7070523980693362e-06, |
| "logits/chosen": -1.532097339630127, |
| "logits/rejected": -1.5424264669418335, |
| "logps/chosen": -270.2457580566406, |
| "logps/rejected": -377.8583679199219, |
| "loss": 0.1488, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.7499938011169434, |
| "rewards/margins": 7.535027980804443, |
| "rewards/rejected": -10.285021781921387, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.4821333333333333, |
| "grad_norm": 3.152971029281616, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.8320585489273071, |
| "logits/rejected": -1.6948597431182861, |
| "logps/chosen": -241.94406127929688, |
| "logps/rejected": -370.0408935546875, |
| "loss": 0.1206, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -2.761826276779175, |
| "rewards/margins": 7.769921779632568, |
| "rewards/rejected": -10.531747817993164, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.4928, |
| "grad_norm": 3.29986834526062, |
| "learning_rate": 2.649619865573666e-06, |
| "logits/chosen": -1.8070224523544312, |
| "logits/rejected": -1.633344292640686, |
| "logps/chosen": -245.9867706298828, |
| "logps/rejected": -370.3143005371094, |
| "loss": 0.1189, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.6095986366271973, |
| "rewards/margins": 7.841919898986816, |
| "rewards/rejected": -10.451518058776855, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.5034666666666667, |
| "grad_norm": 5.474385738372803, |
| "learning_rate": 2.620871930691058e-06, |
| "logits/chosen": -1.7647151947021484, |
| "logits/rejected": -1.6790416240692139, |
| "logps/chosen": -238.916259765625, |
| "logps/rejected": -353.8853759765625, |
| "loss": 0.1491, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.5073959827423096, |
| "rewards/margins": 6.695868015289307, |
| "rewards/rejected": -9.203264236450195, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.5141333333333333, |
| "grad_norm": 1.038760781288147, |
| "learning_rate": 2.5921079658767152e-06, |
| "logits/chosen": -1.7665719985961914, |
| "logits/rejected": -1.689576506614685, |
| "logps/chosen": -251.5525360107422, |
| "logps/rejected": -368.5316467285156, |
| "loss": 0.0539, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.5829997062683105, |
| "rewards/margins": 7.845690727233887, |
| "rewards/rejected": -10.428690910339355, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.5248, |
| "grad_norm": 3.1464333534240723, |
| "learning_rate": 2.56333178578297e-06, |
| "logits/chosen": -1.7736793756484985, |
| "logits/rejected": -1.6533575057983398, |
| "logps/chosen": -244.5554962158203, |
| "logps/rejected": -391.2909240722656, |
| "loss": 0.0767, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.007012128829956, |
| "rewards/margins": 8.108095169067383, |
| "rewards/rejected": -11.115106582641602, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.5354666666666668, |
| "grad_norm": 5.089426040649414, |
| "learning_rate": 2.534547206682136e-06, |
| "logits/chosen": -1.8151180744171143, |
| "logits/rejected": -1.696407675743103, |
| "logps/chosen": -249.90390014648438, |
| "logps/rejected": -380.46685791015625, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.8637096881866455, |
| "rewards/margins": 8.219526290893555, |
| "rewards/rejected": -11.083236694335938, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.5461333333333334, |
| "grad_norm": 1.6263580322265625, |
| "learning_rate": 2.5057580459603947e-06, |
| "logits/chosen": -1.8785078525543213, |
| "logits/rejected": -1.7948106527328491, |
| "logps/chosen": -234.8917694091797, |
| "logps/rejected": -399.46337890625, |
| "loss": 0.0607, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.608294725418091, |
| "rewards/margins": 8.606790542602539, |
| "rewards/rejected": -11.215084075927734, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.5568, |
| "grad_norm": 2.20981764793396, |
| "learning_rate": 2.4769681216115384e-06, |
| "logits/chosen": -1.7893867492675781, |
| "logits/rejected": -1.732273817062378, |
| "logps/chosen": -251.80380249023438, |
| "logps/rejected": -365.3934631347656, |
| "loss": 0.1178, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.6450085639953613, |
| "rewards/margins": 7.855576515197754, |
| "rewards/rejected": -10.500584602355957, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.5674666666666668, |
| "grad_norm": 0.7660602331161499, |
| "learning_rate": 2.448181251730632e-06, |
| "logits/chosen": -1.731345772743225, |
| "logits/rejected": -1.5998746156692505, |
| "logps/chosen": -263.6766662597656, |
| "logps/rejected": -396.3465270996094, |
| "loss": 0.0742, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.3900294303894043, |
| "rewards/margins": 8.074613571166992, |
| "rewards/rejected": -10.464642524719238, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.5781333333333334, |
| "grad_norm": 2.8943593502044678, |
| "learning_rate": 2.419401254007658e-06, |
| "logits/chosen": -1.7047817707061768, |
| "logits/rejected": -1.6618448495864868, |
| "logps/chosen": -270.00677490234375, |
| "logps/rejected": -373.75115966796875, |
| "loss": 0.0823, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.591595411300659, |
| "rewards/margins": 7.437920570373535, |
| "rewards/rejected": -10.029515266418457, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.5888, |
| "grad_norm": 1.2762200832366943, |
| "learning_rate": 2.390631945221221e-06, |
| "logits/chosen": -1.77741277217865, |
| "logits/rejected": -1.6593185663223267, |
| "logps/chosen": -259.33966064453125, |
| "logps/rejected": -387.9762878417969, |
| "loss": 0.0596, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.5528736114501953, |
| "rewards/margins": 8.286169052124023, |
| "rewards/rejected": -10.839041709899902, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.5994666666666668, |
| "grad_norm": 5.301697254180908, |
| "learning_rate": 2.361877140732366e-06, |
| "logits/chosen": -1.751280426979065, |
| "logits/rejected": -1.6818625926971436, |
| "logps/chosen": -244.2352752685547, |
| "logps/rejected": -358.234375, |
| "loss": 0.0983, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.585808277130127, |
| "rewards/margins": 6.942679405212402, |
| "rewards/rejected": -9.528488159179688, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.6101333333333332, |
| "grad_norm": 1.2639341354370117, |
| "learning_rate": 2.333140653978592e-06, |
| "logits/chosen": -1.8316749334335327, |
| "logits/rejected": -1.6760244369506836, |
| "logps/chosen": -247.3037567138672, |
| "logps/rejected": -367.29510498046875, |
| "loss": 0.11, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.250349760055542, |
| "rewards/margins": 7.623064994812012, |
| "rewards/rejected": -9.873414993286133, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.6208, |
| "grad_norm": 1.857134461402893, |
| "learning_rate": 2.3044262959681156e-06, |
| "logits/chosen": -1.769269585609436, |
| "logits/rejected": -1.5758507251739502, |
| "logps/chosen": -239.4264373779297, |
| "logps/rejected": -377.9754638671875, |
| "loss": 0.0973, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9940210580825806, |
| "rewards/margins": 7.984678745269775, |
| "rewards/rejected": -9.978699684143066, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.6314666666666666, |
| "grad_norm": 0.692950963973999, |
| "learning_rate": 2.2757378747744575e-06, |
| "logits/chosen": -1.8032305240631104, |
| "logits/rejected": -1.6365445852279663, |
| "logps/chosen": -232.6775360107422, |
| "logps/rejected": -371.04547119140625, |
| "loss": 0.0768, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.1179115772247314, |
| "rewards/margins": 8.067190170288086, |
| "rewards/rejected": -10.185101509094238, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.6421333333333332, |
| "grad_norm": 1.407834529876709, |
| "learning_rate": 2.24707919503142e-06, |
| "logits/chosen": -1.7327646017074585, |
| "logits/rejected": -1.648236870765686, |
| "logps/chosen": -250.6312713623047, |
| "logps/rejected": -385.9135437011719, |
| "loss": 0.0714, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2371888160705566, |
| "rewards/margins": 7.962684631347656, |
| "rewards/rejected": -10.199872970581055, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.6528, |
| "grad_norm": 4.060446739196777, |
| "learning_rate": 2.2184540574285213e-06, |
| "logits/chosen": -1.7848037481307983, |
| "logits/rejected": -1.646002173423767, |
| "logps/chosen": -259.26116943359375, |
| "logps/rejected": -352.891845703125, |
| "loss": 0.1054, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.8747847080230713, |
| "rewards/margins": 7.4474196434021, |
| "rewards/rejected": -9.322202682495117, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.6634666666666666, |
| "grad_norm": 2.409456968307495, |
| "learning_rate": 2.1898662582069486e-06, |
| "logits/chosen": -1.7882057428359985, |
| "logits/rejected": -1.7148845195770264, |
| "logps/chosen": -222.84060668945312, |
| "logps/rejected": -384.2849426269531, |
| "loss": 0.0724, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.966027855873108, |
| "rewards/margins": 8.193787574768066, |
| "rewards/rejected": -10.159815788269043, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.6741333333333333, |
| "grad_norm": 5.831104278564453, |
| "learning_rate": 2.1613195886561044e-06, |
| "logits/chosen": -1.7008816003799438, |
| "logits/rejected": -1.5513235330581665, |
| "logps/chosen": -243.56265258789062, |
| "logps/rejected": -361.69219970703125, |
| "loss": 0.1043, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.433258533477783, |
| "rewards/margins": 7.737143039703369, |
| "rewards/rejected": -10.170400619506836, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.6848, |
| "grad_norm": 5.545714378356934, |
| "learning_rate": 2.132817834610811e-06, |
| "logits/chosen": -1.7647472620010376, |
| "logits/rejected": -1.5817819833755493, |
| "logps/chosen": -237.57205200195312, |
| "logps/rejected": -385.9304504394531, |
| "loss": 0.1211, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0809152126312256, |
| "rewards/margins": 8.433000564575195, |
| "rewards/rejected": -10.513916015625, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.6954666666666667, |
| "grad_norm": 6.176883220672607, |
| "learning_rate": 2.1043647759492338e-06, |
| "logits/chosen": -1.6110525131225586, |
| "logits/rejected": -1.593756914138794, |
| "logps/chosen": -256.33465576171875, |
| "logps/rejected": -383.64947509765625, |
| "loss": 0.1472, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -1.9406496286392212, |
| "rewards/margins": 7.548769950866699, |
| "rewards/rejected": -9.489419937133789, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.7061333333333333, |
| "grad_norm": 4.009058952331543, |
| "learning_rate": 2.075964186091599e-06, |
| "logits/chosen": -1.7719929218292236, |
| "logits/rejected": -1.6545894145965576, |
| "logps/chosen": -240.3562469482422, |
| "logps/rejected": -371.539794921875, |
| "loss": 0.1034, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.4147119522094727, |
| "rewards/margins": 7.830977439880371, |
| "rewards/rejected": -10.24569034576416, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.7168, |
| "grad_norm": 1.6909562349319458, |
| "learning_rate": 2.047619831499765e-06, |
| "logits/chosen": -1.6593084335327148, |
| "logits/rejected": -1.5761816501617432, |
| "logps/chosen": -255.69204711914062, |
| "logps/rejected": -393.48992919921875, |
| "loss": 0.1291, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.199592113494873, |
| "rewards/margins": 7.6634626388549805, |
| "rewards/rejected": -9.863054275512695, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.7274666666666667, |
| "grad_norm": 5.588451862335205, |
| "learning_rate": 2.0193354711777155e-06, |
| "logits/chosen": -1.7532539367675781, |
| "logits/rejected": -1.6666532754898071, |
| "logps/chosen": -245.4121551513672, |
| "logps/rejected": -357.675048828125, |
| "loss": 0.1185, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.0722060203552246, |
| "rewards/margins": 7.035107612609863, |
| "rewards/rejected": -9.10731315612793, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.7381333333333333, |
| "grad_norm": 2.641136646270752, |
| "learning_rate": 1.991114856173048e-06, |
| "logits/chosen": -1.8587242364883423, |
| "logits/rejected": -1.757472276687622, |
| "logps/chosen": -224.07275390625, |
| "logps/rejected": -398.39373779296875, |
| "loss": 0.0711, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.255558967590332, |
| "rewards/margins": 8.534242630004883, |
| "rewards/rejected": -10.789800643920898, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.7488000000000001, |
| "grad_norm": 1.2425836324691772, |
| "learning_rate": 1.9629617290795115e-06, |
| "logits/chosen": -1.7562110424041748, |
| "logits/rejected": -1.6735032796859741, |
| "logps/chosen": -225.7513885498047, |
| "logps/rejected": -351.25933837890625, |
| "loss": 0.0889, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.073871612548828, |
| "rewards/margins": 7.723903656005859, |
| "rewards/rejected": -9.797774314880371, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.7594666666666665, |
| "grad_norm": 3.0822105407714844, |
| "learning_rate": 1.934879823540663e-06, |
| "logits/chosen": -1.7950897216796875, |
| "logits/rejected": -1.6102466583251953, |
| "logps/chosen": -242.1965789794922, |
| "logps/rejected": -376.4043884277344, |
| "loss": 0.1112, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.1286075115203857, |
| "rewards/margins": 8.270095825195312, |
| "rewards/rejected": -10.398704528808594, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.7701333333333333, |
| "grad_norm": 3.5807950496673584, |
| "learning_rate": 1.9068728637547195e-06, |
| "logits/chosen": -1.7565339803695679, |
| "logits/rejected": -1.6050052642822266, |
| "logps/chosen": -266.1463623046875, |
| "logps/rejected": -386.4732971191406, |
| "loss": 0.0924, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.24617338180542, |
| "rewards/margins": 8.325594902038574, |
| "rewards/rejected": -10.571767807006836, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.7808000000000002, |
| "grad_norm": 2.8825063705444336, |
| "learning_rate": 1.878944563980658e-06, |
| "logits/chosen": -1.7978938817977905, |
| "logits/rejected": -1.6335837841033936, |
| "logps/chosen": -234.8115234375, |
| "logps/rejected": -386.7230529785156, |
| "loss": 0.0674, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.029653310775757, |
| "rewards/margins": 8.498917579650879, |
| "rewards/rejected": -10.528570175170898, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.7914666666666665, |
| "grad_norm": 2.1994361877441406, |
| "learning_rate": 1.85109862804563e-06, |
| "logits/chosen": -1.7133188247680664, |
| "logits/rejected": -1.6955077648162842, |
| "logps/chosen": -253.84384155273438, |
| "logps/rejected": -391.4021301269531, |
| "loss": 0.1069, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.2396225929260254, |
| "rewards/margins": 7.77692174911499, |
| "rewards/rejected": -10.016545295715332, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.8021333333333334, |
| "grad_norm": 4.0932769775390625, |
| "learning_rate": 1.823338748853766e-06, |
| "logits/chosen": -1.794632911682129, |
| "logits/rejected": -1.7020469903945923, |
| "logps/chosen": -257.8614501953125, |
| "logps/rejected": -385.7154846191406, |
| "loss": 0.0992, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.1005051136016846, |
| "rewards/margins": 7.511053562164307, |
| "rewards/rejected": -9.611557960510254, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.8128, |
| "grad_norm": 3.011971950531006, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.7527557611465454, |
| "logits/rejected": -1.655992865562439, |
| "logps/chosen": -255.39816284179688, |
| "logps/rejected": -365.300048828125, |
| "loss": 0.1209, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.193533420562744, |
| "rewards/margins": 7.039409637451172, |
| "rewards/rejected": -9.232943534851074, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.8234666666666666, |
| "grad_norm": 2.4162063598632812, |
| "learning_rate": 1.7680918747639608e-06, |
| "logits/chosen": -1.7931512594223022, |
| "logits/rejected": -1.7776679992675781, |
| "logps/chosen": -250.3501434326172, |
| "logps/rejected": -377.2782897949219, |
| "loss": 0.1042, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -2.256478786468506, |
| "rewards/margins": 7.774122714996338, |
| "rewards/rejected": -10.030603408813477, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.8341333333333334, |
| "grad_norm": 2.753272533416748, |
| "learning_rate": 1.740612206659057e-06, |
| "logits/chosen": -1.7774744033813477, |
| "logits/rejected": -1.6850471496582031, |
| "logps/chosen": -249.3313751220703, |
| "logps/rejected": -375.1966552734375, |
| "loss": 0.1116, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.082430601119995, |
| "rewards/margins": 7.262882232666016, |
| "rewards/rejected": -9.345311164855957, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.8448, |
| "grad_norm": 5.52640438079834, |
| "learning_rate": 1.7132332479117204e-06, |
| "logits/chosen": -1.735599160194397, |
| "logits/rejected": -1.6529653072357178, |
| "logps/chosen": -242.7247314453125, |
| "logps/rejected": -383.18292236328125, |
| "loss": 0.102, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.9350744485855103, |
| "rewards/margins": 8.235949516296387, |
| "rewards/rejected": -10.171025276184082, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.8554666666666666, |
| "grad_norm": 3.504727840423584, |
| "learning_rate": 1.6859586294959666e-06, |
| "logits/chosen": -1.7679811716079712, |
| "logits/rejected": -1.6120868921279907, |
| "logps/chosen": -256.9747619628906, |
| "logps/rejected": -386.1347351074219, |
| "loss": 0.0554, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.9044342041015625, |
| "rewards/margins": 8.597646713256836, |
| "rewards/rejected": -10.502081871032715, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.8661333333333334, |
| "grad_norm": 2.296985626220703, |
| "learning_rate": 1.6587919685482884e-06, |
| "logits/chosen": -1.8257315158843994, |
| "logits/rejected": -1.6303374767303467, |
| "logps/chosen": -242.2867889404297, |
| "logps/rejected": -359.695556640625, |
| "loss": 0.0673, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.22623348236084, |
| "rewards/margins": 7.586366176605225, |
| "rewards/rejected": -9.812600135803223, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.8768, |
| "grad_norm": 1.439993143081665, |
| "learning_rate": 1.6317368678879497e-06, |
| "logits/chosen": -1.6686086654663086, |
| "logits/rejected": -1.5566412210464478, |
| "logps/chosen": -231.3364715576172, |
| "logps/rejected": -358.9420166015625, |
| "loss": 0.0984, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.624817132949829, |
| "rewards/margins": 7.7060065269470215, |
| "rewards/rejected": -9.33082389831543, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.8874666666666666, |
| "grad_norm": 2.0391299724578857, |
| "learning_rate": 1.6047969155391856e-06, |
| "logits/chosen": -1.6875035762786865, |
| "logits/rejected": -1.5945326089859009, |
| "logps/chosen": -264.5698547363281, |
| "logps/rejected": -373.34814453125, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.7919718027114868, |
| "rewards/margins": 7.827820777893066, |
| "rewards/rejected": -9.619791984558105, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.8981333333333335, |
| "grad_norm": 2.268467426300049, |
| "learning_rate": 1.5779756842553617e-06, |
| "logits/chosen": -1.7344768047332764, |
| "logits/rejected": -1.6171687841415405, |
| "logps/chosen": -244.0048828125, |
| "logps/rejected": -379.1364440917969, |
| "loss": 0.1076, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.9673188924789429, |
| "rewards/margins": 7.8214263916015625, |
| "rewards/rejected": -9.788744926452637, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.9088, |
| "grad_norm": 1.3239772319793701, |
| "learning_rate": 1.551276731045157e-06, |
| "logits/chosen": -1.758827567100525, |
| "logits/rejected": -1.612735390663147, |
| "logps/chosen": -256.748779296875, |
| "logps/rejected": -374.85943603515625, |
| "loss": 0.0809, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.4343198537826538, |
| "rewards/margins": 7.9875168800354, |
| "rewards/rejected": -9.421835899353027, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.9194666666666667, |
| "grad_norm": 2.2805094718933105, |
| "learning_rate": 1.5247035967008369e-06, |
| "logits/chosen": -1.7128002643585205, |
| "logits/rejected": -1.556166410446167, |
| "logps/chosen": -239.7552490234375, |
| "logps/rejected": -344.92205810546875, |
| "loss": 0.1143, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.8794777393341064, |
| "rewards/margins": 7.6836981773376465, |
| "rewards/rejected": -9.563177108764648, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9301333333333335, |
| "grad_norm": 0.7183341979980469, |
| "learning_rate": 1.498259805328678e-06, |
| "logits/chosen": -1.7574424743652344, |
| "logits/rejected": -1.6206070184707642, |
| "logps/chosen": -240.24465942382812, |
| "logps/rejected": -349.49127197265625, |
| "loss": 0.1213, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0217463970184326, |
| "rewards/margins": 7.537483215332031, |
| "rewards/rejected": -9.559228897094727, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.9407999999999999, |
| "grad_norm": 3.615495443344116, |
| "learning_rate": 1.471948863881601e-06, |
| "logits/chosen": -1.759718656539917, |
| "logits/rejected": -1.6743332147598267, |
| "logps/chosen": -258.42999267578125, |
| "logps/rejected": -365.8958435058594, |
| "loss": 0.073, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8100903034210205, |
| "rewards/margins": 7.1778883934021, |
| "rewards/rejected": -8.9879789352417, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.9514666666666667, |
| "grad_norm": 3.234266996383667, |
| "learning_rate": 1.4457742616940864e-06, |
| "logits/chosen": -1.6585487127304077, |
| "logits/rejected": -1.5793458223342896, |
| "logps/chosen": -274.3389587402344, |
| "logps/rejected": -381.27581787109375, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9123642444610596, |
| "rewards/margins": 7.680113792419434, |
| "rewards/rejected": -9.59247875213623, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.9621333333333333, |
| "grad_norm": 1.0968949794769287, |
| "learning_rate": 1.4197394700194165e-06, |
| "logits/chosen": -1.7973190546035767, |
| "logits/rejected": -1.7022031545639038, |
| "logps/chosen": -233.85910034179688, |
| "logps/rejected": -363.92987060546875, |
| "loss": 0.0566, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6231153011322021, |
| "rewards/margins": 8.063187599182129, |
| "rewards/rejected": -9.68630313873291, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.9727999999999999, |
| "grad_norm": 5.4347100257873535, |
| "learning_rate": 1.3938479415693244e-06, |
| "logits/chosen": -1.8134357929229736, |
| "logits/rejected": -1.7727657556533813, |
| "logps/chosen": -241.57418823242188, |
| "logps/rejected": -374.43780517578125, |
| "loss": 0.107, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.7810213565826416, |
| "rewards/margins": 7.694109916687012, |
| "rewards/rejected": -9.475130081176758, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.9834666666666667, |
| "grad_norm": 3.721148729324341, |
| "learning_rate": 1.368103110056096e-06, |
| "logits/chosen": -1.7720524072647095, |
| "logits/rejected": -1.690016508102417, |
| "logps/chosen": -250.36215209960938, |
| "logps/rejected": -392.93994140625, |
| "loss": 0.0789, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -1.9710109233856201, |
| "rewards/margins": 8.534833908081055, |
| "rewards/rejected": -10.50584602355957, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.9941333333333333, |
| "grad_norm": 1.1054441928863525, |
| "learning_rate": 1.3425083897371983e-06, |
| "logits/chosen": -1.6688096523284912, |
| "logits/rejected": -1.5874191522598267, |
| "logps/chosen": -264.9595031738281, |
| "logps/rejected": -363.69940185546875, |
| "loss": 0.1013, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.360398530960083, |
| "rewards/margins": 7.5831756591796875, |
| "rewards/rejected": -8.943574905395508, |
| "step": 935 |
| }, |
| { |
| "epoch": 2.0042666666666666, |
| "grad_norm": 1.7867152690887451, |
| "learning_rate": 1.317067174962478e-06, |
| "logits/chosen": -1.7479714155197144, |
| "logits/rejected": -1.5715677738189697, |
| "logps/chosen": -248.16004943847656, |
| "logps/rejected": -350.427978515625, |
| "loss": 0.0779, |
| "rewards/accuracies": 0.9802631735801697, |
| "rewards/chosen": -1.6614056825637817, |
| "rewards/margins": 7.558930397033691, |
| "rewards/rejected": -9.220335960388184, |
| "step": 940 |
| }, |
| { |
| "epoch": 2.0149333333333335, |
| "grad_norm": 2.6329009532928467, |
| "learning_rate": 1.291782839724009e-06, |
| "logits/chosen": -1.724474549293518, |
| "logits/rejected": -1.6523854732513428, |
| "logps/chosen": -265.9844665527344, |
| "logps/rejected": -377.27471923828125, |
| "loss": 0.0861, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9542032480239868, |
| "rewards/margins": 7.510810852050781, |
| "rewards/rejected": -9.46501350402832, |
| "step": 945 |
| }, |
| { |
| "epoch": 2.0256, |
| "grad_norm": 3.648782968521118, |
| "learning_rate": 1.2666587372086352e-06, |
| "logits/chosen": -1.7386844158172607, |
| "logits/rejected": -1.630690336227417, |
| "logps/chosen": -201.80908203125, |
| "logps/rejected": -352.9446716308594, |
| "loss": 0.078, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6233279705047607, |
| "rewards/margins": 7.797116756439209, |
| "rewards/rejected": -9.420445442199707, |
| "step": 950 |
| }, |
| { |
| "epoch": 2.0362666666666667, |
| "grad_norm": 0.9289124011993408, |
| "learning_rate": 1.241698199353275e-06, |
| "logits/chosen": -1.8293838500976562, |
| "logits/rejected": -1.7059133052825928, |
| "logps/chosen": -230.53854370117188, |
| "logps/rejected": -376.99884033203125, |
| "loss": 0.0661, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.746546983718872, |
| "rewards/margins": 8.739092826843262, |
| "rewards/rejected": -10.485639572143555, |
| "step": 955 |
| }, |
| { |
| "epoch": 2.0469333333333335, |
| "grad_norm": 0.8716382384300232, |
| "learning_rate": 1.2169045364030404e-06, |
| "logits/chosen": -1.8371355533599854, |
| "logits/rejected": -1.6864582300186157, |
| "logps/chosen": -228.82418823242188, |
| "logps/rejected": -379.0071105957031, |
| "loss": 0.0664, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8618011474609375, |
| "rewards/margins": 8.489458084106445, |
| "rewards/rejected": -10.351259231567383, |
| "step": 960 |
| }, |
| { |
| "epoch": 2.0576, |
| "grad_norm": 2.5190982818603516, |
| "learning_rate": 1.1922810364722402e-06, |
| "logits/chosen": -1.7426660060882568, |
| "logits/rejected": -1.6516612768173218, |
| "logps/chosen": -269.8039855957031, |
| "logps/rejected": -393.49322509765625, |
| "loss": 0.0921, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.9855560064315796, |
| "rewards/margins": 7.847367286682129, |
| "rewards/rejected": -9.832923889160156, |
| "step": 965 |
| }, |
| { |
| "epoch": 2.0682666666666667, |
| "grad_norm": 0.5930553078651428, |
| "learning_rate": 1.1678309651083055e-06, |
| "logits/chosen": -1.806814193725586, |
| "logits/rejected": -1.695652961730957, |
| "logps/chosen": -225.7905731201172, |
| "logps/rejected": -384.2733459472656, |
| "loss": 0.0463, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9584178924560547, |
| "rewards/margins": 9.098180770874023, |
| "rewards/rejected": -11.056598663330078, |
| "step": 970 |
| }, |
| { |
| "epoch": 2.0789333333333335, |
| "grad_norm": 0.8087952136993408, |
| "learning_rate": 1.1435575648587224e-06, |
| "logits/chosen": -1.695840835571289, |
| "logits/rejected": -1.6272556781768799, |
| "logps/chosen": -250.8284454345703, |
| "logps/rejected": -380.99847412109375, |
| "loss": 0.0551, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.8700387477874756, |
| "rewards/margins": 8.868024826049805, |
| "rewards/rejected": -10.738062858581543, |
| "step": 975 |
| }, |
| { |
| "epoch": 2.0896, |
| "grad_norm": 1.1131638288497925, |
| "learning_rate": 1.1194640548410039e-06, |
| "logits/chosen": -1.637101173400879, |
| "logits/rejected": -1.589843511581421, |
| "logps/chosen": -263.36175537109375, |
| "logps/rejected": -357.9180603027344, |
| "loss": 0.0818, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.8525962829589844, |
| "rewards/margins": 7.566631317138672, |
| "rewards/rejected": -9.419228553771973, |
| "step": 980 |
| }, |
| { |
| "epoch": 2.1002666666666667, |
| "grad_norm": 1.2178566455841064, |
| "learning_rate": 1.095553630315774e-06, |
| "logits/chosen": -1.698826789855957, |
| "logits/rejected": -1.611319899559021, |
| "logps/chosen": -253.77139282226562, |
| "logps/rejected": -394.76702880859375, |
| "loss": 0.0705, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.7703306674957275, |
| "rewards/margins": 8.478639602661133, |
| "rewards/rejected": -10.248971939086914, |
| "step": 985 |
| }, |
| { |
| "epoch": 2.1109333333333336, |
| "grad_norm": 1.0288853645324707, |
| "learning_rate": 1.0718294622630188e-06, |
| "logits/chosen": -1.676661729812622, |
| "logits/rejected": -1.6332896947860718, |
| "logps/chosen": -252.76467895507812, |
| "logps/rejected": -379.791748046875, |
| "loss": 0.0929, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8333266973495483, |
| "rewards/margins": 7.959515571594238, |
| "rewards/rejected": -9.792842864990234, |
| "step": 990 |
| }, |
| { |
| "epoch": 2.1216, |
| "grad_norm": 2.387310266494751, |
| "learning_rate": 1.0482946969615495e-06, |
| "logits/chosen": -1.8010175228118896, |
| "logits/rejected": -1.6718385219573975, |
| "logps/chosen": -241.35171508789062, |
| "logps/rejected": -376.3408508300781, |
| "loss": 0.0469, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.301366090774536, |
| "rewards/margins": 8.878290176391602, |
| "rewards/rejected": -11.179656982421875, |
| "step": 995 |
| }, |
| { |
| "epoch": 2.1322666666666668, |
| "grad_norm": 3.4971799850463867, |
| "learning_rate": 1.0249524555717461e-06, |
| "logits/chosen": -1.787679672241211, |
| "logits/rejected": -1.628638505935669, |
| "logps/chosen": -230.5235137939453, |
| "logps/rejected": -347.1987609863281, |
| "loss": 0.0644, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.6777393817901611, |
| "rewards/margins": 7.927115440368652, |
| "rewards/rejected": -9.604854583740234, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.142933333333333, |
| "grad_norm": 0.9783017635345459, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.654268503189087, |
| "logits/rejected": -1.5726966857910156, |
| "logps/chosen": -252.07546997070312, |
| "logps/rejected": -372.8138732910156, |
| "loss": 0.0587, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -1.9420814514160156, |
| "rewards/margins": 8.198111534118652, |
| "rewards/rejected": -10.140192985534668, |
| "step": 1005 |
| }, |
| { |
| "epoch": 2.1536, |
| "grad_norm": 2.6391026973724365, |
| "learning_rate": 9.788579010963387e-07, |
| "logits/chosen": -1.7386356592178345, |
| "logits/rejected": -1.6452341079711914, |
| "logps/chosen": -250.1106719970703, |
| "logps/rejected": -376.52764892578125, |
| "loss": 0.0753, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.8499062061309814, |
| "rewards/margins": 8.821678161621094, |
| "rewards/rejected": -10.671584129333496, |
| "step": 1010 |
| }, |
| { |
| "epoch": 2.164266666666667, |
| "grad_norm": 4.805166721343994, |
| "learning_rate": 9.56111701030998e-07, |
| "logits/chosen": -1.8246469497680664, |
| "logits/rejected": -1.643933892250061, |
| "logps/chosen": -256.89422607421875, |
| "logps/rejected": -388.6980895996094, |
| "loss": 0.0949, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.3212974071502686, |
| "rewards/margins": 8.553018569946289, |
| "rewards/rejected": -10.874316215515137, |
| "step": 1015 |
| }, |
| { |
| "epoch": 2.174933333333333, |
| "grad_norm": 1.8791966438293457, |
| "learning_rate": 9.335702501071459e-07, |
| "logits/chosen": -1.7584949731826782, |
| "logits/rejected": -1.5843560695648193, |
| "logps/chosen": -254.95584106445312, |
| "logps/rejected": -365.45166015625, |
| "loss": 0.1029, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.2223076820373535, |
| "rewards/margins": 7.929934501647949, |
| "rewards/rejected": -10.152241706848145, |
| "step": 1020 |
| }, |
| { |
| "epoch": 2.1856, |
| "grad_norm": 1.4675019979476929, |
| "learning_rate": 9.112365377526636e-07, |
| "logits/chosen": -1.7484461069107056, |
| "logits/rejected": -1.598569631576538, |
| "logps/chosen": -268.4307861328125, |
| "logps/rejected": -397.59100341796875, |
| "loss": 0.0757, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.514547348022461, |
| "rewards/margins": 8.377134323120117, |
| "rewards/rejected": -10.891680717468262, |
| "step": 1025 |
| }, |
| { |
| "epoch": 2.196266666666667, |
| "grad_norm": 2.629225015640259, |
| "learning_rate": 8.891135258453195e-07, |
| "logits/chosen": -1.6625139713287354, |
| "logits/rejected": -1.5738680362701416, |
| "logps/chosen": -246.3118133544922, |
| "logps/rejected": -382.7955627441406, |
| "loss": 0.0669, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.123089551925659, |
| "rewards/margins": 8.360490798950195, |
| "rewards/rejected": -10.483579635620117, |
| "step": 1030 |
| }, |
| { |
| "epoch": 2.206933333333333, |
| "grad_norm": 1.3968008756637573, |
| "learning_rate": 8.672041483199709e-07, |
| "logits/chosen": -1.670629858970642, |
| "logits/rejected": -1.5821272134780884, |
| "logps/chosen": -268.6837158203125, |
| "logps/rejected": -375.99835205078125, |
| "loss": 0.071, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3144733905792236, |
| "rewards/margins": 8.026359558105469, |
| "rewards/rejected": -10.340832710266113, |
| "step": 1035 |
| }, |
| { |
| "epoch": 2.2176, |
| "grad_norm": 2.3471174240112305, |
| "learning_rate": 8.455113107794652e-07, |
| "logits/chosen": -1.736445665359497, |
| "logits/rejected": -1.6636123657226562, |
| "logps/chosen": -240.90966796875, |
| "logps/rejected": -368.02459716796875, |
| "loss": 0.0671, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.1975467205047607, |
| "rewards/margins": 8.53138542175293, |
| "rewards/rejected": -10.72893238067627, |
| "step": 1040 |
| }, |
| { |
| "epoch": 2.228266666666667, |
| "grad_norm": 2.631293773651123, |
| "learning_rate": 8.240378901093035e-07, |
| "logits/chosen": -1.728139877319336, |
| "logits/rejected": -1.565740942955017, |
| "logps/chosen": -247.2626190185547, |
| "logps/rejected": -365.80255126953125, |
| "loss": 0.064, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.0829832553863525, |
| "rewards/margins": 8.492073059082031, |
| "rewards/rejected": -10.575056076049805, |
| "step": 1045 |
| }, |
| { |
| "epoch": 2.238933333333333, |
| "grad_norm": 0.5728951692581177, |
| "learning_rate": 8.027867340961113e-07, |
| "logits/chosen": -1.673462152481079, |
| "logits/rejected": -1.5609209537506104, |
| "logps/chosen": -229.69677734375, |
| "logps/rejected": -372.80145263671875, |
| "loss": 0.047, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.137375593185425, |
| "rewards/margins": 8.91349983215332, |
| "rewards/rejected": -11.050874710083008, |
| "step": 1050 |
| }, |
| { |
| "epoch": 2.2496, |
| "grad_norm": 3.399742364883423, |
| "learning_rate": 7.817606610499639e-07, |
| "logits/chosen": -1.8059459924697876, |
| "logits/rejected": -1.6398189067840576, |
| "logps/chosen": -245.95883178710938, |
| "logps/rejected": -367.82415771484375, |
| "loss": 0.0761, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.1454949378967285, |
| "rewards/margins": 8.3766450881958, |
| "rewards/rejected": -10.522140502929688, |
| "step": 1055 |
| }, |
| { |
| "epoch": 2.2602666666666664, |
| "grad_norm": 1.9253159761428833, |
| "learning_rate": 7.609624594306278e-07, |
| "logits/chosen": -1.772464394569397, |
| "logits/rejected": -1.6853172779083252, |
| "logps/chosen": -240.6079559326172, |
| "logps/rejected": -407.1769104003906, |
| "loss": 0.0615, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.328904628753662, |
| "rewards/margins": 8.971172332763672, |
| "rewards/rejected": -11.300076484680176, |
| "step": 1060 |
| }, |
| { |
| "epoch": 2.2709333333333332, |
| "grad_norm": 0.7144515514373779, |
| "learning_rate": 7.403948874777556e-07, |
| "logits/chosen": -1.7335847616195679, |
| "logits/rejected": -1.6609535217285156, |
| "logps/chosen": -245.9636993408203, |
| "logps/rejected": -373.9549560546875, |
| "loss": 0.0584, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3022031784057617, |
| "rewards/margins": 8.158570289611816, |
| "rewards/rejected": -10.460772514343262, |
| "step": 1065 |
| }, |
| { |
| "epoch": 2.2816, |
| "grad_norm": 1.1433680057525635, |
| "learning_rate": 7.20060672845091e-07, |
| "logits/chosen": -1.7069857120513916, |
| "logits/rejected": -1.6626923084259033, |
| "logps/chosen": -247.3767547607422, |
| "logps/rejected": -388.65264892578125, |
| "loss": 0.0797, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.4776415824890137, |
| "rewards/margins": 8.408472061157227, |
| "rewards/rejected": -10.886113166809082, |
| "step": 1070 |
| }, |
| { |
| "epoch": 2.2922666666666665, |
| "grad_norm": 2.0684447288513184, |
| "learning_rate": 6.99962512238733e-07, |
| "logits/chosen": -1.5935136079788208, |
| "logits/rejected": -1.552176833152771, |
| "logps/chosen": -240.5950927734375, |
| "logps/rejected": -382.0034484863281, |
| "loss": 0.0682, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.435925245285034, |
| "rewards/margins": 8.210624694824219, |
| "rewards/rejected": -10.646550178527832, |
| "step": 1075 |
| }, |
| { |
| "epoch": 2.3029333333333333, |
| "grad_norm": 3.9305777549743652, |
| "learning_rate": 6.801030710594949e-07, |
| "logits/chosen": -1.6194684505462646, |
| "logits/rejected": -1.521071195602417, |
| "logps/chosen": -240.2565155029297, |
| "logps/rejected": -384.17315673828125, |
| "loss": 0.0681, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.3575754165649414, |
| "rewards/margins": 9.207578659057617, |
| "rewards/rejected": -11.565154075622559, |
| "step": 1080 |
| }, |
| { |
| "epoch": 2.3136, |
| "grad_norm": 3.65750789642334, |
| "learning_rate": 6.604849830494264e-07, |
| "logits/chosen": -1.7517938613891602, |
| "logits/rejected": -1.7062915563583374, |
| "logps/chosen": -236.4104461669922, |
| "logps/rejected": -396.84033203125, |
| "loss": 0.0568, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.530136823654175, |
| "rewards/margins": 9.359679222106934, |
| "rewards/rejected": -11.889817237854004, |
| "step": 1085 |
| }, |
| { |
| "epoch": 2.3242666666666665, |
| "grad_norm": 1.4848530292510986, |
| "learning_rate": 6.411108499425253e-07, |
| "logits/chosen": -1.7016388177871704, |
| "logits/rejected": -1.6188669204711914, |
| "logps/chosen": -243.3367462158203, |
| "logps/rejected": -396.5169982910156, |
| "loss": 0.0546, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.394667625427246, |
| "rewards/margins": 8.645791053771973, |
| "rewards/rejected": -11.040457725524902, |
| "step": 1090 |
| }, |
| { |
| "epoch": 2.3349333333333333, |
| "grad_norm": 3.0178022384643555, |
| "learning_rate": 6.219832411197013e-07, |
| "logits/chosen": -1.6480319499969482, |
| "logits/rejected": -1.4744436740875244, |
| "logps/chosen": -265.7557067871094, |
| "logps/rejected": -374.42816162109375, |
| "loss": 0.0744, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.212043046951294, |
| "rewards/margins": 8.311385154724121, |
| "rewards/rejected": -10.523427963256836, |
| "step": 1095 |
| }, |
| { |
| "epoch": 2.3456, |
| "grad_norm": 2.0852251052856445, |
| "learning_rate": 6.031046932680229e-07, |
| "logits/chosen": -1.7345670461654663, |
| "logits/rejected": -1.609113335609436, |
| "logps/chosen": -247.75338745117188, |
| "logps/rejected": -375.01605224609375, |
| "loss": 0.0828, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.2977194786071777, |
| "rewards/margins": 7.741724967956543, |
| "rewards/rejected": -10.039443969726562, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.3562666666666665, |
| "grad_norm": 3.485975980758667, |
| "learning_rate": 5.844777100443091e-07, |
| "logits/chosen": -1.7265936136245728, |
| "logits/rejected": -1.6472886800765991, |
| "logps/chosen": -248.5940704345703, |
| "logps/rejected": -372.21319580078125, |
| "loss": 0.0872, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.629234790802002, |
| "rewards/margins": 8.200715065002441, |
| "rewards/rejected": -10.829948425292969, |
| "step": 1105 |
| }, |
| { |
| "epoch": 2.3669333333333333, |
| "grad_norm": 0.6572098135948181, |
| "learning_rate": 5.66104761743092e-07, |
| "logits/chosen": -1.7917197942733765, |
| "logits/rejected": -1.7144348621368408, |
| "logps/chosen": -228.86181640625, |
| "logps/rejected": -405.03009033203125, |
| "loss": 0.0416, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5428788661956787, |
| "rewards/margins": 9.689337730407715, |
| "rewards/rejected": -12.232217788696289, |
| "step": 1110 |
| }, |
| { |
| "epoch": 2.3776, |
| "grad_norm": 4.4979777336120605, |
| "learning_rate": 5.479882849690102e-07, |
| "logits/chosen": -1.708397626876831, |
| "logits/rejected": -1.6033588647842407, |
| "logps/chosen": -244.21762084960938, |
| "logps/rejected": -382.5280456542969, |
| "loss": 0.0715, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.0844390392303467, |
| "rewards/margins": 9.081504821777344, |
| "rewards/rejected": -11.165945053100586, |
| "step": 1115 |
| }, |
| { |
| "epoch": 2.3882666666666665, |
| "grad_norm": 1.708503007888794, |
| "learning_rate": 5.301306823136687e-07, |
| "logits/chosen": -1.6154632568359375, |
| "logits/rejected": -1.5289294719696045, |
| "logps/chosen": -249.3914794921875, |
| "logps/rejected": -383.60150146484375, |
| "loss": 0.0559, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.4148969650268555, |
| "rewards/margins": 8.664996147155762, |
| "rewards/rejected": -11.079894065856934, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.3989333333333334, |
| "grad_norm": 4.531334400177002, |
| "learning_rate": 5.125343220370088e-07, |
| "logits/chosen": -1.7035802602767944, |
| "logits/rejected": -1.6139618158340454, |
| "logps/chosen": -239.21157836914062, |
| "logps/rejected": -351.17828369140625, |
| "loss": 0.0821, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -1.9980465173721313, |
| "rewards/margins": 8.510173797607422, |
| "rewards/rejected": -10.508219718933105, |
| "step": 1125 |
| }, |
| { |
| "epoch": 2.4096, |
| "grad_norm": 1.7990802526474, |
| "learning_rate": 4.952015377532329e-07, |
| "logits/chosen": -1.7770929336547852, |
| "logits/rejected": -1.6199886798858643, |
| "logps/chosen": -244.71224975585938, |
| "logps/rejected": -403.7447509765625, |
| "loss": 0.0349, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.51416015625, |
| "rewards/margins": 8.962092399597168, |
| "rewards/rejected": -11.476252555847168, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.4202666666666666, |
| "grad_norm": 0.9469951391220093, |
| "learning_rate": 4.781346281213204e-07, |
| "logits/chosen": -1.7762229442596436, |
| "logits/rejected": -1.6669120788574219, |
| "logps/chosen": -256.1883544921875, |
| "logps/rejected": -393.80706787109375, |
| "loss": 0.0476, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.724867820739746, |
| "rewards/margins": 9.057721138000488, |
| "rewards/rejected": -11.78258991241455, |
| "step": 1135 |
| }, |
| { |
| "epoch": 2.4309333333333334, |
| "grad_norm": 2.5484354496002197, |
| "learning_rate": 4.613358565401818e-07, |
| "logits/chosen": -1.6508283615112305, |
| "logits/rejected": -1.5555996894836426, |
| "logps/chosen": -270.6368408203125, |
| "logps/rejected": -405.41363525390625, |
| "loss": 0.0857, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.6728577613830566, |
| "rewards/margins": 8.623315811157227, |
| "rewards/rejected": -11.296173095703125, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.4416, |
| "grad_norm": 3.9779036045074463, |
| "learning_rate": 4.448074508484906e-07, |
| "logits/chosen": -1.67898690700531, |
| "logits/rejected": -1.5510108470916748, |
| "logps/chosen": -245.95962524414062, |
| "logps/rejected": -371.1490783691406, |
| "loss": 0.0697, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.447220802307129, |
| "rewards/margins": 8.701111793518066, |
| "rewards/rejected": -11.148332595825195, |
| "step": 1145 |
| }, |
| { |
| "epoch": 2.4522666666666666, |
| "grad_norm": 2.5895211696624756, |
| "learning_rate": 4.2855160302922777e-07, |
| "logits/chosen": -1.7640804052352905, |
| "logits/rejected": -1.6478509902954102, |
| "logps/chosen": -235.4073028564453, |
| "logps/rejected": -381.1341247558594, |
| "loss": 0.0629, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.591489791870117, |
| "rewards/margins": 8.705648422241211, |
| "rewards/rejected": -11.297138214111328, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.4629333333333334, |
| "grad_norm": 2.4404029846191406, |
| "learning_rate": 4.125704689189819e-07, |
| "logits/chosen": -1.6468175649642944, |
| "logits/rejected": -1.5339540243148804, |
| "logps/chosen": -265.91986083984375, |
| "logps/rejected": -380.76019287109375, |
| "loss": 0.0784, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.748948574066162, |
| "rewards/margins": 7.9840497970581055, |
| "rewards/rejected": -10.732998847961426, |
| "step": 1155 |
| }, |
| { |
| "epoch": 2.4736000000000002, |
| "grad_norm": 2.37691330909729, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.718951940536499, |
| "logits/rejected": -1.6564605236053467, |
| "logps/chosen": -266.5101013183594, |
| "logps/rejected": -405.48638916015625, |
| "loss": 0.05, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.9024240970611572, |
| "rewards/margins": 9.125859260559082, |
| "rewards/rejected": -12.028284072875977, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.4842666666666666, |
| "grad_norm": 1.4729171991348267, |
| "learning_rate": 3.8144078272934676e-07, |
| "logits/chosen": -1.6833305358886719, |
| "logits/rejected": -1.5863052606582642, |
| "logps/chosen": -255.60757446289062, |
| "logps/rejected": -390.80413818359375, |
| "loss": 0.045, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.29118275642395, |
| "rewards/margins": 9.308106422424316, |
| "rewards/rejected": -11.59929084777832, |
| "step": 1165 |
| }, |
| { |
| "epoch": 2.4949333333333334, |
| "grad_norm": 2.2732648849487305, |
| "learning_rate": 3.6629635904222943e-07, |
| "logits/chosen": -1.6575851440429688, |
| "logits/rejected": -1.6193211078643799, |
| "logps/chosen": -245.96743774414062, |
| "logps/rejected": -375.4206848144531, |
| "loss": 0.0516, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.344364881515503, |
| "rewards/margins": 8.420816421508789, |
| "rewards/rejected": -10.765182495117188, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.5056000000000003, |
| "grad_norm": 1.3393036127090454, |
| "learning_rate": 3.514349053011698e-07, |
| "logits/chosen": -1.6279098987579346, |
| "logits/rejected": -1.5382213592529297, |
| "logps/chosen": -235.5139923095703, |
| "logps/rejected": -388.8031921386719, |
| "loss": 0.1437, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.400181531906128, |
| "rewards/margins": 8.55920124053955, |
| "rewards/rejected": -10.959383010864258, |
| "step": 1175 |
| }, |
| { |
| "epoch": 2.5162666666666667, |
| "grad_norm": 2.17390775680542, |
| "learning_rate": 3.3685839241941133e-07, |
| "logits/chosen": -1.6647920608520508, |
| "logits/rejected": -1.5797088146209717, |
| "logps/chosen": -253.29006958007812, |
| "logps/rejected": -357.0965881347656, |
| "loss": 0.0557, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.246269464492798, |
| "rewards/margins": 8.394041061401367, |
| "rewards/rejected": -10.640310287475586, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.5269333333333335, |
| "grad_norm": 3.978947877883911, |
| "learning_rate": 3.2256875352158435e-07, |
| "logits/chosen": -1.7043958902359009, |
| "logits/rejected": -1.5851062536239624, |
| "logps/chosen": -250.3873748779297, |
| "logps/rejected": -386.988037109375, |
| "loss": 0.0816, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.7624685764312744, |
| "rewards/margins": 8.765403747558594, |
| "rewards/rejected": -11.527873039245605, |
| "step": 1185 |
| }, |
| { |
| "epoch": 2.5376, |
| "grad_norm": 1.3339006900787354, |
| "learning_rate": 3.085678836873399e-07, |
| "logits/chosen": -1.7824872732162476, |
| "logits/rejected": -1.6766750812530518, |
| "logps/chosen": -248.77224731445312, |
| "logps/rejected": -390.64532470703125, |
| "loss": 0.0765, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.545628309249878, |
| "rewards/margins": 8.543272972106934, |
| "rewards/rejected": -11.088900566101074, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.5482666666666667, |
| "grad_norm": 2.1457443237304688, |
| "learning_rate": 2.9485763970002106e-07, |
| "logits/chosen": -1.7480427026748657, |
| "logits/rejected": -1.588936448097229, |
| "logps/chosen": -245.76425170898438, |
| "logps/rejected": -367.2337646484375, |
| "loss": 0.0665, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.6265664100646973, |
| "rewards/margins": 8.219732284545898, |
| "rewards/rejected": -10.846298217773438, |
| "step": 1195 |
| }, |
| { |
| "epoch": 2.558933333333333, |
| "grad_norm": 2.72080397605896, |
| "learning_rate": 2.8143983980042174e-07, |
| "logits/chosen": -1.7567298412322998, |
| "logits/rejected": -1.6189864873886108, |
| "logps/chosen": -238.1021728515625, |
| "logps/rejected": -386.2373962402344, |
| "loss": 0.0649, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4193553924560547, |
| "rewards/margins": 9.376398086547852, |
| "rewards/rejected": -11.795753479003906, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.5696, |
| "grad_norm": 3.7256271839141846, |
| "learning_rate": 2.683162634456515e-07, |
| "logits/chosen": -1.6302845478057861, |
| "logits/rejected": -1.535885214805603, |
| "logps/chosen": -259.4132385253906, |
| "logps/rejected": -407.36553955078125, |
| "loss": 0.0608, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.306084394454956, |
| "rewards/margins": 9.6066312789917, |
| "rewards/rejected": -11.91271686553955, |
| "step": 1205 |
| }, |
| { |
| "epoch": 2.5802666666666667, |
| "grad_norm": 4.272429943084717, |
| "learning_rate": 2.5548865107314606e-07, |
| "logits/chosen": -1.6210415363311768, |
| "logits/rejected": -1.532249927520752, |
| "logps/chosen": -274.93890380859375, |
| "logps/rejected": -375.2662353515625, |
| "loss": 0.0568, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.343566417694092, |
| "rewards/margins": 8.680849075317383, |
| "rewards/rejected": -11.024415969848633, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.590933333333333, |
| "grad_norm": 3.4025235176086426, |
| "learning_rate": 2.429587038698511e-07, |
| "logits/chosen": -1.687776803970337, |
| "logits/rejected": -1.6682751178741455, |
| "logps/chosen": -225.93594360351562, |
| "logps/rejected": -393.0034484863281, |
| "loss": 0.0495, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5923023223876953, |
| "rewards/margins": 9.531915664672852, |
| "rewards/rejected": -12.124217987060547, |
| "step": 1215 |
| }, |
| { |
| "epoch": 2.6016, |
| "grad_norm": 1.6154532432556152, |
| "learning_rate": 2.307280835466144e-07, |
| "logits/chosen": -1.658247709274292, |
| "logits/rejected": -1.6115080118179321, |
| "logps/chosen": -238.1659698486328, |
| "logps/rejected": -411.9134826660156, |
| "loss": 0.054, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.8724894523620605, |
| "rewards/margins": 9.156352043151855, |
| "rewards/rejected": -12.028841018676758, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.6122666666666667, |
| "grad_norm": 1.3085912466049194, |
| "learning_rate": 2.1879841211780856e-07, |
| "logits/chosen": -1.6801115274429321, |
| "logits/rejected": -1.6072133779525757, |
| "logps/chosen": -246.29122924804688, |
| "logps/rejected": -375.62725830078125, |
| "loss": 0.0622, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.3886702060699463, |
| "rewards/margins": 8.405167579650879, |
| "rewards/rejected": -10.793837547302246, |
| "step": 1225 |
| }, |
| { |
| "epoch": 2.622933333333333, |
| "grad_norm": 1.8979953527450562, |
| "learning_rate": 2.0717127168622202e-07, |
| "logits/chosen": -1.7112290859222412, |
| "logits/rejected": -1.634482979774475, |
| "logps/chosen": -248.1447296142578, |
| "logps/rejected": -380.28729248046875, |
| "loss": 0.0548, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.502384901046753, |
| "rewards/margins": 8.42490005493164, |
| "rewards/rejected": -10.927285194396973, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.6336, |
| "grad_norm": 1.139436960220337, |
| "learning_rate": 1.958482042332424e-07, |
| "logits/chosen": -1.6665029525756836, |
| "logits/rejected": -1.5504449605941772, |
| "logps/chosen": -265.2987365722656, |
| "logps/rejected": -388.6627502441406, |
| "loss": 0.0446, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.944598436355591, |
| "rewards/margins": 8.778532028198242, |
| "rewards/rejected": -11.72313117980957, |
| "step": 1235 |
| }, |
| { |
| "epoch": 2.6442666666666668, |
| "grad_norm": 6.541167736053467, |
| "learning_rate": 1.8483071141435937e-07, |
| "logits/chosen": -1.6075607538223267, |
| "logits/rejected": -1.5328432321548462, |
| "logps/chosen": -263.16680908203125, |
| "logps/rejected": -365.10736083984375, |
| "loss": 0.0794, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.3820810317993164, |
| "rewards/margins": 8.6204252243042, |
| "rewards/rejected": -11.002506256103516, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.654933333333333, |
| "grad_norm": 3.5753588676452637, |
| "learning_rate": 1.741202543600176e-07, |
| "logits/chosen": -1.742559790611267, |
| "logits/rejected": -1.6055104732513428, |
| "logps/chosen": -245.0276336669922, |
| "logps/rejected": -397.55206298828125, |
| "loss": 0.0687, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.889216661453247, |
| "rewards/margins": 9.328636169433594, |
| "rewards/rejected": -12.217851638793945, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.6656, |
| "grad_norm": 3.1118340492248535, |
| "learning_rate": 1.6371825348184155e-07, |
| "logits/chosen": -1.6370012760162354, |
| "logits/rejected": -1.5552282333374023, |
| "logps/chosen": -254.6912078857422, |
| "logps/rejected": -362.8533020019531, |
| "loss": 0.0585, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.709108829498291, |
| "rewards/margins": 8.306825637817383, |
| "rewards/rejected": -11.015934944152832, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.676266666666667, |
| "grad_norm": 1.2242422103881836, |
| "learning_rate": 1.5362608828426246e-07, |
| "logits/chosen": -1.7631776332855225, |
| "logits/rejected": -1.6615889072418213, |
| "logps/chosen": -275.11981201171875, |
| "logps/rejected": -398.39996337890625, |
| "loss": 0.0543, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.662536382675171, |
| "rewards/margins": 8.675864219665527, |
| "rewards/rejected": -11.338400840759277, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.686933333333333, |
| "grad_norm": 3.2008516788482666, |
| "learning_rate": 1.438450971815686e-07, |
| "logits/chosen": -1.717780351638794, |
| "logits/rejected": -1.6827186346054077, |
| "logps/chosen": -232.81961059570312, |
| "logps/rejected": -409.2147521972656, |
| "loss": 0.0843, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -3.2906346321105957, |
| "rewards/margins": 9.57332706451416, |
| "rewards/rejected": -12.863962173461914, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.6976, |
| "grad_norm": 2.0423855781555176, |
| "learning_rate": 1.3437657732040783e-07, |
| "logits/chosen": -1.7342054843902588, |
| "logits/rejected": -1.6255111694335938, |
| "logps/chosen": -253.91708374023438, |
| "logps/rejected": -384.605712890625, |
| "loss": 0.0548, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.2692885398864746, |
| "rewards/margins": 9.20631217956543, |
| "rewards/rejected": -11.475601196289062, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.708266666666667, |
| "grad_norm": 6.765120506286621, |
| "learning_rate": 1.2522178440776022e-07, |
| "logits/chosen": -1.704227089881897, |
| "logits/rejected": -1.6806764602661133, |
| "logps/chosen": -253.9882354736328, |
| "logps/rejected": -389.9085998535156, |
| "loss": 0.1024, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.5248208045959473, |
| "rewards/margins": 8.699395179748535, |
| "rewards/rejected": -11.224214553833008, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.718933333333333, |
| "grad_norm": 1.7233941555023193, |
| "learning_rate": 1.1638193254440655e-07, |
| "logits/chosen": -1.7507209777832031, |
| "logits/rejected": -1.6442060470581055, |
| "logps/chosen": -235.013671875, |
| "logps/rejected": -387.20709228515625, |
| "loss": 0.0289, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.367755174636841, |
| "rewards/margins": 9.576312065124512, |
| "rewards/rejected": -11.944067001342773, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.7296, |
| "grad_norm": 7.248271465301514, |
| "learning_rate": 1.0785819406391562e-07, |
| "logits/chosen": -1.65673828125, |
| "logits/rejected": -1.6092407703399658, |
| "logps/chosen": -228.1624755859375, |
| "logps/rejected": -392.2158508300781, |
| "loss": 0.1043, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.5122058391571045, |
| "rewards/margins": 9.066630363464355, |
| "rewards/rejected": -11.578836441040039, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.740266666666667, |
| "grad_norm": 3.1239840984344482, |
| "learning_rate": 9.965169937717095e-08, |
| "logits/chosen": -1.7200371026992798, |
| "logits/rejected": -1.6642881631851196, |
| "logps/chosen": -239.06405639648438, |
| "logps/rejected": -381.20440673828125, |
| "loss": 0.0773, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4310803413391113, |
| "rewards/margins": 8.770525932312012, |
| "rewards/rejected": -11.201605796813965, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.7509333333333332, |
| "grad_norm": 2.2311618328094482, |
| "learning_rate": 9.176353682245676e-08, |
| "logits/chosen": -1.6120647192001343, |
| "logits/rejected": -1.5454223155975342, |
| "logps/chosen": -246.0549774169922, |
| "logps/rejected": -379.45660400390625, |
| "loss": 0.0733, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.705291271209717, |
| "rewards/margins": 8.565595626831055, |
| "rewards/rejected": -11.270886421203613, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.7616, |
| "grad_norm": 0.7328024506568909, |
| "learning_rate": 8.41947525211223e-08, |
| "logits/chosen": -1.5624516010284424, |
| "logits/rejected": -1.5214122533798218, |
| "logps/chosen": -251.17922973632812, |
| "logps/rejected": -388.30224609375, |
| "loss": 0.063, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.610011339187622, |
| "rewards/margins": 8.611980438232422, |
| "rewards/rejected": -11.221991539001465, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.772266666666667, |
| "grad_norm": 4.4146623611450195, |
| "learning_rate": 7.694635023884789e-08, |
| "logits/chosen": -1.6453464031219482, |
| "logits/rejected": -1.5596413612365723, |
| "logps/chosen": -261.37005615234375, |
| "logps/rejected": -395.37274169921875, |
| "loss": 0.1003, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -2.5829854011535645, |
| "rewards/margins": 8.859365463256836, |
| "rewards/rejected": -11.442350387573242, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7829333333333333, |
| "grad_norm": 3.653667688369751, |
| "learning_rate": 7.00192912525241e-08, |
| "logits/chosen": -1.6690375804901123, |
| "logits/rejected": -1.559043526649475, |
| "logps/chosen": -259.91229248046875, |
| "logps/rejected": -400.84014892578125, |
| "loss": 0.0712, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8678531646728516, |
| "rewards/margins": 9.116159439086914, |
| "rewards/rejected": -11.98401165008545, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.7936, |
| "grad_norm": 3.5960304737091064, |
| "learning_rate": 6.341449422277047e-08, |
| "logits/chosen": -1.6120831966400146, |
| "logits/rejected": -1.5400011539459229, |
| "logps/chosen": -263.94940185546875, |
| "logps/rejected": -413.22259521484375, |
| "loss": 0.0698, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.4007437229156494, |
| "rewards/margins": 9.813273429870605, |
| "rewards/rejected": -12.214017868041992, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.804266666666667, |
| "grad_norm": 6.226192951202393, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.7602459192276, |
| "logits/rejected": -1.6944478750228882, |
| "logps/chosen": -247.07528686523438, |
| "logps/rejected": -404.52813720703125, |
| "loss": 0.0735, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.5961649417877197, |
| "rewards/margins": 9.362703323364258, |
| "rewards/rejected": -11.958867073059082, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.8149333333333333, |
| "grad_norm": 1.191611409187317, |
| "learning_rate": 5.117514686876379e-08, |
| "logits/chosen": -1.6201938390731812, |
| "logits/rejected": -1.5016851425170898, |
| "logps/chosen": -262.31060791015625, |
| "logps/rejected": -399.95880126953125, |
| "loss": 0.1033, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.614260196685791, |
| "rewards/margins": 8.749483108520508, |
| "rewards/rejected": -11.363743782043457, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.8256, |
| "grad_norm": 0.6301141977310181, |
| "learning_rate": 4.5542219716255274e-08, |
| "logits/chosen": -1.6695480346679688, |
| "logits/rejected": -1.5834704637527466, |
| "logps/chosen": -268.4365234375, |
| "logps/rejected": -412.6512756347656, |
| "loss": 0.0368, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.7909514904022217, |
| "rewards/margins": 9.708473205566406, |
| "rewards/rejected": -12.499424934387207, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.836266666666667, |
| "grad_norm": 1.9661916494369507, |
| "learning_rate": 4.023480064854174e-08, |
| "logits/chosen": -1.7694295644760132, |
| "logits/rejected": -1.6655480861663818, |
| "logps/chosen": -307.1631774902344, |
| "logps/rejected": -419.0650939941406, |
| "loss": 0.0428, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.6897337436676025, |
| "rewards/margins": 9.9656982421875, |
| "rewards/rejected": -12.655431747436523, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.8469333333333333, |
| "grad_norm": 2.0274875164031982, |
| "learning_rate": 3.525359353098584e-08, |
| "logits/chosen": -1.7370857000350952, |
| "logits/rejected": -1.6126264333724976, |
| "logps/chosen": -258.9969177246094, |
| "logps/rejected": -390.44873046875, |
| "loss": 0.0781, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.651517391204834, |
| "rewards/margins": 8.74222469329834, |
| "rewards/rejected": -11.393742561340332, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.8576, |
| "grad_norm": 0.6840654015541077, |
| "learning_rate": 3.059925896700094e-08, |
| "logits/chosen": -1.6134666204452515, |
| "logits/rejected": -1.4918162822723389, |
| "logps/chosen": -270.07086181640625, |
| "logps/rejected": -397.61126708984375, |
| "loss": 0.0576, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.371833324432373, |
| "rewards/margins": 8.95458984375, |
| "rewards/rejected": -11.326423645019531, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.8682666666666665, |
| "grad_norm": 1.605663537979126, |
| "learning_rate": 2.6272414210443964e-08, |
| "logits/chosen": -1.6418520212173462, |
| "logits/rejected": -1.5910496711730957, |
| "logps/chosen": -270.58575439453125, |
| "logps/rejected": -397.8790283203125, |
| "loss": 0.0935, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.725590467453003, |
| "rewards/margins": 8.987241744995117, |
| "rewards/rejected": -11.712831497192383, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.8789333333333333, |
| "grad_norm": 3.390230894088745, |
| "learning_rate": 2.2273633083753632e-08, |
| "logits/chosen": -1.756787657737732, |
| "logits/rejected": -1.7060184478759766, |
| "logps/chosen": -237.4143524169922, |
| "logps/rejected": -390.0992431640625, |
| "loss": 0.0485, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.657024621963501, |
| "rewards/margins": 9.306405067443848, |
| "rewards/rejected": -11.96342945098877, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.8895999999999997, |
| "grad_norm": 1.4694249629974365, |
| "learning_rate": 1.8603445901851647e-08, |
| "logits/chosen": -1.6919931173324585, |
| "logits/rejected": -1.6533225774765015, |
| "logps/chosen": -259.6629638671875, |
| "logps/rejected": -387.4039001464844, |
| "loss": 0.0995, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -2.740100383758545, |
| "rewards/margins": 8.346617698669434, |
| "rewards/rejected": -11.08671760559082, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.9002666666666665, |
| "grad_norm": 2.6060729026794434, |
| "learning_rate": 1.5262339401813375e-08, |
| "logits/chosen": -1.7318048477172852, |
| "logits/rejected": -1.6237519979476929, |
| "logps/chosen": -221.9154815673828, |
| "logps/rejected": -387.20001220703125, |
| "loss": 0.0549, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4740569591522217, |
| "rewards/margins": 9.128717422485352, |
| "rewards/rejected": -11.602774620056152, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.9109333333333334, |
| "grad_norm": 0.737140953540802, |
| "learning_rate": 1.2250756678315601e-08, |
| "logits/chosen": -1.7253105640411377, |
| "logits/rejected": -1.6364980936050415, |
| "logps/chosen": -254.7071533203125, |
| "logps/rejected": -397.3651123046875, |
| "loss": 0.058, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.506072521209717, |
| "rewards/margins": 8.95616340637207, |
| "rewards/rejected": -11.462237358093262, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.9215999999999998, |
| "grad_norm": 2.5475025177001953, |
| "learning_rate": 9.569097124875193e-09, |
| "logits/chosen": -1.67425537109375, |
| "logits/rejected": -1.5568588972091675, |
| "logps/chosen": -247.0236358642578, |
| "logps/rejected": -387.4052429199219, |
| "loss": 0.0878, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.781790256500244, |
| "rewards/margins": 9.150073051452637, |
| "rewards/rejected": -11.931864738464355, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.9322666666666666, |
| "grad_norm": 5.0270867347717285, |
| "learning_rate": 7.217716380881479e-09, |
| "logits/chosen": -1.7101695537567139, |
| "logits/rejected": -1.637942910194397, |
| "logps/chosen": -269.4486999511719, |
| "logps/rejected": -391.3731384277344, |
| "loss": 0.0823, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.8127238750457764, |
| "rewards/margins": 8.60261058807373, |
| "rewards/rejected": -11.415335655212402, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.9429333333333334, |
| "grad_norm": 1.2320939302444458, |
| "learning_rate": 5.196926284430359e-09, |
| "logits/chosen": -1.6496827602386475, |
| "logits/rejected": -1.570379614830017, |
| "logps/chosen": -254.0030517578125, |
| "logps/rejected": -404.6939392089844, |
| "loss": 0.0626, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -2.8904614448547363, |
| "rewards/margins": 8.672920227050781, |
| "rewards/rejected": -11.563383102416992, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.9536, |
| "grad_norm": 2.0147035121917725, |
| "learning_rate": 3.5069948309712775e-09, |
| "logits/chosen": -1.6875463724136353, |
| "logits/rejected": -1.5886619091033936, |
| "logps/chosen": -248.8668212890625, |
| "logps/rejected": -395.17657470703125, |
| "loss": 0.0602, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4937005043029785, |
| "rewards/margins": 9.028209686279297, |
| "rewards/rejected": -11.521909713745117, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.9642666666666666, |
| "grad_norm": 1.6050134897232056, |
| "learning_rate": 2.1481461377634294e-09, |
| "logits/chosen": -1.700669527053833, |
| "logits/rejected": -1.642332673072815, |
| "logps/chosen": -245.1533660888672, |
| "logps/rejected": -398.7891845703125, |
| "loss": 0.039, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.403536796569824, |
| "rewards/margins": 9.711051940917969, |
| "rewards/rejected": -12.114588737487793, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.9749333333333334, |
| "grad_norm": 2.986335039138794, |
| "learning_rate": 1.120560414154259e-09, |
| "logits/chosen": -1.6862636804580688, |
| "logits/rejected": -1.5859509706497192, |
| "logps/chosen": -245.3629913330078, |
| "logps/rejected": -389.30328369140625, |
| "loss": 0.0603, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.827134132385254, |
| "rewards/margins": 9.108485221862793, |
| "rewards/rejected": -11.935619354248047, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.9856, |
| "grad_norm": 1.4037060737609863, |
| "learning_rate": 4.2437393768079983e-10, |
| "logits/chosen": -1.5650889873504639, |
| "logits/rejected": -1.5250481367111206, |
| "logps/chosen": -265.4774475097656, |
| "logps/rejected": -370.9742736816406, |
| "loss": 0.0463, |
| "rewards/accuracies": 0.9937499761581421, |
| "rewards/chosen": -2.532733917236328, |
| "rewards/margins": 8.557954788208008, |
| "rewards/rejected": -11.090688705444336, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.9962666666666666, |
| "grad_norm": 1.7923967838287354, |
| "learning_rate": 5.967903599635171e-11, |
| "logits/chosen": -1.688177466392517, |
| "logits/rejected": -1.509224534034729, |
| "logps/chosen": -274.61065673828125, |
| "logps/rejected": -385.02349853515625, |
| "loss": 0.0709, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.7587332725524902, |
| "rewards/margins": 8.69700813293457, |
| "rewards/rejected": -11.455740928649902, |
| "step": 1405 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1407, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
