| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.8888888888888888, |
| "eval_steps": 500, |
| "global_step": 1000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.0044444444444444444, |
| "grad_norm": 3.6353344917297363, |
| "learning_rate": 1e-05, |
| "logits/chosen": -0.4628738462924957, |
| "logits/rejected": -0.46038827300071716, |
| "logps/chosen": -305.24371337890625, |
| "logps/rejected": -217.2339324951172, |
| "loss": 0.69, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.006762671284377575, |
| "rewards/margins": 0.005093236453831196, |
| "rewards/rejected": 0.0016694354126229882, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.008888888888888889, |
| "grad_norm": 3.840994119644165, |
| "learning_rate": 9.999945685076187e-06, |
| "logits/chosen": -0.4660520553588867, |
| "logits/rejected": -0.4597313404083252, |
| "logps/chosen": -295.14178466796875, |
| "logps/rejected": -215.0008544921875, |
| "loss": 0.6585, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.07166890054941177, |
| "rewards/margins": 0.057903312146663666, |
| "rewards/rejected": 0.013765583746135235, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.013333333333333334, |
| "grad_norm": 3.3536572456359863, |
| "learning_rate": 9.99978274148479e-06, |
| "logits/chosen": -0.46407952904701233, |
| "logits/rejected": -0.46835923194885254, |
| "logps/chosen": -307.1814880371094, |
| "logps/rejected": -220.930908203125, |
| "loss": 0.6155, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.18775026500225067, |
| "rewards/margins": 0.13849034905433655, |
| "rewards/rejected": 0.049259938299655914, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.017777777777777778, |
| "grad_norm": 3.603736400604248, |
| "learning_rate": 9.999511172765917e-06, |
| "logits/chosen": -0.4126955568790436, |
| "logits/rejected": -0.4344128668308258, |
| "logps/chosen": -295.6951599121094, |
| "logps/rejected": -219.573974609375, |
| "loss": 0.572, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.3320372700691223, |
| "rewards/margins": 0.22264714539051056, |
| "rewards/rejected": 0.10939009487628937, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.022222222222222223, |
| "grad_norm": 3.7065272331237793, |
| "learning_rate": 9.999130984819662e-06, |
| "logits/chosen": -0.42767876386642456, |
| "logits/rejected": -0.4458894729614258, |
| "logps/chosen": -331.75592041015625, |
| "logps/rejected": -231.910400390625, |
| "loss": 0.5323, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": 0.5343211889266968, |
| "rewards/margins": 0.32672011852264404, |
| "rewards/rejected": 0.20760111510753632, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.02666666666666667, |
| "grad_norm": 3.656710624694824, |
| "learning_rate": 9.998642185905977e-06, |
| "logits/chosen": -0.44063276052474976, |
| "logits/rejected": -0.4492092728614807, |
| "logps/chosen": -311.40277099609375, |
| "logps/rejected": -233.438720703125, |
| "loss": 0.5154, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": 0.6043025851249695, |
| "rewards/margins": 0.3682531714439392, |
| "rewards/rejected": 0.23604938387870789, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.03111111111111111, |
| "grad_norm": 3.5971853733062744, |
| "learning_rate": 9.998044786644492e-06, |
| "logits/chosen": -0.39475446939468384, |
| "logits/rejected": -0.4055609703063965, |
| "logps/chosen": -298.6465759277344, |
| "logps/rejected": -219.363525390625, |
| "loss": 0.4452, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.8082733154296875, |
| "rewards/margins": 0.5571426153182983, |
| "rewards/rejected": 0.25113070011138916, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.035555555555555556, |
| "grad_norm": 3.67197322845459, |
| "learning_rate": 9.997338800014284e-06, |
| "logits/chosen": -0.41250643134117126, |
| "logits/rejected": -0.4259340167045593, |
| "logps/chosen": -293.3608703613281, |
| "logps/rejected": -224.9442901611328, |
| "loss": 0.4586, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.867678165435791, |
| "rewards/margins": 0.5272501707077026, |
| "rewards/rejected": 0.34042787551879883, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.04, |
| "grad_norm": 5.049458026885986, |
| "learning_rate": 9.9965242413536e-06, |
| "logits/chosen": -0.41178879141807556, |
| "logits/rejected": -0.4304323196411133, |
| "logps/chosen": -306.1034851074219, |
| "logps/rejected": -228.5247802734375, |
| "loss": 0.3777, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 1.235072374343872, |
| "rewards/margins": 0.7781749367713928, |
| "rewards/rejected": 0.4568973183631897, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.044444444444444446, |
| "grad_norm": 3.9216673374176025, |
| "learning_rate": 9.995601128359516e-06, |
| "logits/chosen": -0.40246009826660156, |
| "logits/rejected": -0.3950818181037903, |
| "logps/chosen": -303.0498352050781, |
| "logps/rejected": -226.4988250732422, |
| "loss": 0.3999, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": 1.3293551206588745, |
| "rewards/margins": 0.8089650869369507, |
| "rewards/rejected": 0.5203902721405029, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.04888888888888889, |
| "grad_norm": 4.434176921844482, |
| "learning_rate": 9.994569481087552e-06, |
| "logits/chosen": -0.39378249645233154, |
| "logits/rejected": -0.40684300661087036, |
| "logps/chosen": -329.46173095703125, |
| "logps/rejected": -223.7794952392578, |
| "loss": 0.3168, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 1.6077896356582642, |
| "rewards/margins": 1.0661401748657227, |
| "rewards/rejected": 0.5416494607925415, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 4.062209129333496, |
| "learning_rate": 9.993429321951251e-06, |
| "logits/chosen": -0.34955719113349915, |
| "logits/rejected": -0.3819810748100281, |
| "logps/chosen": -291.4505615234375, |
| "logps/rejected": -212.7031707763672, |
| "loss": 0.2348, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 1.9593639373779297, |
| "rewards/margins": 1.3969981670379639, |
| "rewards/rejected": 0.5623658299446106, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.057777777777777775, |
| "grad_norm": 4.022445201873779, |
| "learning_rate": 9.992180675721671e-06, |
| "logits/chosen": -0.3607024550437927, |
| "logits/rejected": -0.3758237659931183, |
| "logps/chosen": -325.44622802734375, |
| "logps/rejected": -226.538818359375, |
| "loss": 0.1438, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 2.2623605728149414, |
| "rewards/margins": 1.6984504461288452, |
| "rewards/rejected": 0.5639100670814514, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.06222222222222222, |
| "grad_norm": 4.516530513763428, |
| "learning_rate": 9.990823569526868e-06, |
| "logits/chosen": -0.3758849501609802, |
| "logits/rejected": -0.401409387588501, |
| "logps/chosen": -293.59283447265625, |
| "logps/rejected": -212.1451873779297, |
| "loss": 0.2153, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 2.2533986568450928, |
| "rewards/margins": 1.599491000175476, |
| "rewards/rejected": 0.6539075374603271, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.06666666666666667, |
| "grad_norm": 4.2673563957214355, |
| "learning_rate": 9.989358032851283e-06, |
| "logits/chosen": -0.38496989011764526, |
| "logits/rejected": -0.4185038208961487, |
| "logps/chosen": -331.06707763671875, |
| "logps/rejected": -238.4701690673828, |
| "loss": 0.2667, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.29125714302063, |
| "rewards/margins": 1.5516706705093384, |
| "rewards/rejected": 0.739586353302002, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.07111111111111111, |
| "grad_norm": 4.720420837402344, |
| "learning_rate": 9.987784097535126e-06, |
| "logits/chosen": -0.36235010623931885, |
| "logits/rejected": -0.3792596757411957, |
| "logps/chosen": -303.70196533203125, |
| "logps/rejected": -226.2982940673828, |
| "loss": 0.1933, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 2.5473880767822266, |
| "rewards/margins": 1.8545589447021484, |
| "rewards/rejected": 0.6928290128707886, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.07555555555555556, |
| "grad_norm": 7.281383037567139, |
| "learning_rate": 9.986101797773667e-06, |
| "logits/chosen": -0.380900114774704, |
| "logits/rejected": -0.3917911946773529, |
| "logps/chosen": -295.6128234863281, |
| "logps/rejected": -229.4749298095703, |
| "loss": 0.213, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.3351025581359863, |
| "rewards/margins": 1.7777379751205444, |
| "rewards/rejected": 0.5573645830154419, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.08, |
| "grad_norm": 4.908195972442627, |
| "learning_rate": 9.984311170116497e-06, |
| "logits/chosen": -0.37983238697052, |
| "logits/rejected": -0.3918471932411194, |
| "logps/chosen": -291.7480163574219, |
| "logps/rejected": -220.09652709960938, |
| "loss": 0.1737, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 2.4933111667633057, |
| "rewards/margins": 1.936655044555664, |
| "rewards/rejected": 0.5566561222076416, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.08444444444444445, |
| "grad_norm": 5.108543395996094, |
| "learning_rate": 9.98241225346674e-06, |
| "logits/chosen": -0.34952667355537415, |
| "logits/rejected": -0.3850114643573761, |
| "logps/chosen": -302.5187683105469, |
| "logps/rejected": -222.33834838867188, |
| "loss": 0.1069, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.5169692039489746, |
| "rewards/margins": 2.1484999656677246, |
| "rewards/rejected": 0.36846891045570374, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.08888888888888889, |
| "grad_norm": 4.840832233428955, |
| "learning_rate": 9.9804050890802e-06, |
| "logits/chosen": -0.33506280183792114, |
| "logits/rejected": -0.3472011089324951, |
| "logps/chosen": -269.5045471191406, |
| "logps/rejected": -205.8480987548828, |
| "loss": 0.1074, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 2.4071407318115234, |
| "rewards/margins": 2.1650052070617676, |
| "rewards/rejected": 0.24213531613349915, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.09333333333333334, |
| "grad_norm": 5.977898120880127, |
| "learning_rate": 9.978289720564471e-06, |
| "logits/chosen": -0.33771952986717224, |
| "logits/rejected": -0.3690803050994873, |
| "logps/chosen": -308.5394287109375, |
| "logps/rejected": -226.24813842773438, |
| "loss": 0.2107, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": 2.450155735015869, |
| "rewards/margins": 2.019880771636963, |
| "rewards/rejected": 0.4302748143672943, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.09777777777777778, |
| "grad_norm": 6.031980514526367, |
| "learning_rate": 9.976066193877982e-06, |
| "logits/chosen": -0.347932904958725, |
| "logits/rejected": -0.3845617175102234, |
| "logps/chosen": -293.0547790527344, |
| "logps/rejected": -220.5181427001953, |
| "loss": 0.1324, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 2.321216106414795, |
| "rewards/margins": 2.2665910720825195, |
| "rewards/rejected": 0.054625045508146286, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.10222222222222223, |
| "grad_norm": 4.3759660720825195, |
| "learning_rate": 9.97373455732901e-06, |
| "logits/chosen": -0.34049180150032043, |
| "logits/rejected": -0.35589173436164856, |
| "logps/chosen": -294.77117919921875, |
| "logps/rejected": -228.1370086669922, |
| "loss": 0.0647, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 2.3987247943878174, |
| "rewards/margins": 2.2897610664367676, |
| "rewards/rejected": 0.1089634895324707, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 4.7751336097717285, |
| "learning_rate": 9.971294861574617e-06, |
| "logits/chosen": -0.3569382429122925, |
| "logits/rejected": -0.35876479744911194, |
| "logps/chosen": -285.916748046875, |
| "logps/rejected": -219.81753540039062, |
| "loss": 0.0106, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 2.5361573696136475, |
| "rewards/margins": 2.6316070556640625, |
| "rewards/rejected": -0.09544976055622101, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.1111111111111111, |
| "grad_norm": 6.836686134338379, |
| "learning_rate": 9.968747159619556e-06, |
| "logits/chosen": -0.3644478917121887, |
| "logits/rejected": -0.3773222863674164, |
| "logps/chosen": -301.05084228515625, |
| "logps/rejected": -231.2653045654297, |
| "loss": 0.1055, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": 2.7265372276306152, |
| "rewards/margins": 2.74798846244812, |
| "rewards/rejected": -0.021450763568282127, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.11555555555555555, |
| "grad_norm": 6.012020587921143, |
| "learning_rate": 9.966091506815128e-06, |
| "logits/chosen": -0.34487825632095337, |
| "logits/rejected": -0.3683899939060211, |
| "logps/chosen": -296.5931701660156, |
| "logps/rejected": -223.12753295898438, |
| "loss": 0.0823, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": 2.7003228664398193, |
| "rewards/margins": 2.8483641147613525, |
| "rewards/rejected": -0.14804117381572723, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.12, |
| "grad_norm": 5.459484100341797, |
| "learning_rate": 9.963327960857962e-06, |
| "logits/chosen": -0.3142702579498291, |
| "logits/rejected": -0.36442944407463074, |
| "logps/chosen": -310.71368408203125, |
| "logps/rejected": -211.5228729248047, |
| "loss": -0.0914, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 3.0668811798095703, |
| "rewards/margins": 3.4564356803894043, |
| "rewards/rejected": -0.3895547688007355, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.12444444444444444, |
| "grad_norm": 4.6439080238342285, |
| "learning_rate": 9.960456581788771e-06, |
| "logits/chosen": -0.3213174343109131, |
| "logits/rejected": -0.35702863335609436, |
| "logps/chosen": -295.9752502441406, |
| "logps/rejected": -218.5254364013672, |
| "loss": -0.0509, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.7388923168182373, |
| "rewards/margins": 3.2099480628967285, |
| "rewards/rejected": -0.47105544805526733, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.1288888888888889, |
| "grad_norm": 7.5686140060424805, |
| "learning_rate": 9.957477431991053e-06, |
| "logits/chosen": -0.3489062190055847, |
| "logits/rejected": -0.38331982493400574, |
| "logps/chosen": -301.20574951171875, |
| "logps/rejected": -221.79617309570312, |
| "loss": 0.0026, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 2.3142552375793457, |
| "rewards/margins": 3.0163302421569824, |
| "rewards/rejected": -0.7020750045776367, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.13333333333333333, |
| "grad_norm": 7.296176910400391, |
| "learning_rate": 9.954390576189726e-06, |
| "logits/chosen": -0.32641178369522095, |
| "logits/rejected": -0.3621976673603058, |
| "logps/chosen": -312.5970153808594, |
| "logps/rejected": -246.51974487304688, |
| "loss": 0.1054, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 2.6623806953430176, |
| "rewards/margins": 2.9366531372070312, |
| "rewards/rejected": -0.27427244186401367, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.13777777777777778, |
| "grad_norm": 6.089158535003662, |
| "learning_rate": 9.95119608144972e-06, |
| "logits/chosen": -0.34094589948654175, |
| "logits/rejected": -0.36057132482528687, |
| "logps/chosen": -298.9684143066406, |
| "logps/rejected": -232.59097290039062, |
| "loss": 0.0235, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 2.0118796825408936, |
| "rewards/margins": 2.952807664871216, |
| "rewards/rejected": -0.9409275054931641, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.14222222222222222, |
| "grad_norm": 5.5049662590026855, |
| "learning_rate": 9.947894017174535e-06, |
| "logits/chosen": -0.30161410570144653, |
| "logits/rejected": -0.3480113446712494, |
| "logps/chosen": -306.0929870605469, |
| "logps/rejected": -226.927734375, |
| "loss": -0.0148, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.6642236709594727, |
| "rewards/margins": 3.523815631866455, |
| "rewards/rejected": -0.8595919609069824, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.14666666666666667, |
| "grad_norm": 5.161729335784912, |
| "learning_rate": 9.944484455104716e-06, |
| "logits/chosen": -0.3171108067035675, |
| "logits/rejected": -0.35261866450309753, |
| "logps/chosen": -290.88250732421875, |
| "logps/rejected": -234.3745880126953, |
| "loss": 0.085, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 1.7442996501922607, |
| "rewards/margins": 2.7870707511901855, |
| "rewards/rejected": -1.0427708625793457, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.1511111111111111, |
| "grad_norm": 5.5866618156433105, |
| "learning_rate": 9.940967469316307e-06, |
| "logits/chosen": -0.3179735541343689, |
| "logits/rejected": -0.3568040728569031, |
| "logps/chosen": -328.8294372558594, |
| "logps/rejected": -225.0006561279297, |
| "loss": -0.2403, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 3.2372944355010986, |
| "rewards/margins": 4.343171119689941, |
| "rewards/rejected": -1.105877161026001, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.15555555555555556, |
| "grad_norm": 6.542994499206543, |
| "learning_rate": 9.937343136219234e-06, |
| "logits/chosen": -0.2941819429397583, |
| "logits/rejected": -0.3379240930080414, |
| "logps/chosen": -301.7574462890625, |
| "logps/rejected": -217.0867919921875, |
| "loss": -0.1938, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 2.6589813232421875, |
| "rewards/margins": 4.12928581237793, |
| "rewards/rejected": -1.4703044891357422, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 8.16442584991455, |
| "learning_rate": 9.933611534555645e-06, |
| "logits/chosen": -0.3271011710166931, |
| "logits/rejected": -0.3445083200931549, |
| "logps/chosen": -319.7223205566406, |
| "logps/rejected": -247.19967651367188, |
| "loss": -0.0822, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 2.863480806350708, |
| "rewards/margins": 3.848937511444092, |
| "rewards/rejected": -0.9854568243026733, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.16444444444444445, |
| "grad_norm": 5.072612285614014, |
| "learning_rate": 9.929772745398207e-06, |
| "logits/chosen": -0.3311443328857422, |
| "logits/rejected": -0.34855595231056213, |
| "logps/chosen": -304.24371337890625, |
| "logps/rejected": -228.12423706054688, |
| "loss": -0.0953, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 2.729543685913086, |
| "rewards/margins": 3.911928653717041, |
| "rewards/rejected": -1.1823843717575073, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.1688888888888889, |
| "grad_norm": 5.6365966796875, |
| "learning_rate": 9.925826852148332e-06, |
| "logits/chosen": -0.37482309341430664, |
| "logits/rejected": -0.37685567140579224, |
| "logps/chosen": -323.057373046875, |
| "logps/rejected": -238.3128204345703, |
| "loss": -0.1351, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 2.4461398124694824, |
| "rewards/margins": 4.063778400421143, |
| "rewards/rejected": -1.6176389455795288, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.17333333333333334, |
| "grad_norm": 7.323070049285889, |
| "learning_rate": 9.921773940534382e-06, |
| "logits/chosen": -0.30995437502861023, |
| "logits/rejected": -0.3514579236507416, |
| "logps/chosen": -283.3837890625, |
| "logps/rejected": -228.30905151367188, |
| "loss": 0.0783, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 1.8092823028564453, |
| "rewards/margins": 3.168578624725342, |
| "rewards/rejected": -1.3592965602874756, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.17777777777777778, |
| "grad_norm": 5.961920738220215, |
| "learning_rate": 9.917614098609786e-06, |
| "logits/chosen": -0.3327783942222595, |
| "logits/rejected": -0.36006277799606323, |
| "logps/chosen": -307.5805358886719, |
| "logps/rejected": -228.3577880859375, |
| "loss": -0.0971, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 2.8803937435150146, |
| "rewards/margins": 3.999189853668213, |
| "rewards/rejected": -1.1187958717346191, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.18222222222222223, |
| "grad_norm": 8.006556510925293, |
| "learning_rate": 9.913347416751148e-06, |
| "logits/chosen": -0.290499210357666, |
| "logits/rejected": -0.32564371824264526, |
| "logps/chosen": -310.0920715332031, |
| "logps/rejected": -222.1564178466797, |
| "loss": -0.1483, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 3.187587022781372, |
| "rewards/margins": 4.311644554138184, |
| "rewards/rejected": -1.124057650566101, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.18666666666666668, |
| "grad_norm": 5.825204849243164, |
| "learning_rate": 9.908973987656263e-06, |
| "logits/chosen": -0.3070078492164612, |
| "logits/rejected": -0.3182796239852905, |
| "logps/chosen": -289.46490478515625, |
| "logps/rejected": -223.34725952148438, |
| "loss": -0.2937, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 3.196002244949341, |
| "rewards/margins": 4.649127006530762, |
| "rewards/rejected": -1.453124761581421, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.19111111111111112, |
| "grad_norm": 8.087427139282227, |
| "learning_rate": 9.904493906342124e-06, |
| "logits/chosen": -0.284060001373291, |
| "logits/rejected": -0.3289189636707306, |
| "logps/chosen": -292.52984619140625, |
| "logps/rejected": -227.1925811767578, |
| "loss": -0.0803, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 2.883082628250122, |
| "rewards/margins": 4.214940071105957, |
| "rewards/rejected": -1.3318575620651245, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.19555555555555557, |
| "grad_norm": 6.931927680969238, |
| "learning_rate": 9.899907270142835e-06, |
| "logits/chosen": -0.29949700832366943, |
| "logits/rejected": -0.3155062794685364, |
| "logps/chosen": -305.6365661621094, |
| "logps/rejected": -233.4442901611328, |
| "loss": -0.0835, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 2.9822609424591064, |
| "rewards/margins": 4.396633148193359, |
| "rewards/rejected": -1.4143723249435425, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.2, |
| "grad_norm": 9.130791664123535, |
| "learning_rate": 9.895214178707516e-06, |
| "logits/chosen": -0.31096282601356506, |
| "logits/rejected": -0.3551832437515259, |
| "logps/chosen": -313.84320068359375, |
| "logps/rejected": -236.7030029296875, |
| "loss": -0.0991, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 2.626882553100586, |
| "rewards/margins": 4.163486003875732, |
| "rewards/rejected": -1.536603331565857, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.20444444444444446, |
| "grad_norm": 5.590844631195068, |
| "learning_rate": 9.890414733998131e-06, |
| "logits/chosen": -0.2635526657104492, |
| "logits/rejected": -0.29329806566238403, |
| "logps/chosen": -297.04638671875, |
| "logps/rejected": -236.12271118164062, |
| "loss": 0.0262, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 2.713446855545044, |
| "rewards/margins": 3.905104875564575, |
| "rewards/rejected": -1.1916577816009521, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.2088888888888889, |
| "grad_norm": 4.7747979164123535, |
| "learning_rate": 9.885509040287267e-06, |
| "logits/chosen": -0.30965957045555115, |
| "logits/rejected": -0.31621426343917847, |
| "logps/chosen": -288.59014892578125, |
| "logps/rejected": -220.62765502929688, |
| "loss": -0.342, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 3.3641788959503174, |
| "rewards/margins": 5.042544841766357, |
| "rewards/rejected": -1.6783654689788818, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 6.6951680183410645, |
| "learning_rate": 9.880497204155879e-06, |
| "logits/chosen": -0.27586597204208374, |
| "logits/rejected": -0.3355752229690552, |
| "logps/chosen": -316.9598388671875, |
| "logps/rejected": -244.87149047851562, |
| "loss": -0.0482, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 2.7237601280212402, |
| "rewards/margins": 3.941441774368286, |
| "rewards/rejected": -1.217681646347046, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.21777777777777776, |
| "grad_norm": 5.907822608947754, |
| "learning_rate": 9.875379334490962e-06, |
| "logits/chosen": -0.3292551338672638, |
| "logits/rejected": -0.31635525822639465, |
| "logps/chosen": -291.8619689941406, |
| "logps/rejected": -231.01016235351562, |
| "loss": -0.1414, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 2.5336103439331055, |
| "rewards/margins": 4.392641067504883, |
| "rewards/rejected": -1.8590309619903564, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.2222222222222222, |
| "grad_norm": 7.046641826629639, |
| "learning_rate": 9.870155542483199e-06, |
| "logits/chosen": -0.3067111372947693, |
| "logits/rejected": -0.35157865285873413, |
| "logps/chosen": -319.73187255859375, |
| "logps/rejected": -236.04483032226562, |
| "loss": -0.3243, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 3.8645987510681152, |
| "rewards/margins": 5.3452653884887695, |
| "rewards/rejected": -1.4806665182113647, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.22666666666666666, |
| "grad_norm": 9.379409790039062, |
| "learning_rate": 9.864825941624538e-06, |
| "logits/chosen": -0.267128050327301, |
| "logits/rejected": -0.2918349802494049, |
| "logps/chosen": -313.8056335449219, |
| "logps/rejected": -247.0101776123047, |
| "loss": -0.1449, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 3.3342041969299316, |
| "rewards/margins": 4.6097564697265625, |
| "rewards/rejected": -1.27555251121521, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.2311111111111111, |
| "grad_norm": 7.974717617034912, |
| "learning_rate": 9.85939064770572e-06, |
| "logits/chosen": -0.3181043267250061, |
| "logits/rejected": -0.3094359338283539, |
| "logps/chosen": -317.86505126953125, |
| "logps/rejected": -245.4908905029297, |
| "loss": -0.3156, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 3.143451452255249, |
| "rewards/margins": 5.211213111877441, |
| "rewards/rejected": -2.0677614212036133, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.23555555555555555, |
| "grad_norm": 6.511713027954102, |
| "learning_rate": 9.853849778813777e-06, |
| "logits/chosen": -0.29388368129730225, |
| "logits/rejected": -0.3029894530773163, |
| "logps/chosen": -297.4751892089844, |
| "logps/rejected": -231.9884796142578, |
| "loss": -0.167, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 3.295167922973633, |
| "rewards/margins": 5.074382305145264, |
| "rewards/rejected": -1.77921462059021, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.24, |
| "grad_norm": 6.8637776374816895, |
| "learning_rate": 9.848203455329459e-06, |
| "logits/chosen": -0.31308668851852417, |
| "logits/rejected": -0.3360288441181183, |
| "logps/chosen": -296.82318115234375, |
| "logps/rejected": -236.3435821533203, |
| "loss": -0.2143, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 2.4768126010894775, |
| "rewards/margins": 4.899205684661865, |
| "rewards/rejected": -2.422393321990967, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.24444444444444444, |
| "grad_norm": 10.344189643859863, |
| "learning_rate": 9.842451799924616e-06, |
| "logits/chosen": -0.2888021171092987, |
| "logits/rejected": -0.3189722001552582, |
| "logps/chosen": -321.9690246582031, |
| "logps/rejected": -239.86520385742188, |
| "loss": -0.327, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 2.709468364715576, |
| "rewards/margins": 5.345309257507324, |
| "rewards/rejected": -2.635840892791748, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.24888888888888888, |
| "grad_norm": 11.444976806640625, |
| "learning_rate": 9.836594937559541e-06, |
| "logits/chosen": -0.28263232111930847, |
| "logits/rejected": -0.2964705526828766, |
| "logps/chosen": -296.26995849609375, |
| "logps/rejected": -232.54934692382812, |
| "loss": 0.0017, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 2.52939772605896, |
| "rewards/margins": 4.631046295166016, |
| "rewards/rejected": -2.1016488075256348, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.25333333333333335, |
| "grad_norm": 5.921905040740967, |
| "learning_rate": 9.830632995480243e-06, |
| "logits/chosen": -0.26743844151496887, |
| "logits/rejected": -0.27696385979652405, |
| "logps/chosen": -298.95477294921875, |
| "logps/rejected": -236.3730926513672, |
| "loss": -0.2483, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 2.9082038402557373, |
| "rewards/margins": 5.20479154586792, |
| "rewards/rejected": -2.2965879440307617, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.2577777777777778, |
| "grad_norm": 6.96235990524292, |
| "learning_rate": 9.824566103215697e-06, |
| "logits/chosen": -0.2472468614578247, |
| "logits/rejected": -0.2931605279445648, |
| "logps/chosen": -298.02581787109375, |
| "logps/rejected": -231.60879516601562, |
| "loss": -0.2804, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 2.8193132877349854, |
| "rewards/margins": 5.249671459197998, |
| "rewards/rejected": -2.430358648300171, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.26222222222222225, |
| "grad_norm": 8.460125923156738, |
| "learning_rate": 9.818394392575018e-06, |
| "logits/chosen": -0.30542343854904175, |
| "logits/rejected": -0.32763975858688354, |
| "logps/chosen": -285.7476501464844, |
| "logps/rejected": -243.5345458984375, |
| "loss": -0.1747, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 2.6046016216278076, |
| "rewards/margins": 4.996693134307861, |
| "rewards/rejected": -2.3920915126800537, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 7.488274097442627, |
| "learning_rate": 9.812117997644606e-06, |
| "logits/chosen": -0.2731490731239319, |
| "logits/rejected": -0.30121108889579773, |
| "logps/chosen": -284.916259765625, |
| "logps/rejected": -233.185546875, |
| "loss": -0.1565, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.169166088104248, |
| "rewards/margins": 5.066960334777832, |
| "rewards/rejected": -2.897794246673584, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.27111111111111114, |
| "grad_norm": 8.804214477539062, |
| "learning_rate": 9.805737054785223e-06, |
| "logits/chosen": -0.2953334450721741, |
| "logits/rejected": -0.327360063791275, |
| "logps/chosen": -300.7308349609375, |
| "logps/rejected": -236.43685913085938, |
| "loss": -0.0923, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 2.0204293727874756, |
| "rewards/margins": 4.94085168838501, |
| "rewards/rejected": -2.920422315597534, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.27555555555555555, |
| "grad_norm": 8.035072326660156, |
| "learning_rate": 9.79925170262904e-06, |
| "logits/chosen": -0.26204347610473633, |
| "logits/rejected": -0.31125301122665405, |
| "logps/chosen": -280.49102783203125, |
| "logps/rejected": -226.66110229492188, |
| "loss": -0.2221, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 1.1722501516342163, |
| "rewards/margins": 4.767660140991211, |
| "rewards/rejected": -3.595409870147705, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.28, |
| "grad_norm": 6.416834831237793, |
| "learning_rate": 9.792662082076618e-06, |
| "logits/chosen": -0.2821267247200012, |
| "logits/rejected": -0.29524296522140503, |
| "logps/chosen": -313.2020568847656, |
| "logps/rejected": -236.20578002929688, |
| "loss": -0.3875, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 2.725848913192749, |
| "rewards/margins": 5.90293025970459, |
| "rewards/rejected": -3.17708158493042, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.28444444444444444, |
| "grad_norm": 9.474376678466797, |
| "learning_rate": 9.785968336293859e-06, |
| "logits/chosen": -0.2762632966041565, |
| "logits/rejected": -0.34091368317604065, |
| "logps/chosen": -315.105224609375, |
| "logps/rejected": -250.1154327392578, |
| "loss": -0.2361, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 1.9460265636444092, |
| "rewards/margins": 5.044549465179443, |
| "rewards/rejected": -3.0985231399536133, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.28888888888888886, |
| "grad_norm": 7.876622200012207, |
| "learning_rate": 9.779170610708872e-06, |
| "logits/chosen": -0.26600781083106995, |
| "logits/rejected": -0.2999460697174072, |
| "logps/chosen": -315.525146484375, |
| "logps/rejected": -239.6782989501953, |
| "loss": -0.3024, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 2.9207816123962402, |
| "rewards/margins": 6.140283107757568, |
| "rewards/rejected": -3.219501495361328, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.29333333333333333, |
| "grad_norm": 9.389948844909668, |
| "learning_rate": 9.772269053008841e-06, |
| "logits/chosen": -0.2716449201107025, |
| "logits/rejected": -0.31395813822746277, |
| "logps/chosen": -293.0248107910156, |
| "logps/rejected": -221.9087371826172, |
| "loss": -0.1898, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 2.0722451210021973, |
| "rewards/margins": 5.335482597351074, |
| "rewards/rejected": -3.263237714767456, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.29777777777777775, |
| "grad_norm": 8.571460723876953, |
| "learning_rate": 9.765263813136796e-06, |
| "logits/chosen": -0.27379176020622253, |
| "logits/rejected": -0.31927746534347534, |
| "logps/chosen": -306.12799072265625, |
| "logps/rejected": -229.9273681640625, |
| "loss": -0.1855, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 1.6866050958633423, |
| "rewards/margins": 4.964447975158691, |
| "rewards/rejected": -3.2778429985046387, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.3022222222222222, |
| "grad_norm": 7.001428127288818, |
| "learning_rate": 9.758155043288367e-06, |
| "logits/chosen": -0.28565549850463867, |
| "logits/rejected": -0.3229166865348816, |
| "logps/chosen": -297.42449951171875, |
| "logps/rejected": -250.22286987304688, |
| "loss": -0.2147, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 2.1362271308898926, |
| "rewards/margins": 5.681948661804199, |
| "rewards/rejected": -3.5457210540771484, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.30666666666666664, |
| "grad_norm": 8.346474647521973, |
| "learning_rate": 9.750942897908468e-06, |
| "logits/chosen": -0.24829097092151642, |
| "logits/rejected": -0.2842785120010376, |
| "logps/chosen": -293.74859619140625, |
| "logps/rejected": -236.0409698486328, |
| "loss": -0.4852, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 2.535234212875366, |
| "rewards/margins": 6.377307891845703, |
| "rewards/rejected": -3.842073917388916, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.3111111111111111, |
| "grad_norm": 6.549100875854492, |
| "learning_rate": 9.743627533687953e-06, |
| "logits/chosen": -0.2822897136211395, |
| "logits/rejected": -0.3249056041240692, |
| "logps/chosen": -297.6363830566406, |
| "logps/rejected": -229.268798828125, |
| "loss": -0.4093, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 2.632903814315796, |
| "rewards/margins": 6.515559196472168, |
| "rewards/rejected": -3.882655620574951, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.31555555555555553, |
| "grad_norm": 6.955848217010498, |
| "learning_rate": 9.736209109560201e-06, |
| "logits/chosen": -0.2583480179309845, |
| "logits/rejected": -0.31130915880203247, |
| "logps/chosen": -286.8586730957031, |
| "logps/rejected": -228.97238159179688, |
| "loss": -0.3784, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 2.1645522117614746, |
| "rewards/margins": 5.964513301849365, |
| "rewards/rejected": -3.7999610900878906, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 9.709640502929688, |
| "learning_rate": 9.728687786697667e-06, |
| "logits/chosen": -0.2713499069213867, |
| "logits/rejected": -0.3231387734413147, |
| "logps/chosen": -310.96929931640625, |
| "logps/rejected": -238.51025390625, |
| "loss": -0.3302, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 2.8049497604370117, |
| "rewards/margins": 6.414994239807129, |
| "rewards/rejected": -3.6100432872772217, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.3244444444444444, |
| "grad_norm": 6.710853099822998, |
| "learning_rate": 9.721063728508384e-06, |
| "logits/chosen": -0.28875869512557983, |
| "logits/rejected": -0.32300078868865967, |
| "logps/chosen": -297.21221923828125, |
| "logps/rejected": -246.2225799560547, |
| "loss": -0.3494, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 2.3010525703430176, |
| "rewards/margins": 6.263821601867676, |
| "rewards/rejected": -3.9627685546875, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.3288888888888889, |
| "grad_norm": 9.562369346618652, |
| "learning_rate": 9.713337100632407e-06, |
| "logits/chosen": -0.23941664397716522, |
| "logits/rejected": -0.2882528305053711, |
| "logps/chosen": -297.01116943359375, |
| "logps/rejected": -246.925048828125, |
| "loss": -0.2107, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 1.6213099956512451, |
| "rewards/margins": 5.55633020401001, |
| "rewards/rejected": -3.935020923614502, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.3333333333333333, |
| "grad_norm": 8.356274604797363, |
| "learning_rate": 9.705508070938219e-06, |
| "logits/chosen": -0.26807016134262085, |
| "logits/rejected": -0.29893961548805237, |
| "logps/chosen": -310.1183776855469, |
| "logps/rejected": -234.883544921875, |
| "loss": -0.6507, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 3.1703178882598877, |
| "rewards/margins": 7.669167995452881, |
| "rewards/rejected": -4.498850345611572, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.3377777777777778, |
| "grad_norm": 7.058998107910156, |
| "learning_rate": 9.697576809519079e-06, |
| "logits/chosen": -0.2949567139148712, |
| "logits/rejected": -0.3223188519477844, |
| "logps/chosen": -311.98773193359375, |
| "logps/rejected": -245.71194458007812, |
| "loss": -0.3221, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 1.6648155450820923, |
| "rewards/margins": 6.265153884887695, |
| "rewards/rejected": -4.600337982177734, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.3422222222222222, |
| "grad_norm": 8.904199600219727, |
| "learning_rate": 9.689543488689332e-06, |
| "logits/chosen": -0.25813308358192444, |
| "logits/rejected": -0.29112708568573, |
| "logps/chosen": -301.86834716796875, |
| "logps/rejected": -247.51974487304688, |
| "loss": -0.3268, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.9261104464530945, |
| "rewards/margins": 5.75935173034668, |
| "rewards/rejected": -4.8332414627075195, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.3466666666666667, |
| "grad_norm": 6.82271671295166, |
| "learning_rate": 9.68140828298066e-06, |
| "logits/chosen": -0.27823004126548767, |
| "logits/rejected": -0.30908042192459106, |
| "logps/chosen": -302.1865539550781, |
| "logps/rejected": -234.9086456298828, |
| "loss": -0.5093, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 1.626147985458374, |
| "rewards/margins": 6.896539211273193, |
| "rewards/rejected": -5.270391941070557, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.3511111111111111, |
| "grad_norm": 10.339046478271484, |
| "learning_rate": 9.673171369138297e-06, |
| "logits/chosen": -0.2561442255973816, |
| "logits/rejected": -0.2945733666419983, |
| "logps/chosen": -304.0384216308594, |
| "logps/rejected": -242.31137084960938, |
| "loss": -0.3237, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 1.6995117664337158, |
| "rewards/margins": 6.307187557220459, |
| "rewards/rejected": -4.607676029205322, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.35555555555555557, |
| "grad_norm": 7.32875919342041, |
| "learning_rate": 9.66483292611718e-06, |
| "logits/chosen": -0.2525383234024048, |
| "logits/rejected": -0.28177526593208313, |
| "logps/chosen": -292.54046630859375, |
| "logps/rejected": -235.5553741455078, |
| "loss": -0.4031, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 1.865870714187622, |
| "rewards/margins": 6.680575370788574, |
| "rewards/rejected": -4.814703941345215, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 12.206645011901855, |
| "learning_rate": 9.656393135078067e-06, |
| "logits/chosen": -0.2548236846923828, |
| "logits/rejected": -0.30014172196388245, |
| "logps/chosen": -300.7771911621094, |
| "logps/rejected": -239.86367797851562, |
| "loss": -0.538, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 1.971895456314087, |
| "rewards/margins": 7.470471382141113, |
| "rewards/rejected": -5.498574733734131, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.36444444444444446, |
| "grad_norm": 8.910218238830566, |
| "learning_rate": 9.647852179383606e-06, |
| "logits/chosen": -0.27060994505882263, |
| "logits/rejected": -0.3155694603919983, |
| "logps/chosen": -300.47210693359375, |
| "logps/rejected": -234.78250122070312, |
| "loss": -0.5798, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 1.5166006088256836, |
| "rewards/margins": 7.203047275543213, |
| "rewards/rejected": -5.686446189880371, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.3688888888888889, |
| "grad_norm": 10.572662353515625, |
| "learning_rate": 9.639210244594335e-06, |
| "logits/chosen": -0.2864235043525696, |
| "logits/rejected": -0.30632856488227844, |
| "logps/chosen": -301.4001159667969, |
| "logps/rejected": -254.0180206298828, |
| "loss": -0.4088, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 1.349990963935852, |
| "rewards/margins": 6.885331630706787, |
| "rewards/rejected": -5.535341262817383, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 9.515912055969238, |
| "learning_rate": 9.630467518464666e-06, |
| "logits/chosen": -0.2558160424232483, |
| "logits/rejected": -0.2956928312778473, |
| "logps/chosen": -304.60302734375, |
| "logps/rejected": -240.9836883544922, |
| "loss": -0.2976, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 1.0677791833877563, |
| "rewards/margins": 6.517449378967285, |
| "rewards/rejected": -5.44966983795166, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.37777777777777777, |
| "grad_norm": 7.812021255493164, |
| "learning_rate": 9.621624190938802e-06, |
| "logits/chosen": -0.2350511997938156, |
| "logits/rejected": -0.2532605528831482, |
| "logps/chosen": -320.7872009277344, |
| "logps/rejected": -254.2796173095703, |
| "loss": -0.4306, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 1.7731481790542603, |
| "rewards/margins": 7.587254524230957, |
| "rewards/rejected": -5.814105987548828, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.38222222222222224, |
| "grad_norm": 7.678309917449951, |
| "learning_rate": 9.612680454146609e-06, |
| "logits/chosen": -0.22189001739025116, |
| "logits/rejected": -0.2614109218120575, |
| "logps/chosen": -325.14239501953125, |
| "logps/rejected": -250.6171875, |
| "loss": -0.6808, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 2.040872573852539, |
| "rewards/margins": 8.241477012634277, |
| "rewards/rejected": -6.200604438781738, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.38666666666666666, |
| "grad_norm": 10.421648979187012, |
| "learning_rate": 9.603636502399436e-06, |
| "logits/chosen": -0.2654271721839905, |
| "logits/rejected": -0.302105575799942, |
| "logps/chosen": -332.26593017578125, |
| "logps/rejected": -245.6372528076172, |
| "loss": -0.6012, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 1.5932146310806274, |
| "rewards/margins": 8.200170516967773, |
| "rewards/rejected": -6.606956481933594, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.39111111111111113, |
| "grad_norm": 6.699859142303467, |
| "learning_rate": 9.594492532185909e-06, |
| "logits/chosen": -0.2850594222545624, |
| "logits/rejected": -0.3033252954483032, |
| "logps/chosen": -307.18463134765625, |
| "logps/rejected": -248.4318389892578, |
| "loss": -0.3909, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.7229059934616089, |
| "rewards/margins": 6.767951011657715, |
| "rewards/rejected": -6.045044422149658, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.39555555555555555, |
| "grad_norm": 9.903278350830078, |
| "learning_rate": 9.585248742167638e-06, |
| "logits/chosen": -0.2718963325023651, |
| "logits/rejected": -0.30517634749412537, |
| "logps/chosen": -309.89031982421875, |
| "logps/rejected": -250.9574432373047, |
| "loss": -0.3108, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -0.4615413546562195, |
| "rewards/margins": 6.469930171966553, |
| "rewards/rejected": -6.9314703941345215, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.4, |
| "grad_norm": 5.433151721954346, |
| "learning_rate": 9.57590533317493e-06, |
| "logits/chosen": -0.2626163959503174, |
| "logits/rejected": -0.32341477274894714, |
| "logps/chosen": -312.3274230957031, |
| "logps/rejected": -233.1435546875, |
| "loss": -0.7956, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.924404501914978, |
| "rewards/margins": 8.514490127563477, |
| "rewards/rejected": -7.590085029602051, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.40444444444444444, |
| "grad_norm": 8.837213516235352, |
| "learning_rate": 9.566462508202403e-06, |
| "logits/chosen": -0.248914435505867, |
| "logits/rejected": -0.3085024952888489, |
| "logps/chosen": -311.30328369140625, |
| "logps/rejected": -252.2725067138672, |
| "loss": -0.4715, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.2921377718448639, |
| "rewards/margins": 7.126054286956787, |
| "rewards/rejected": -6.833916664123535, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.4088888888888889, |
| "grad_norm": 7.693172454833984, |
| "learning_rate": 9.55692047240458e-06, |
| "logits/chosen": -0.2304973304271698, |
| "logits/rejected": -0.28682953119277954, |
| "logps/chosen": -311.18023681640625, |
| "logps/rejected": -240.9733123779297, |
| "loss": -0.6889, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.7987859845161438, |
| "rewards/margins": 8.30670166015625, |
| "rewards/rejected": -7.507915496826172, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.41333333333333333, |
| "grad_norm": 8.775394439697266, |
| "learning_rate": 9.547279433091446e-06, |
| "logits/chosen": -0.2938714325428009, |
| "logits/rejected": -0.314927875995636, |
| "logps/chosen": -307.7293701171875, |
| "logps/rejected": -249.83523559570312, |
| "loss": -0.5757, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.2602098286151886, |
| "rewards/margins": 7.839123725891113, |
| "rewards/rejected": -7.57891321182251, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.4177777777777778, |
| "grad_norm": 6.385190010070801, |
| "learning_rate": 9.537539599723924e-06, |
| "logits/chosen": -0.2282254993915558, |
| "logits/rejected": -0.29543009400367737, |
| "logps/chosen": -302.2158508300781, |
| "logps/rejected": -243.7989501953125, |
| "loss": -0.7249, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 1.112415075302124, |
| "rewards/margins": 8.864578247070312, |
| "rewards/rejected": -7.752162933349609, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.4222222222222222, |
| "grad_norm": 9.705090522766113, |
| "learning_rate": 9.527701183909336e-06, |
| "logits/chosen": -0.255817174911499, |
| "logits/rejected": -0.30061060190200806, |
| "logps/chosen": -319.11309814453125, |
| "logps/rejected": -252.5963592529297, |
| "loss": -0.4242, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -0.3803827464580536, |
| "rewards/margins": 7.071600437164307, |
| "rewards/rejected": -7.4519829750061035, |
| "step": 475 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 10.644119262695312, |
| "learning_rate": 9.51776439939681e-06, |
| "logits/chosen": -0.24410729110240936, |
| "logits/rejected": -0.31472498178482056, |
| "logps/chosen": -326.570556640625, |
| "logps/rejected": -254.5313262939453, |
| "loss": -0.6844, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.5738657116889954, |
| "rewards/margins": 9.123512268066406, |
| "rewards/rejected": -8.549646377563477, |
| "step": 480 |
| }, |
| { |
| "epoch": 0.4311111111111111, |
| "grad_norm": 12.36355972290039, |
| "learning_rate": 9.507729462072615e-06, |
| "logits/chosen": -0.24467067420482635, |
| "logits/rejected": -0.3331097364425659, |
| "logps/chosen": -322.2784729003906, |
| "logps/rejected": -260.8439025878906, |
| "loss": -0.5093, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.3209023177623749, |
| "rewards/margins": 8.416958808898926, |
| "rewards/rejected": -8.09605598449707, |
| "step": 485 |
| }, |
| { |
| "epoch": 0.43555555555555553, |
| "grad_norm": 11.505626678466797, |
| "learning_rate": 9.4975965899555e-06, |
| "logits/chosen": -0.28976163268089294, |
| "logits/rejected": -0.3090762794017792, |
| "logps/chosen": -307.57489013671875, |
| "logps/rejected": -246.5322265625, |
| "loss": -0.4303, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -1.3077417612075806, |
| "rewards/margins": 6.817173004150391, |
| "rewards/rejected": -8.124914169311523, |
| "step": 490 |
| }, |
| { |
| "epoch": 0.44, |
| "grad_norm": 15.380836486816406, |
| "learning_rate": 9.48736600319193e-06, |
| "logits/chosen": -0.2653730809688568, |
| "logits/rejected": -0.2980864644050598, |
| "logps/chosen": -319.14019775390625, |
| "logps/rejected": -263.46063232421875, |
| "loss": -0.3595, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.1717190742492676, |
| "rewards/margins": 7.247198581695557, |
| "rewards/rejected": -8.418917655944824, |
| "step": 495 |
| }, |
| { |
| "epoch": 0.4444444444444444, |
| "grad_norm": 7.199528694152832, |
| "learning_rate": 9.47703792405133e-06, |
| "logits/chosen": -0.2643812596797943, |
| "logits/rejected": -0.3057587146759033, |
| "logps/chosen": -305.49395751953125, |
| "logps/rejected": -250.5885467529297, |
| "loss": -0.7595, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -1.0399351119995117, |
| "rewards/margins": 8.203756332397461, |
| "rewards/rejected": -9.243691444396973, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.4444444444444444, |
| "eval_logits/chosen": -0.2629312574863434, |
| "eval_logits/rejected": -0.30713388323783875, |
| "eval_logps/chosen": -313.8876953125, |
| "eval_logps/rejected": -254.3212127685547, |
| "eval_loss": -0.574113667011261, |
| "eval_rewards/accuracies": 0.828249990940094, |
| "eval_rewards/chosen": -1.1603001356124878, |
| "eval_rewards/margins": 8.141514778137207, |
| "eval_rewards/rejected": -9.301814079284668, |
| "eval_runtime": 2192.8697, |
| "eval_samples_per_second": 1.824, |
| "eval_steps_per_second": 0.912, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.4488888888888889, |
| "grad_norm": 11.993717193603516, |
| "learning_rate": 9.466612576921223e-06, |
| "logits/chosen": -0.2699393332004547, |
| "logits/rejected": -0.3285272717475891, |
| "logps/chosen": -319.2097473144531, |
| "logps/rejected": -263.5858154296875, |
| "loss": -0.394, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -1.1724228858947754, |
| "rewards/margins": 7.439938545227051, |
| "rewards/rejected": -8.612360954284668, |
| "step": 505 |
| }, |
| { |
| "epoch": 0.4533333333333333, |
| "grad_norm": 11.781710624694824, |
| "learning_rate": 9.456090188302389e-06, |
| "logits/chosen": -0.26111698150634766, |
| "logits/rejected": -0.28280287981033325, |
| "logps/chosen": -309.879638671875, |
| "logps/rejected": -263.6683654785156, |
| "loss": -0.6619, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.2497330904006958, |
| "rewards/margins": 8.266626358032227, |
| "rewards/rejected": -9.516359329223633, |
| "step": 510 |
| }, |
| { |
| "epoch": 0.4577777777777778, |
| "grad_norm": 11.226400375366211, |
| "learning_rate": 9.445470986803922e-06, |
| "logits/chosen": -0.2626830041408539, |
| "logits/rejected": -0.3101075291633606, |
| "logps/chosen": -299.5857238769531, |
| "logps/rejected": -248.7251434326172, |
| "loss": -0.6192, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -1.528692603111267, |
| "rewards/margins": 8.301843643188477, |
| "rewards/rejected": -9.830536842346191, |
| "step": 515 |
| }, |
| { |
| "epoch": 0.4622222222222222, |
| "grad_norm": 8.692116737365723, |
| "learning_rate": 9.434755203138269e-06, |
| "logits/chosen": -0.27712422609329224, |
| "logits/rejected": -0.33624228835105896, |
| "logps/chosen": -341.59759521484375, |
| "logps/rejected": -257.9901123046875, |
| "loss": -0.5867, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -1.3974745273590088, |
| "rewards/margins": 8.428323745727539, |
| "rewards/rejected": -9.825799942016602, |
| "step": 520 |
| }, |
| { |
| "epoch": 0.4666666666666667, |
| "grad_norm": 7.316524982452393, |
| "learning_rate": 9.423943070116219e-06, |
| "logits/chosen": -0.3034690320491791, |
| "logits/rejected": -0.3194289803504944, |
| "logps/chosen": -294.5498046875, |
| "logps/rejected": -247.59463500976562, |
| "loss": -0.5768, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -2.136320114135742, |
| "rewards/margins": 7.421705722808838, |
| "rewards/rejected": -9.558026313781738, |
| "step": 525 |
| }, |
| { |
| "epoch": 0.4711111111111111, |
| "grad_norm": 15.097468376159668, |
| "learning_rate": 9.413034822641845e-06, |
| "logits/chosen": -0.29432040452957153, |
| "logits/rejected": -0.3473649322986603, |
| "logps/chosen": -314.51007080078125, |
| "logps/rejected": -255.99453735351562, |
| "loss": -0.3577, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -2.039400815963745, |
| "rewards/margins": 7.780667304992676, |
| "rewards/rejected": -9.820066452026367, |
| "step": 530 |
| }, |
| { |
| "epoch": 0.47555555555555556, |
| "grad_norm": 10.856350898742676, |
| "learning_rate": 9.402030697707398e-06, |
| "logits/chosen": -0.27809661626815796, |
| "logits/rejected": -0.3084755539894104, |
| "logps/chosen": -290.03839111328125, |
| "logps/rejected": -246.6155242919922, |
| "loss": -0.7648, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -1.642029047012329, |
| "rewards/margins": 8.514683723449707, |
| "rewards/rejected": -10.156713485717773, |
| "step": 535 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 10.466115951538086, |
| "learning_rate": 9.390930934388164e-06, |
| "logits/chosen": -0.25123220682144165, |
| "logits/rejected": -0.28659194707870483, |
| "logps/chosen": -310.38702392578125, |
| "logps/rejected": -264.1778869628906, |
| "loss": -0.6251, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -2.053196430206299, |
| "rewards/margins": 8.635331153869629, |
| "rewards/rejected": -10.68852710723877, |
| "step": 540 |
| }, |
| { |
| "epoch": 0.48444444444444446, |
| "grad_norm": 14.518105506896973, |
| "learning_rate": 9.37973577383726e-06, |
| "logits/chosen": -0.2105627954006195, |
| "logits/rejected": -0.2725834250450134, |
| "logps/chosen": -309.6773681640625, |
| "logps/rejected": -247.98507690429688, |
| "loss": -0.7806, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -1.5902966260910034, |
| "rewards/margins": 9.257573127746582, |
| "rewards/rejected": -10.847868919372559, |
| "step": 545 |
| }, |
| { |
| "epoch": 0.4888888888888889, |
| "grad_norm": 9.231803894042969, |
| "learning_rate": 9.368445459280405e-06, |
| "logits/chosen": -0.26593995094299316, |
| "logits/rejected": -0.28871750831604004, |
| "logps/chosen": -315.306884765625, |
| "logps/rejected": -262.83197021484375, |
| "loss": -0.5635, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -3.177497148513794, |
| "rewards/margins": 7.756557464599609, |
| "rewards/rejected": -10.934054374694824, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.49333333333333335, |
| "grad_norm": 9.906770706176758, |
| "learning_rate": 9.357060236010626e-06, |
| "logits/chosen": -0.25906693935394287, |
| "logits/rejected": -0.32186049222946167, |
| "logps/chosen": -326.8785095214844, |
| "logps/rejected": -273.9308166503906, |
| "loss": -0.6905, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -2.1445183753967285, |
| "rewards/margins": 9.113534927368164, |
| "rewards/rejected": -11.258054733276367, |
| "step": 555 |
| }, |
| { |
| "epoch": 0.49777777777777776, |
| "grad_norm": 7.245250225067139, |
| "learning_rate": 9.345580351382939e-06, |
| "logits/chosen": -0.2802310585975647, |
| "logits/rejected": -0.2841408848762512, |
| "logps/chosen": -293.36065673828125, |
| "logps/rejected": -265.8360900878906, |
| "loss": -0.7463, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -3.965707778930664, |
| "rewards/margins": 9.175572395324707, |
| "rewards/rejected": -13.141279220581055, |
| "step": 560 |
| }, |
| { |
| "epoch": 0.5022222222222222, |
| "grad_norm": 11.543607711791992, |
| "learning_rate": 9.334006054808966e-06, |
| "logits/chosen": -0.2962619960308075, |
| "logits/rejected": -0.3181178569793701, |
| "logps/chosen": -326.261962890625, |
| "logps/rejected": -281.9925231933594, |
| "loss": -0.5012, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -4.311570644378662, |
| "rewards/margins": 8.469701766967773, |
| "rewards/rejected": -12.781272888183594, |
| "step": 565 |
| }, |
| { |
| "epoch": 0.5066666666666667, |
| "grad_norm": 18.13285255432129, |
| "learning_rate": 9.322337597751525e-06, |
| "logits/chosen": -0.29192933440208435, |
| "logits/rejected": -0.32068902254104614, |
| "logps/chosen": -320.00146484375, |
| "logps/rejected": -267.2696838378906, |
| "loss": 0.0417, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -5.439437389373779, |
| "rewards/margins": 6.237511157989502, |
| "rewards/rejected": -11.676947593688965, |
| "step": 570 |
| }, |
| { |
| "epoch": 0.5111111111111111, |
| "grad_norm": 15.025782585144043, |
| "learning_rate": 9.310575233719155e-06, |
| "logits/chosen": -0.2575679421424866, |
| "logits/rejected": -0.28944242000579834, |
| "logps/chosen": -312.4665222167969, |
| "logps/rejected": -266.6491394042969, |
| "loss": -0.5652, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -2.7938873767852783, |
| "rewards/margins": 8.282114028930664, |
| "rewards/rejected": -11.07600212097168, |
| "step": 575 |
| }, |
| { |
| "epoch": 0.5155555555555555, |
| "grad_norm": 10.936811447143555, |
| "learning_rate": 9.29871921826062e-06, |
| "logits/chosen": -0.2927904725074768, |
| "logits/rejected": -0.35370174050331116, |
| "logps/chosen": -321.13885498046875, |
| "logps/rejected": -268.1432800292969, |
| "loss": -0.2429, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -4.02617883682251, |
| "rewards/margins": 8.042525291442871, |
| "rewards/rejected": -12.068704605102539, |
| "step": 580 |
| }, |
| { |
| "epoch": 0.52, |
| "grad_norm": 12.064530372619629, |
| "learning_rate": 9.28676980895935e-06, |
| "logits/chosen": -0.24123439192771912, |
| "logits/rejected": -0.2889128625392914, |
| "logps/chosen": -308.1340637207031, |
| "logps/rejected": -256.29632568359375, |
| "loss": -0.8401, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -3.284236431121826, |
| "rewards/margins": 10.122550010681152, |
| "rewards/rejected": -13.40678596496582, |
| "step": 585 |
| }, |
| { |
| "epoch": 0.5244444444444445, |
| "grad_norm": 13.778730392456055, |
| "learning_rate": 9.274727265427849e-06, |
| "logits/chosen": -0.2769649922847748, |
| "logits/rejected": -0.31647247076034546, |
| "logps/chosen": -306.0679626464844, |
| "logps/rejected": -254.93179321289062, |
| "loss": -0.7553, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -4.035617828369141, |
| "rewards/margins": 9.203435897827148, |
| "rewards/rejected": -13.239053726196289, |
| "step": 590 |
| }, |
| { |
| "epoch": 0.5288888888888889, |
| "grad_norm": 8.471962928771973, |
| "learning_rate": 9.262591849302049e-06, |
| "logits/chosen": -0.2713521122932434, |
| "logits/rejected": -0.3014729619026184, |
| "logps/chosen": -299.43475341796875, |
| "logps/rejected": -266.22686767578125, |
| "loss": -0.4191, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -4.5517988204956055, |
| "rewards/margins": 8.98070240020752, |
| "rewards/rejected": -13.532503128051758, |
| "step": 595 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 12.954193115234375, |
| "learning_rate": 9.250363824235629e-06, |
| "logits/chosen": -0.2955438494682312, |
| "logits/rejected": -0.3413962721824646, |
| "logps/chosen": -318.26068115234375, |
| "logps/rejected": -256.75689697265625, |
| "loss": -0.5025, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -3.381830930709839, |
| "rewards/margins": 9.101526260375977, |
| "rewards/rejected": -12.483358383178711, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.5377777777777778, |
| "grad_norm": 10.307146072387695, |
| "learning_rate": 9.238043455894294e-06, |
| "logits/chosen": -0.27938082814216614, |
| "logits/rejected": -0.3281027674674988, |
| "logps/chosen": -327.4522399902344, |
| "logps/rejected": -257.4443054199219, |
| "loss": -0.7464, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -4.192465305328369, |
| "rewards/margins": 8.572819709777832, |
| "rewards/rejected": -12.765284538269043, |
| "step": 605 |
| }, |
| { |
| "epoch": 0.5422222222222223, |
| "grad_norm": 8.460762023925781, |
| "learning_rate": 9.225631011949987e-06, |
| "logits/chosen": -0.2906576991081238, |
| "logits/rejected": -0.32649320363998413, |
| "logps/chosen": -341.95928955078125, |
| "logps/rejected": -275.3990783691406, |
| "loss": -0.9454, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -2.693796396255493, |
| "rewards/margins": 10.737831115722656, |
| "rewards/rejected": -13.43162727355957, |
| "step": 610 |
| }, |
| { |
| "epoch": 0.5466666666666666, |
| "grad_norm": 18.260358810424805, |
| "learning_rate": 9.213126762075088e-06, |
| "logits/chosen": -0.3098008632659912, |
| "logits/rejected": -0.3394979238510132, |
| "logps/chosen": -306.67449951171875, |
| "logps/rejected": -263.02349853515625, |
| "loss": -0.5332, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -4.892157554626465, |
| "rewards/margins": 9.199603080749512, |
| "rewards/rejected": -14.091761589050293, |
| "step": 615 |
| }, |
| { |
| "epoch": 0.5511111111111111, |
| "grad_norm": 12.752532005310059, |
| "learning_rate": 9.200530977936551e-06, |
| "logits/chosen": -0.3172837793827057, |
| "logits/rejected": -0.3619407117366791, |
| "logps/chosen": -349.4862976074219, |
| "logps/rejected": -279.5086669921875, |
| "loss": -0.6241, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -4.374999046325684, |
| "rewards/margins": 9.857782363891602, |
| "rewards/rejected": -14.232782363891602, |
| "step": 620 |
| }, |
| { |
| "epoch": 0.5555555555555556, |
| "grad_norm": 15.203842163085938, |
| "learning_rate": 9.187843933189994e-06, |
| "logits/chosen": -0.28893885016441345, |
| "logits/rejected": -0.3426817059516907, |
| "logps/chosen": -328.72979736328125, |
| "logps/rejected": -271.71099853515625, |
| "loss": -0.8634, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -3.5763354301452637, |
| "rewards/margins": 11.408061027526855, |
| "rewards/rejected": -14.984395980834961, |
| "step": 625 |
| }, |
| { |
| "epoch": 0.56, |
| "grad_norm": 10.816106796264648, |
| "learning_rate": 9.175065903473769e-06, |
| "logits/chosen": -0.2791399657726288, |
| "logits/rejected": -0.2996821403503418, |
| "logps/chosen": -321.34771728515625, |
| "logps/rejected": -280.1087341308594, |
| "loss": -0.5511, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -5.1469244956970215, |
| "rewards/margins": 9.314436912536621, |
| "rewards/rejected": -14.4613618850708, |
| "step": 630 |
| }, |
| { |
| "epoch": 0.5644444444444444, |
| "grad_norm": 8.36387825012207, |
| "learning_rate": 9.162197166402957e-06, |
| "logits/chosen": -0.29182273149490356, |
| "logits/rejected": -0.32408252358436584, |
| "logps/chosen": -316.68487548828125, |
| "logps/rejected": -270.219482421875, |
| "loss": -1.0203, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -4.753455638885498, |
| "rewards/margins": 10.925695419311523, |
| "rewards/rejected": -15.679153442382812, |
| "step": 635 |
| }, |
| { |
| "epoch": 0.5688888888888889, |
| "grad_norm": 11.839485168457031, |
| "learning_rate": 9.149238001563348e-06, |
| "logits/chosen": -0.30801886320114136, |
| "logits/rejected": -0.329951673746109, |
| "logps/chosen": -312.73577880859375, |
| "logps/rejected": -267.54437255859375, |
| "loss": -0.5904, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -6.468144416809082, |
| "rewards/margins": 8.780177116394043, |
| "rewards/rejected": -15.248323440551758, |
| "step": 640 |
| }, |
| { |
| "epoch": 0.5733333333333334, |
| "grad_norm": 16.650188446044922, |
| "learning_rate": 9.136188690505363e-06, |
| "logits/chosen": -0.2637523412704468, |
| "logits/rejected": -0.31697210669517517, |
| "logps/chosen": -331.0145568847656, |
| "logps/rejected": -271.3872985839844, |
| "loss": -0.5915, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -5.373086452484131, |
| "rewards/margins": 8.960186958312988, |
| "rewards/rejected": -14.333274841308594, |
| "step": 645 |
| }, |
| { |
| "epoch": 0.5777777777777777, |
| "grad_norm": 18.575305938720703, |
| "learning_rate": 9.123049516737936e-06, |
| "logits/chosen": -0.3117810785770416, |
| "logits/rejected": -0.3635488450527191, |
| "logps/chosen": -328.19989013671875, |
| "logps/rejected": -272.4068603515625, |
| "loss": -0.6851, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -5.281620979309082, |
| "rewards/margins": 10.281137466430664, |
| "rewards/rejected": -15.562756538391113, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.5822222222222222, |
| "grad_norm": 16.429004669189453, |
| "learning_rate": 9.109820765722357e-06, |
| "logits/chosen": -0.27543455362319946, |
| "logits/rejected": -0.31441715359687805, |
| "logps/chosen": -335.43609619140625, |
| "logps/rejected": -286.1941833496094, |
| "loss": -0.8026, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -5.2430877685546875, |
| "rewards/margins": 10.86163330078125, |
| "rewards/rejected": -16.10472297668457, |
| "step": 655 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 9.399615287780762, |
| "learning_rate": 9.096502724866067e-06, |
| "logits/chosen": -0.3014602065086365, |
| "logits/rejected": -0.35103824734687805, |
| "logps/chosen": -359.2838439941406, |
| "logps/rejected": -294.8686828613281, |
| "loss": -1.0671, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -4.225809574127197, |
| "rewards/margins": 11.496713638305664, |
| "rewards/rejected": -15.722521781921387, |
| "step": 660 |
| }, |
| { |
| "epoch": 0.5911111111111111, |
| "grad_norm": 14.605314254760742, |
| "learning_rate": 9.083095683516414e-06, |
| "logits/chosen": -0.28259098529815674, |
| "logits/rejected": -0.32325831055641174, |
| "logps/chosen": -350.68878173828125, |
| "logps/rejected": -277.5746154785156, |
| "loss": -0.9032, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -6.993232727050781, |
| "rewards/margins": 10.017694473266602, |
| "rewards/rejected": -17.010927200317383, |
| "step": 665 |
| }, |
| { |
| "epoch": 0.5955555555555555, |
| "grad_norm": 17.158226013183594, |
| "learning_rate": 9.069599932954371e-06, |
| "logits/chosen": -0.29114705324172974, |
| "logits/rejected": -0.3473047912120819, |
| "logps/chosen": -339.568603515625, |
| "logps/rejected": -287.64666748046875, |
| "loss": -0.5308, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -8.217833518981934, |
| "rewards/margins": 8.741876602172852, |
| "rewards/rejected": -16.9597110748291, |
| "step": 670 |
| }, |
| { |
| "epoch": 0.6, |
| "grad_norm": 16.861963272094727, |
| "learning_rate": 9.056015766388205e-06, |
| "logits/chosen": -0.3223651945590973, |
| "logits/rejected": -0.36349570751190186, |
| "logps/chosen": -334.6025695800781, |
| "logps/rejected": -285.73828125, |
| "loss": -0.7831, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -7.5147528648376465, |
| "rewards/margins": 10.620495796203613, |
| "rewards/rejected": -18.1352481842041, |
| "step": 675 |
| }, |
| { |
| "epoch": 0.6044444444444445, |
| "grad_norm": 14.257497787475586, |
| "learning_rate": 9.042343478947103e-06, |
| "logits/chosen": -0.3066635727882385, |
| "logits/rejected": -0.32420462369918823, |
| "logps/chosen": -337.81097412109375, |
| "logps/rejected": -286.6188049316406, |
| "loss": -0.9349, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -6.352471351623535, |
| "rewards/margins": 11.51642894744873, |
| "rewards/rejected": -17.868900299072266, |
| "step": 680 |
| }, |
| { |
| "epoch": 0.6088888888888889, |
| "grad_norm": 16.795236587524414, |
| "learning_rate": 9.028583367674767e-06, |
| "logits/chosen": -0.34612902998924255, |
| "logits/rejected": -0.36396104097366333, |
| "logps/chosen": -332.6644287109375, |
| "logps/rejected": -287.1361083984375, |
| "loss": -0.9376, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -5.387200355529785, |
| "rewards/margins": 11.706459999084473, |
| "rewards/rejected": -17.093660354614258, |
| "step": 685 |
| }, |
| { |
| "epoch": 0.6133333333333333, |
| "grad_norm": 11.827564239501953, |
| "learning_rate": 9.014735731522952e-06, |
| "logits/chosen": -0.3044932782649994, |
| "logits/rejected": -0.33501502871513367, |
| "logps/chosen": -331.37835693359375, |
| "logps/rejected": -279.79718017578125, |
| "loss": -1.0226, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -5.231140613555908, |
| "rewards/margins": 12.324139595031738, |
| "rewards/rejected": -17.555278778076172, |
| "step": 690 |
| }, |
| { |
| "epoch": 0.6177777777777778, |
| "grad_norm": 13.102601051330566, |
| "learning_rate": 9.00080087134498e-06, |
| "logits/chosen": -0.31660374999046326, |
| "logits/rejected": -0.3677740693092346, |
| "logps/chosen": -340.77081298828125, |
| "logps/rejected": -275.3927001953125, |
| "loss": -0.8796, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -5.748688220977783, |
| "rewards/margins": 11.431347846984863, |
| "rewards/rejected": -17.180036544799805, |
| "step": 695 |
| }, |
| { |
| "epoch": 0.6222222222222222, |
| "grad_norm": 28.49679946899414, |
| "learning_rate": 8.9867790898892e-06, |
| "logits/chosen": -0.3224649131298065, |
| "logits/rejected": -0.3925584852695465, |
| "logps/chosen": -349.48919677734375, |
| "logps/rejected": -279.79119873046875, |
| "loss": -0.8641, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -6.573420524597168, |
| "rewards/margins": 11.63664436340332, |
| "rewards/rejected": -18.210065841674805, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.6266666666666667, |
| "grad_norm": 17.276578903198242, |
| "learning_rate": 8.972670691792409e-06, |
| "logits/chosen": -0.3031178414821625, |
| "logits/rejected": -0.347816526889801, |
| "logps/chosen": -332.0860900878906, |
| "logps/rejected": -278.52264404296875, |
| "loss": -0.6865, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -7.202242851257324, |
| "rewards/margins": 10.682793617248535, |
| "rewards/rejected": -17.88503646850586, |
| "step": 705 |
| }, |
| { |
| "epoch": 0.6311111111111111, |
| "grad_norm": 13.52579402923584, |
| "learning_rate": 8.958475983573234e-06, |
| "logits/chosen": -0.3044522702693939, |
| "logits/rejected": -0.34488362073898315, |
| "logps/chosen": -337.5535583496094, |
| "logps/rejected": -295.95428466796875, |
| "loss": -0.9102, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -6.68551778793335, |
| "rewards/margins": 11.271787643432617, |
| "rewards/rejected": -17.957305908203125, |
| "step": 710 |
| }, |
| { |
| "epoch": 0.6355555555555555, |
| "grad_norm": 12.732015609741211, |
| "learning_rate": 8.944195273625472e-06, |
| "logits/chosen": -0.2973068356513977, |
| "logits/rejected": -0.35419678688049316, |
| "logps/chosen": -318.517822265625, |
| "logps/rejected": -271.3919677734375, |
| "loss": -0.8689, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -7.6208038330078125, |
| "rewards/margins": 10.748211860656738, |
| "rewards/rejected": -18.369014739990234, |
| "step": 715 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 15.25625991821289, |
| "learning_rate": 8.92982887221139e-06, |
| "logits/chosen": -0.3091279864311218, |
| "logits/rejected": -0.3462229073047638, |
| "logps/chosen": -345.78131103515625, |
| "logps/rejected": -292.4202575683594, |
| "loss": -0.4038, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -9.038877487182617, |
| "rewards/margins": 10.578144073486328, |
| "rewards/rejected": -19.617021560668945, |
| "step": 720 |
| }, |
| { |
| "epoch": 0.6444444444444445, |
| "grad_norm": 8.678261756896973, |
| "learning_rate": 8.915377091454992e-06, |
| "logits/chosen": -0.2622337341308594, |
| "logits/rejected": -0.3454502820968628, |
| "logps/chosen": -330.6527404785156, |
| "logps/rejected": -271.9297790527344, |
| "loss": -0.9941, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -6.34903621673584, |
| "rewards/margins": 11.77011775970459, |
| "rewards/rejected": -18.119152069091797, |
| "step": 725 |
| }, |
| { |
| "epoch": 0.6488888888888888, |
| "grad_norm": 18.099544525146484, |
| "learning_rate": 8.900840245335225e-06, |
| "logits/chosen": -0.29967910051345825, |
| "logits/rejected": -0.3550174832344055, |
| "logps/chosen": -339.3348388671875, |
| "logps/rejected": -284.2086181640625, |
| "loss": -0.6141, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -8.690653800964355, |
| "rewards/margins": 10.278780937194824, |
| "rewards/rejected": -18.969436645507812, |
| "step": 730 |
| }, |
| { |
| "epoch": 0.6533333333333333, |
| "grad_norm": 17.058128356933594, |
| "learning_rate": 8.886218649679162e-06, |
| "logits/chosen": -0.30947160720825195, |
| "logits/rejected": -0.3345088064670563, |
| "logps/chosen": -320.3451232910156, |
| "logps/rejected": -277.7720031738281, |
| "loss": -1.014, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -8.53862190246582, |
| "rewards/margins": 12.284268379211426, |
| "rewards/rejected": -20.822891235351562, |
| "step": 735 |
| }, |
| { |
| "epoch": 0.6577777777777778, |
| "grad_norm": 13.636448860168457, |
| "learning_rate": 8.871512622155147e-06, |
| "logits/chosen": -0.2878524363040924, |
| "logits/rejected": -0.3395880162715912, |
| "logps/chosen": -362.52899169921875, |
| "logps/rejected": -303.67620849609375, |
| "loss": -0.9753, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -8.569745063781738, |
| "rewards/margins": 12.254476547241211, |
| "rewards/rejected": -20.824222564697266, |
| "step": 740 |
| }, |
| { |
| "epoch": 0.6622222222222223, |
| "grad_norm": 13.943758964538574, |
| "learning_rate": 8.856722482265886e-06, |
| "logits/chosen": -0.2777239978313446, |
| "logits/rejected": -0.2970428466796875, |
| "logps/chosen": -317.4947509765625, |
| "logps/rejected": -292.8834533691406, |
| "loss": -0.952, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -9.092373847961426, |
| "rewards/margins": 11.070058822631836, |
| "rewards/rejected": -20.162433624267578, |
| "step": 745 |
| }, |
| { |
| "epoch": 0.6666666666666666, |
| "grad_norm": 10.792975425720215, |
| "learning_rate": 8.841848551341506e-06, |
| "logits/chosen": -0.300568550825119, |
| "logits/rejected": -0.35186997056007385, |
| "logps/chosen": -342.993408203125, |
| "logps/rejected": -283.03594970703125, |
| "loss": -0.9528, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -8.736288070678711, |
| "rewards/margins": 11.417104721069336, |
| "rewards/rejected": -20.153392791748047, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.6711111111111111, |
| "grad_norm": 14.272412300109863, |
| "learning_rate": 8.826891152532579e-06, |
| "logits/chosen": -0.24646346271038055, |
| "logits/rejected": -0.3181930184364319, |
| "logps/chosen": -330.8204040527344, |
| "logps/rejected": -290.7005310058594, |
| "loss": -0.8532, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -8.7890625, |
| "rewards/margins": 12.464083671569824, |
| "rewards/rejected": -21.25314712524414, |
| "step": 755 |
| }, |
| { |
| "epoch": 0.6755555555555556, |
| "grad_norm": 18.93645477294922, |
| "learning_rate": 8.811850610803094e-06, |
| "logits/chosen": -0.2739986181259155, |
| "logits/rejected": -0.3238711357116699, |
| "logps/chosen": -351.69171142578125, |
| "logps/rejected": -305.0896301269531, |
| "loss": -0.9423, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -8.920026779174805, |
| "rewards/margins": 12.025522232055664, |
| "rewards/rejected": -20.945547103881836, |
| "step": 760 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 18.738351821899414, |
| "learning_rate": 8.796727252923403e-06, |
| "logits/chosen": -0.31761056184768677, |
| "logits/rejected": -0.3448847532272339, |
| "logps/chosen": -331.40557861328125, |
| "logps/rejected": -296.1661071777344, |
| "loss": -0.5371, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -10.65619945526123, |
| "rewards/margins": 11.044633865356445, |
| "rewards/rejected": -21.700834274291992, |
| "step": 765 |
| }, |
| { |
| "epoch": 0.6844444444444444, |
| "grad_norm": 17.68587875366211, |
| "learning_rate": 8.781521407463119e-06, |
| "logits/chosen": -0.29655805230140686, |
| "logits/rejected": -0.35701996088027954, |
| "logps/chosen": -353.18145751953125, |
| "logps/rejected": -300.9978942871094, |
| "loss": -1.0413, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -8.621820449829102, |
| "rewards/margins": 13.096748352050781, |
| "rewards/rejected": -21.718570709228516, |
| "step": 770 |
| }, |
| { |
| "epoch": 0.6888888888888889, |
| "grad_norm": 8.765332221984863, |
| "learning_rate": 8.766233404783975e-06, |
| "logits/chosen": -0.33828821778297424, |
| "logits/rejected": -0.35216769576072693, |
| "logps/chosen": -349.8067626953125, |
| "logps/rejected": -313.57818603515625, |
| "loss": -0.856, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -9.512666702270508, |
| "rewards/margins": 12.066935539245605, |
| "rewards/rejected": -21.57960319519043, |
| "step": 775 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 20.219892501831055, |
| "learning_rate": 8.750863577032652e-06, |
| "logits/chosen": -0.3195672929286957, |
| "logits/rejected": -0.3713618218898773, |
| "logps/chosen": -358.82037353515625, |
| "logps/rejected": -303.29571533203125, |
| "loss": -1.2232, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -9.787614822387695, |
| "rewards/margins": 13.660125732421875, |
| "rewards/rejected": -23.447738647460938, |
| "step": 780 |
| }, |
| { |
| "epoch": 0.6977777777777778, |
| "grad_norm": 14.765336036682129, |
| "learning_rate": 8.735412258133562e-06, |
| "logits/chosen": -0.3235880136489868, |
| "logits/rejected": -0.36679068207740784, |
| "logps/chosen": -354.3951416015625, |
| "logps/rejected": -299.1098937988281, |
| "loss": -0.8905, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -10.815263748168945, |
| "rewards/margins": 12.30003547668457, |
| "rewards/rejected": -23.115299224853516, |
| "step": 785 |
| }, |
| { |
| "epoch": 0.7022222222222222, |
| "grad_norm": 13.638387680053711, |
| "learning_rate": 8.719879783781585e-06, |
| "logits/chosen": -0.33130335807800293, |
| "logits/rejected": -0.3633490204811096, |
| "logps/chosen": -351.318603515625, |
| "logps/rejected": -302.3772888183594, |
| "loss": -1.0524, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -9.303329467773438, |
| "rewards/margins": 13.00804615020752, |
| "rewards/rejected": -22.31137466430664, |
| "step": 790 |
| }, |
| { |
| "epoch": 0.7066666666666667, |
| "grad_norm": 10.990659713745117, |
| "learning_rate": 8.704266491434787e-06, |
| "logits/chosen": -0.3289201259613037, |
| "logits/rejected": -0.36471107602119446, |
| "logps/chosen": -330.88763427734375, |
| "logps/rejected": -286.8157958984375, |
| "loss": -0.6598, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -10.67319107055664, |
| "rewards/margins": 11.411112785339355, |
| "rewards/rejected": -22.084304809570312, |
| "step": 795 |
| }, |
| { |
| "epoch": 0.7111111111111111, |
| "grad_norm": 12.232583045959473, |
| "learning_rate": 8.688572720307083e-06, |
| "logits/chosen": -0.3058468997478485, |
| "logits/rejected": -0.3826626241207123, |
| "logps/chosen": -360.0539855957031, |
| "logps/rejected": -302.0265808105469, |
| "loss": -0.9798, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -9.977747917175293, |
| "rewards/margins": 13.34874153137207, |
| "rewards/rejected": -23.326488494873047, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.7155555555555555, |
| "grad_norm": 28.097835540771484, |
| "learning_rate": 8.672798811360863e-06, |
| "logits/chosen": -0.3440350890159607, |
| "logits/rejected": -0.3669665455818176, |
| "logps/chosen": -348.4472351074219, |
| "logps/rejected": -317.9486083984375, |
| "loss": -1.158, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -9.318201065063477, |
| "rewards/margins": 13.85308837890625, |
| "rewards/rejected": -23.17129135131836, |
| "step": 805 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 13.566072463989258, |
| "learning_rate": 8.656945107299598e-06, |
| "logits/chosen": -0.32617539167404175, |
| "logits/rejected": -0.3627128601074219, |
| "logps/chosen": -350.51495361328125, |
| "logps/rejected": -313.4057922363281, |
| "loss": -1.1211, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -9.836647987365723, |
| "rewards/margins": 13.414007186889648, |
| "rewards/rejected": -23.250656127929688, |
| "step": 810 |
| }, |
| { |
| "epoch": 0.7244444444444444, |
| "grad_norm": 10.32918930053711, |
| "learning_rate": 8.641011952560372e-06, |
| "logits/chosen": -0.3140029311180115, |
| "logits/rejected": -0.3582364618778229, |
| "logps/chosen": -333.9091796875, |
| "logps/rejected": -282.69061279296875, |
| "loss": -0.8948, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -10.822344779968262, |
| "rewards/margins": 11.024767875671387, |
| "rewards/rejected": -21.84711265563965, |
| "step": 815 |
| }, |
| { |
| "epoch": 0.7288888888888889, |
| "grad_norm": 14.581253051757812, |
| "learning_rate": 8.624999693306422e-06, |
| "logits/chosen": -0.33729246258735657, |
| "logits/rejected": -0.3753616213798523, |
| "logps/chosen": -342.4247741699219, |
| "logps/rejected": -309.2422790527344, |
| "loss": -0.9419, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -11.228838920593262, |
| "rewards/margins": 13.112091064453125, |
| "rewards/rejected": -24.34092903137207, |
| "step": 820 |
| }, |
| { |
| "epoch": 0.7333333333333333, |
| "grad_norm": 11.523558616638184, |
| "learning_rate": 8.608908677419606e-06, |
| "logits/chosen": -0.37991100549697876, |
| "logits/rejected": -0.40186434984207153, |
| "logps/chosen": -348.12396240234375, |
| "logps/rejected": -309.076171875, |
| "loss": -1.0504, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -12.649127006530762, |
| "rewards/margins": 12.755289077758789, |
| "rewards/rejected": -25.404415130615234, |
| "step": 825 |
| }, |
| { |
| "epoch": 0.7377777777777778, |
| "grad_norm": 17.81552505493164, |
| "learning_rate": 8.592739254492845e-06, |
| "logits/chosen": -0.362493097782135, |
| "logits/rejected": -0.4177095293998718, |
| "logps/chosen": -335.11981201171875, |
| "logps/rejected": -290.92218017578125, |
| "loss": -1.1041, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -11.677441596984863, |
| "rewards/margins": 13.119203567504883, |
| "rewards/rejected": -24.79664421081543, |
| "step": 830 |
| }, |
| { |
| "epoch": 0.7422222222222222, |
| "grad_norm": 20.329221725463867, |
| "learning_rate": 8.576491775822527e-06, |
| "logits/chosen": -0.33437713980674744, |
| "logits/rejected": -0.39904457330703735, |
| "logps/chosen": -357.16943359375, |
| "logps/rejected": -297.7870178222656, |
| "loss": -0.724, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -11.969806671142578, |
| "rewards/margins": 12.316507339477539, |
| "rewards/rejected": -24.28631591796875, |
| "step": 835 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 19.302101135253906, |
| "learning_rate": 8.560166594400878e-06, |
| "logits/chosen": -0.3832574486732483, |
| "logits/rejected": -0.44351863861083984, |
| "logps/chosen": -352.62115478515625, |
| "logps/rejected": -304.46124267578125, |
| "loss": -0.6363, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -12.629277229309082, |
| "rewards/margins": 11.651094436645508, |
| "rewards/rejected": -24.280370712280273, |
| "step": 840 |
| }, |
| { |
| "epoch": 0.7511111111111111, |
| "grad_norm": 14.173489570617676, |
| "learning_rate": 8.543764064908295e-06, |
| "logits/chosen": -0.34056347608566284, |
| "logits/rejected": -0.39399194717407227, |
| "logps/chosen": -340.8840026855469, |
| "logps/rejected": -307.18603515625, |
| "loss": -1.2865, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -11.463663101196289, |
| "rewards/margins": 13.927221298217773, |
| "rewards/rejected": -25.390884399414062, |
| "step": 845 |
| }, |
| { |
| "epoch": 0.7555555555555555, |
| "grad_norm": 22.398326873779297, |
| "learning_rate": 8.527284543705631e-06, |
| "logits/chosen": -0.37620821595191956, |
| "logits/rejected": -0.4051085412502289, |
| "logps/chosen": -341.5446472167969, |
| "logps/rejected": -314.3455810546875, |
| "loss": -1.1236, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -11.881677627563477, |
| "rewards/margins": 14.11926555633545, |
| "rewards/rejected": -26.00094223022461, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.76, |
| "grad_norm": 20.017797470092773, |
| "learning_rate": 8.510728388826464e-06, |
| "logits/chosen": -0.33530497550964355, |
| "logits/rejected": -0.3962380290031433, |
| "logps/chosen": -341.4028625488281, |
| "logps/rejected": -305.88824462890625, |
| "loss": -1.5163, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": -10.922323226928711, |
| "rewards/margins": 15.069772720336914, |
| "rewards/rejected": -25.992095947265625, |
| "step": 855 |
| }, |
| { |
| "epoch": 0.7644444444444445, |
| "grad_norm": 13.252291679382324, |
| "learning_rate": 8.494095959969309e-06, |
| "logits/chosen": -0.34795650839805603, |
| "logits/rejected": -0.40874728560447693, |
| "logps/chosen": -348.9808654785156, |
| "logps/rejected": -308.14349365234375, |
| "loss": -0.9905, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -13.379107475280762, |
| "rewards/margins": 12.994186401367188, |
| "rewards/rejected": -26.373294830322266, |
| "step": 860 |
| }, |
| { |
| "epoch": 0.7688888888888888, |
| "grad_norm": 18.38790512084961, |
| "learning_rate": 8.477387618489808e-06, |
| "logits/chosen": -0.3455773890018463, |
| "logits/rejected": -0.40834465622901917, |
| "logps/chosen": -343.2769470214844, |
| "logps/rejected": -297.47784423828125, |
| "loss": -1.4511, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -10.266695022583008, |
| "rewards/margins": 15.304100036621094, |
| "rewards/rejected": -25.5707950592041, |
| "step": 865 |
| }, |
| { |
| "epoch": 0.7733333333333333, |
| "grad_norm": 21.267852783203125, |
| "learning_rate": 8.460603727392877e-06, |
| "logits/chosen": -0.35729557275772095, |
| "logits/rejected": -0.3905247449874878, |
| "logps/chosen": -370.48577880859375, |
| "logps/rejected": -324.64532470703125, |
| "loss": -1.1358, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -12.016322135925293, |
| "rewards/margins": 15.51159381866455, |
| "rewards/rejected": -27.527912139892578, |
| "step": 870 |
| }, |
| { |
| "epoch": 0.7777777777777778, |
| "grad_norm": 20.68170166015625, |
| "learning_rate": 8.443744651324828e-06, |
| "logits/chosen": -0.3603067994117737, |
| "logits/rejected": -0.40933218598365784, |
| "logps/chosen": -356.02154541015625, |
| "logps/rejected": -310.26666259765625, |
| "loss": -1.0198, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -13.34446907043457, |
| "rewards/margins": 13.9938325881958, |
| "rewards/rejected": -27.338302612304688, |
| "step": 875 |
| }, |
| { |
| "epoch": 0.7822222222222223, |
| "grad_norm": 14.795793533325195, |
| "learning_rate": 8.426810756565428e-06, |
| "logits/chosen": -0.3585900664329529, |
| "logits/rejected": -0.42686209082603455, |
| "logps/chosen": -368.9267272949219, |
| "logps/rejected": -311.18023681640625, |
| "loss": -1.5537, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -13.114725112915039, |
| "rewards/margins": 16.6258487701416, |
| "rewards/rejected": -29.74057388305664, |
| "step": 880 |
| }, |
| { |
| "epoch": 0.7866666666666666, |
| "grad_norm": 19.962947845458984, |
| "learning_rate": 8.409802411019962e-06, |
| "logits/chosen": -0.347336083650589, |
| "logits/rejected": -0.4067932665348053, |
| "logps/chosen": -343.19158935546875, |
| "logps/rejected": -304.2242126464844, |
| "loss": -1.3862, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -13.927999496459961, |
| "rewards/margins": 15.6153564453125, |
| "rewards/rejected": -29.54335594177246, |
| "step": 885 |
| }, |
| { |
| "epoch": 0.7911111111111111, |
| "grad_norm": 16.915666580200195, |
| "learning_rate": 8.392719984211228e-06, |
| "logits/chosen": -0.36178287863731384, |
| "logits/rejected": -0.42369580268859863, |
| "logps/chosen": -363.2778625488281, |
| "logps/rejected": -314.5802001953125, |
| "loss": -1.3641, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -14.133458137512207, |
| "rewards/margins": 13.37634563446045, |
| "rewards/rejected": -27.50980567932129, |
| "step": 890 |
| }, |
| { |
| "epoch": 0.7955555555555556, |
| "grad_norm": 16.34973907470703, |
| "learning_rate": 8.375563847271506e-06, |
| "logits/chosen": -0.3902398645877838, |
| "logits/rejected": -0.4178919792175293, |
| "logps/chosen": -354.6260070800781, |
| "logps/rejected": -320.308837890625, |
| "loss": -1.296, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -14.261367797851562, |
| "rewards/margins": 15.514287948608398, |
| "rewards/rejected": -29.77565574645996, |
| "step": 895 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 20.18850326538086, |
| "learning_rate": 8.35833437293451e-06, |
| "logits/chosen": -0.3586779534816742, |
| "logits/rejected": -0.3966183066368103, |
| "logps/chosen": -353.3863830566406, |
| "logps/rejected": -317.9190979003906, |
| "loss": -1.0465, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -13.865551948547363, |
| "rewards/margins": 14.156455993652344, |
| "rewards/rejected": -28.02200698852539, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.8044444444444444, |
| "grad_norm": 18.74745750427246, |
| "learning_rate": 8.341031935527267e-06, |
| "logits/chosen": -0.35274258255958557, |
| "logits/rejected": -0.4157370626926422, |
| "logps/chosen": -365.7769470214844, |
| "logps/rejected": -320.2703552246094, |
| "loss": -1.0852, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -14.110136032104492, |
| "rewards/margins": 15.161088943481445, |
| "rewards/rejected": -29.271224975585938, |
| "step": 905 |
| }, |
| { |
| "epoch": 0.8088888888888889, |
| "grad_norm": 14.005874633789062, |
| "learning_rate": 8.323656910962011e-06, |
| "logits/chosen": -0.40306010842323303, |
| "logits/rejected": -0.44573473930358887, |
| "logps/chosen": -346.27105712890625, |
| "logps/rejected": -315.6506042480469, |
| "loss": -1.4107, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -13.054827690124512, |
| "rewards/margins": 15.541888236999512, |
| "rewards/rejected": -28.596715927124023, |
| "step": 910 |
| }, |
| { |
| "epoch": 0.8133333333333334, |
| "grad_norm": 25.924898147583008, |
| "learning_rate": 8.306209676727994e-06, |
| "logits/chosen": -0.3658706545829773, |
| "logits/rejected": -0.4349114000797272, |
| "logps/chosen": -358.9135437011719, |
| "logps/rejected": -326.36090087890625, |
| "loss": -1.4081, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -12.446220397949219, |
| "rewards/margins": 17.219030380249023, |
| "rewards/rejected": -29.66524887084961, |
| "step": 915 |
| }, |
| { |
| "epoch": 0.8177777777777778, |
| "grad_norm": 20.495826721191406, |
| "learning_rate": 8.288690611883296e-06, |
| "logits/chosen": -0.39841917157173157, |
| "logits/rejected": -0.4497374892234802, |
| "logps/chosen": -353.8162841796875, |
| "logps/rejected": -313.1631774902344, |
| "loss": -1.556, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -13.111363410949707, |
| "rewards/margins": 17.558149337768555, |
| "rewards/rejected": -30.669513702392578, |
| "step": 920 |
| }, |
| { |
| "epoch": 0.8222222222222222, |
| "grad_norm": 24.87085723876953, |
| "learning_rate": 8.271100097046585e-06, |
| "logits/chosen": -0.3760126233100891, |
| "logits/rejected": -0.42560848593711853, |
| "logps/chosen": -350.9206237792969, |
| "logps/rejected": -320.97637939453125, |
| "loss": -1.4347, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -13.494958877563477, |
| "rewards/margins": 16.944490432739258, |
| "rewards/rejected": -30.439449310302734, |
| "step": 925 |
| }, |
| { |
| "epoch": 0.8266666666666667, |
| "grad_norm": 20.46843910217285, |
| "learning_rate": 8.25343851438885e-06, |
| "logits/chosen": -0.4249737858772278, |
| "logits/rejected": -0.4788896441459656, |
| "logps/chosen": -361.0630187988281, |
| "logps/rejected": -319.73138427734375, |
| "loss": -1.3017, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -14.13292407989502, |
| "rewards/margins": 17.0448055267334, |
| "rewards/rejected": -31.1777286529541, |
| "step": 930 |
| }, |
| { |
| "epoch": 0.8311111111111111, |
| "grad_norm": 22.38373374938965, |
| "learning_rate": 8.235706247625098e-06, |
| "logits/chosen": -0.38224634528160095, |
| "logits/rejected": -0.4391182065010071, |
| "logps/chosen": -361.043701171875, |
| "logps/rejected": -325.9903564453125, |
| "loss": -1.2866, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -13.751859664916992, |
| "rewards/margins": 17.840734481811523, |
| "rewards/rejected": -31.59259605407715, |
| "step": 935 |
| }, |
| { |
| "epoch": 0.8355555555555556, |
| "grad_norm": 21.872596740722656, |
| "learning_rate": 8.217903682006017e-06, |
| "logits/chosen": -0.39942440390586853, |
| "logits/rejected": -0.45849889516830444, |
| "logps/chosen": -373.39013671875, |
| "logps/rejected": -337.3862609863281, |
| "loss": -1.6818, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -15.173101425170898, |
| "rewards/margins": 18.07097053527832, |
| "rewards/rejected": -33.24407196044922, |
| "step": 940 |
| }, |
| { |
| "epoch": 0.84, |
| "grad_norm": 21.81854248046875, |
| "learning_rate": 8.200031204309604e-06, |
| "logits/chosen": -0.40619197487831116, |
| "logits/rejected": -0.4568824768066406, |
| "logps/chosen": -342.34356689453125, |
| "logps/rejected": -317.7866516113281, |
| "loss": -1.6939, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -13.93403148651123, |
| "rewards/margins": 17.663101196289062, |
| "rewards/rejected": -31.597131729125977, |
| "step": 945 |
| }, |
| { |
| "epoch": 0.8444444444444444, |
| "grad_norm": 22.453853607177734, |
| "learning_rate": 8.182089202832767e-06, |
| "logits/chosen": -0.3882743716239929, |
| "logits/rejected": -0.4640750288963318, |
| "logps/chosen": -382.75787353515625, |
| "logps/rejected": -337.13995361328125, |
| "loss": -2.0499, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -14.779397964477539, |
| "rewards/margins": 20.105022430419922, |
| "rewards/rejected": -34.884422302246094, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.8488888888888889, |
| "grad_norm": 18.471513748168945, |
| "learning_rate": 8.16407806738288e-06, |
| "logits/chosen": -0.39945605397224426, |
| "logits/rejected": -0.4585798680782318, |
| "logps/chosen": -383.2483215332031, |
| "logps/rejected": -347.448974609375, |
| "loss": -1.2417, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -15.864675521850586, |
| "rewards/margins": 16.104869842529297, |
| "rewards/rejected": -31.969547271728516, |
| "step": 955 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 37.47999954223633, |
| "learning_rate": 8.145998189269327e-06, |
| "logits/chosen": -0.4188354015350342, |
| "logits/rejected": -0.4583558142185211, |
| "logps/chosen": -377.2878723144531, |
| "logps/rejected": -350.0664978027344, |
| "loss": -1.6863, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -18.30971336364746, |
| "rewards/margins": 19.272193908691406, |
| "rewards/rejected": -37.5819091796875, |
| "step": 960 |
| }, |
| { |
| "epoch": 0.8577777777777778, |
| "grad_norm": 19.40680503845215, |
| "learning_rate": 8.127849961294984e-06, |
| "logits/chosen": -0.4305190145969391, |
| "logits/rejected": -0.477532297372818, |
| "logps/chosen": -372.54443359375, |
| "logps/rejected": -340.7271728515625, |
| "loss": -1.6979, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -17.950782775878906, |
| "rewards/margins": 18.499141693115234, |
| "rewards/rejected": -36.44992446899414, |
| "step": 965 |
| }, |
| { |
| "epoch": 0.8622222222222222, |
| "grad_norm": 28.603803634643555, |
| "learning_rate": 8.109633777747703e-06, |
| "logits/chosen": -0.42268872261047363, |
| "logits/rejected": -0.4787193834781647, |
| "logps/chosen": -373.359619140625, |
| "logps/rejected": -347.9590148925781, |
| "loss": -1.3852, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -20.33183479309082, |
| "rewards/margins": 17.556093215942383, |
| "rewards/rejected": -37.88792419433594, |
| "step": 970 |
| }, |
| { |
| "epoch": 0.8666666666666667, |
| "grad_norm": 23.769119262695312, |
| "learning_rate": 8.091350034391732e-06, |
| "logits/chosen": -0.40240478515625, |
| "logits/rejected": -0.48416176438331604, |
| "logps/chosen": -384.06976318359375, |
| "logps/rejected": -361.1874694824219, |
| "loss": -1.5434, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -18.98853874206543, |
| "rewards/margins": 18.99036979675293, |
| "rewards/rejected": -37.97890853881836, |
| "step": 975 |
| }, |
| { |
| "epoch": 0.8711111111111111, |
| "grad_norm": 36.155155181884766, |
| "learning_rate": 8.072999128459119e-06, |
| "logits/chosen": -0.41284674406051636, |
| "logits/rejected": -0.4507782459259033, |
| "logps/chosen": -360.9227600097656, |
| "logps/rejected": -339.7681884765625, |
| "loss": -1.3714, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -18.811853408813477, |
| "rewards/margins": 17.97747230529785, |
| "rewards/rejected": -36.78932189941406, |
| "step": 980 |
| }, |
| { |
| "epoch": 0.8755555555555555, |
| "grad_norm": 21.57328224182129, |
| "learning_rate": 8.05458145864109e-06, |
| "logits/chosen": -0.39822930097579956, |
| "logits/rejected": -0.4551811218261719, |
| "logps/chosen": -353.82623291015625, |
| "logps/rejected": -345.6108093261719, |
| "loss": -1.2538, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -17.53032875061035, |
| "rewards/margins": 19.313552856445312, |
| "rewards/rejected": -36.8438835144043, |
| "step": 985 |
| }, |
| { |
| "epoch": 0.88, |
| "grad_norm": 39.19557571411133, |
| "learning_rate": 8.036097425079377e-06, |
| "logits/chosen": -0.38101926445961, |
| "logits/rejected": -0.4362686276435852, |
| "logps/chosen": -381.2861328125, |
| "logps/rejected": -345.8175048828125, |
| "loss": -1.2588, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -19.006837844848633, |
| "rewards/margins": 17.608064651489258, |
| "rewards/rejected": -36.61490249633789, |
| "step": 990 |
| }, |
| { |
| "epoch": 0.8844444444444445, |
| "grad_norm": 22.725879669189453, |
| "learning_rate": 8.017547429357532e-06, |
| "logits/chosen": -0.3905089497566223, |
| "logits/rejected": -0.44199681282043457, |
| "logps/chosen": -367.7683410644531, |
| "logps/rejected": -347.76812744140625, |
| "loss": -2.4106, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -15.675936698913574, |
| "rewards/margins": 22.89352798461914, |
| "rewards/rejected": -38.56946563720703, |
| "step": 995 |
| }, |
| { |
| "epoch": 0.8888888888888888, |
| "grad_norm": 20.353073120117188, |
| "learning_rate": 7.998931874492192e-06, |
| "logits/chosen": -0.37944620847702026, |
| "logits/rejected": -0.44008979201316833, |
| "logps/chosen": -352.2929382324219, |
| "logps/rejected": -333.12530517578125, |
| "loss": -1.4519, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -15.595646858215332, |
| "rewards/margins": 17.53089141845703, |
| "rewards/rejected": -33.12653732299805, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.8888888888888888, |
| "eval_logits/chosen": -0.39150622487068176, |
| "eval_logits/rejected": -0.45033136010169983, |
| "eval_logps/chosen": -367.9861755371094, |
| "eval_logps/rejected": -349.3917541503906, |
| "eval_loss": -1.812597393989563, |
| "eval_rewards/accuracies": 0.8402500152587891, |
| "eval_rewards/chosen": -17.389860153198242, |
| "eval_rewards/margins": 20.433107376098633, |
| "eval_rewards/rejected": -37.822967529296875, |
| "eval_runtime": 2196.225, |
| "eval_samples_per_second": 1.821, |
| "eval_steps_per_second": 0.911, |
| "step": 1000 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 3375, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
