| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 20.0, |
| "eval_steps": 500, |
| "global_step": 580, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.3448275862068966, |
| "grad_norm": 3.9107234477996826, |
| "learning_rate": 8.620689655172415e-07, |
| "logits/chosen": -2.363854169845581, |
| "logits/rejected": -2.338671922683716, |
| "logps/chosen": -76.8819351196289, |
| "logps/rejected": -75.86869049072266, |
| "loss": 0.6942, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.003996999468654394, |
| "rewards/margins": -0.004145228303968906, |
| "rewards/rejected": 0.00014822949015069753, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.6896551724137931, |
| "grad_norm": 4.328212738037109, |
| "learning_rate": 1.724137931034483e-06, |
| "logits/chosen": -2.3617663383483887, |
| "logits/rejected": -2.3477540016174316, |
| "logps/chosen": -93.76543426513672, |
| "logps/rejected": -75.16656494140625, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.48750001192092896, |
| "rewards/chosen": 0.002653368515893817, |
| "rewards/margins": 0.006352785974740982, |
| "rewards/rejected": -0.003699416993185878, |
| "step": 20 |
| }, |
| { |
| "epoch": 1.0344827586206897, |
| "grad_norm": 3.473827838897705, |
| "learning_rate": 2.5862068965517246e-06, |
| "logits/chosen": -2.3354058265686035, |
| "logits/rejected": -2.3330740928649902, |
| "logps/chosen": -138.9960479736328, |
| "logps/rejected": -67.00438690185547, |
| "loss": 0.6927, |
| "rewards/accuracies": 0.44999998807907104, |
| "rewards/chosen": 0.01745142787694931, |
| "rewards/margins": 0.013760591857135296, |
| "rewards/rejected": 0.0036908381152898073, |
| "step": 30 |
| }, |
| { |
| "epoch": 1.3793103448275863, |
| "grad_norm": 3.9458820819854736, |
| "learning_rate": 3.448275862068966e-06, |
| "logits/chosen": -2.3478751182556152, |
| "logits/rejected": -2.3700289726257324, |
| "logps/chosen": -75.59983825683594, |
| "logps/rejected": -67.20357513427734, |
| "loss": 0.69, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.00024204826331697404, |
| "rewards/margins": 0.005866119172424078, |
| "rewards/rejected": -0.005624071694910526, |
| "step": 40 |
| }, |
| { |
| "epoch": 1.7241379310344827, |
| "grad_norm": 4.299890041351318, |
| "learning_rate": 4.310344827586207e-06, |
| "logits/chosen": -2.338815212249756, |
| "logits/rejected": -2.3532769680023193, |
| "logps/chosen": -72.74342346191406, |
| "logps/rejected": -87.63409423828125, |
| "loss": 0.6899, |
| "rewards/accuracies": 0.574999988079071, |
| "rewards/chosen": -0.003249581903219223, |
| "rewards/margins": 0.006888723932206631, |
| "rewards/rejected": -0.010138307698071003, |
| "step": 50 |
| }, |
| { |
| "epoch": 2.0689655172413794, |
| "grad_norm": 4.957555294036865, |
| "learning_rate": 4.999818897894192e-06, |
| "logits/chosen": -2.344572067260742, |
| "logits/rejected": -2.345888614654541, |
| "logps/chosen": -80.16046142578125, |
| "logps/rejected": -73.05986022949219, |
| "loss": 0.6802, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.013288321904838085, |
| "rewards/margins": 0.020968889817595482, |
| "rewards/rejected": -0.03425721079111099, |
| "step": 60 |
| }, |
| { |
| "epoch": 2.413793103448276, |
| "grad_norm": 4.212078094482422, |
| "learning_rate": 4.9934830787948756e-06, |
| "logits/chosen": -2.3330090045928955, |
| "logits/rejected": -2.328575372695923, |
| "logps/chosen": -73.63365173339844, |
| "logps/rejected": -74.45356750488281, |
| "loss": 0.6693, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.009874681942164898, |
| "rewards/margins": 0.04785541445016861, |
| "rewards/rejected": -0.05773010104894638, |
| "step": 70 |
| }, |
| { |
| "epoch": 2.7586206896551726, |
| "grad_norm": 4.446742534637451, |
| "learning_rate": 4.978118375700895e-06, |
| "logits/chosen": -2.3472495079040527, |
| "logits/rejected": -2.3697409629821777, |
| "logps/chosen": -73.44490814208984, |
| "logps/rejected": -89.69837951660156, |
| "loss": 0.6553, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.022817375138401985, |
| "rewards/margins": 0.09045850485563278, |
| "rewards/rejected": -0.11327588558197021, |
| "step": 80 |
| }, |
| { |
| "epoch": 3.103448275862069, |
| "grad_norm": 4.435715198516846, |
| "learning_rate": 4.953780424089803e-06, |
| "logits/chosen": -2.346717596054077, |
| "logits/rejected": -2.3626608848571777, |
| "logps/chosen": -85.64796447753906, |
| "logps/rejected": -80.06268310546875, |
| "loss": 0.6432, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.0543200746178627, |
| "rewards/margins": 0.1085677295923233, |
| "rewards/rejected": -0.1628878116607666, |
| "step": 90 |
| }, |
| { |
| "epoch": 3.4482758620689653, |
| "grad_norm": 4.5200910568237305, |
| "learning_rate": 4.920557351506409e-06, |
| "logits/chosen": -2.358900547027588, |
| "logits/rejected": -2.3591980934143066, |
| "logps/chosen": -82.83473205566406, |
| "logps/rejected": -80.66204071044922, |
| "loss": 0.5996, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.046993009746074677, |
| "rewards/margins": 0.20401433110237122, |
| "rewards/rejected": -0.2510073184967041, |
| "step": 100 |
| }, |
| { |
| "epoch": 3.793103448275862, |
| "grad_norm": 4.106562614440918, |
| "learning_rate": 4.878569458453592e-06, |
| "logits/chosen": -2.347762107849121, |
| "logits/rejected": -2.33540415763855, |
| "logps/chosen": -72.23506164550781, |
| "logps/rejected": -85.77088928222656, |
| "loss": 0.6003, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -0.051100969314575195, |
| "rewards/margins": 0.24443332850933075, |
| "rewards/rejected": -0.29553431272506714, |
| "step": 110 |
| }, |
| { |
| "epoch": 4.137931034482759, |
| "grad_norm": 4.67632532119751, |
| "learning_rate": 4.827968782785062e-06, |
| "logits/chosen": -2.353370428085327, |
| "logits/rejected": -2.378964900970459, |
| "logps/chosen": -67.91919708251953, |
| "logps/rejected": -86.26390838623047, |
| "loss": 0.5716, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -0.11952020972967148, |
| "rewards/margins": 0.30119410157203674, |
| "rewards/rejected": -0.42071428894996643, |
| "step": 120 |
| }, |
| { |
| "epoch": 4.482758620689655, |
| "grad_norm": 4.5131025314331055, |
| "learning_rate": 4.7689385491773934e-06, |
| "logits/chosen": -2.3820648193359375, |
| "logits/rejected": -2.403965473175049, |
| "logps/chosen": -71.66841125488281, |
| "logps/rejected": -77.90869140625, |
| "loss": 0.5362, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.0694696456193924, |
| "rewards/margins": 0.47986412048339844, |
| "rewards/rejected": -0.5493337512016296, |
| "step": 130 |
| }, |
| { |
| "epoch": 4.827586206896552, |
| "grad_norm": 4.446605205535889, |
| "learning_rate": 4.70169250567482e-06, |
| "logits/chosen": -2.372788906097412, |
| "logits/rejected": -2.379612684249878, |
| "logps/chosen": -69.44779968261719, |
| "logps/rejected": -80.1120376586914, |
| "loss": 0.5186, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.12896183133125305, |
| "rewards/margins": 0.5137210488319397, |
| "rewards/rejected": -0.6426829099655151, |
| "step": 140 |
| }, |
| { |
| "epoch": 5.172413793103448, |
| "grad_norm": 4.605374813079834, |
| "learning_rate": 4.626474149709127e-06, |
| "logits/chosen": -2.3140041828155518, |
| "logits/rejected": -2.336975574493408, |
| "logps/chosen": -87.05641174316406, |
| "logps/rejected": -94.53668212890625, |
| "loss": 0.4794, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -0.15513856709003448, |
| "rewards/margins": 0.6417296528816223, |
| "rewards/rejected": -0.7968682050704956, |
| "step": 150 |
| }, |
| { |
| "epoch": 5.517241379310345, |
| "grad_norm": 6.027809143066406, |
| "learning_rate": 4.54355584639723e-06, |
| "logits/chosen": -2.365192413330078, |
| "logits/rejected": -2.3872745037078857, |
| "logps/chosen": -73.7781753540039, |
| "logps/rejected": -82.1873550415039, |
| "loss": 0.4599, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.25834327936172485, |
| "rewards/margins": 0.5800828337669373, |
| "rewards/rejected": -0.8384261131286621, |
| "step": 160 |
| }, |
| { |
| "epoch": 5.862068965517241, |
| "grad_norm": 5.196141242980957, |
| "learning_rate": 4.45323784230908e-06, |
| "logits/chosen": -2.391653299331665, |
| "logits/rejected": -2.394854784011841, |
| "logps/chosen": -72.33963775634766, |
| "logps/rejected": -86.77699279785156, |
| "loss": 0.4467, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.2543284595012665, |
| "rewards/margins": 0.7806032299995422, |
| "rewards/rejected": -1.0349315404891968, |
| "step": 170 |
| }, |
| { |
| "epoch": 6.206896551724138, |
| "grad_norm": 5.285089015960693, |
| "learning_rate": 4.355847178277025e-06, |
| "logits/chosen": -2.3918795585632324, |
| "logits/rejected": -2.4050662517547607, |
| "logps/chosen": -77.20832061767578, |
| "logps/rejected": -98.5592041015625, |
| "loss": 0.4052, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.35020238161087036, |
| "rewards/margins": 1.0169063806533813, |
| "rewards/rejected": -1.367108702659607, |
| "step": 180 |
| }, |
| { |
| "epoch": 6.551724137931035, |
| "grad_norm": 5.308840751647949, |
| "learning_rate": 4.2517365051833564e-06, |
| "logits/chosen": -2.3778843879699707, |
| "logits/rejected": -2.3865675926208496, |
| "logps/chosen": -69.90904235839844, |
| "logps/rejected": -86.66047668457031, |
| "loss": 0.3809, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.42647725343704224, |
| "rewards/margins": 0.9949015378952026, |
| "rewards/rejected": -1.4213788509368896, |
| "step": 190 |
| }, |
| { |
| "epoch": 6.896551724137931, |
| "grad_norm": 6.095545768737793, |
| "learning_rate": 4.141282807014034e-06, |
| "logits/chosen": -2.407437801361084, |
| "logits/rejected": -2.4296727180480957, |
| "logps/chosen": -82.95701599121094, |
| "logps/rejected": -83.4615478515625, |
| "loss": 0.3742, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.6070786714553833, |
| "rewards/margins": 0.968428909778595, |
| "rewards/rejected": -1.5755075216293335, |
| "step": 200 |
| }, |
| { |
| "epoch": 7.241379310344827, |
| "grad_norm": 5.683840751647949, |
| "learning_rate": 4.024886035802432e-06, |
| "logits/chosen": -2.411830186843872, |
| "logits/rejected": -2.4354729652404785, |
| "logps/chosen": -61.66484451293945, |
| "logps/rejected": -90.95805358886719, |
| "loss": 0.3421, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.5454065203666687, |
| "rewards/margins": 1.2011836767196655, |
| "rewards/rejected": -1.7465900182724, |
| "step": 210 |
| }, |
| { |
| "epoch": 7.586206896551724, |
| "grad_norm": 5.584890842437744, |
| "learning_rate": 3.9029676634059565e-06, |
| "logits/chosen": -2.3816287517547607, |
| "logits/rejected": -2.37958025932312, |
| "logps/chosen": -87.12813568115234, |
| "logps/rejected": -92.26463317871094, |
| "loss": 0.3138, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.5636588335037231, |
| "rewards/margins": 1.3944618701934814, |
| "rewards/rejected": -1.9581207036972046, |
| "step": 220 |
| }, |
| { |
| "epoch": 7.931034482758621, |
| "grad_norm": 7.526435375213623, |
| "learning_rate": 3.7759691553595214e-06, |
| "logits/chosen": -2.399634599685669, |
| "logits/rejected": -2.407938241958618, |
| "logps/chosen": -82.55473327636719, |
| "logps/rejected": -89.76951599121094, |
| "loss": 0.3046, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.9372364282608032, |
| "rewards/margins": 1.309622049331665, |
| "rewards/rejected": -2.246858596801758, |
| "step": 230 |
| }, |
| { |
| "epoch": 8.275862068965518, |
| "grad_norm": 5.150575637817383, |
| "learning_rate": 3.6443503723320837e-06, |
| "logits/chosen": -2.36824893951416, |
| "logits/rejected": -2.3758621215820312, |
| "logps/chosen": -85.68852996826172, |
| "logps/rejected": -102.09485626220703, |
| "loss": 0.2638, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.9749407768249512, |
| "rewards/margins": 1.6947702169418335, |
| "rewards/rejected": -2.669710874557495, |
| "step": 240 |
| }, |
| { |
| "epoch": 8.620689655172415, |
| "grad_norm": 6.282366752624512, |
| "learning_rate": 3.508587904974522e-06, |
| "logits/chosen": -2.3659470081329346, |
| "logits/rejected": -2.367462635040283, |
| "logps/chosen": -87.00023651123047, |
| "logps/rejected": -123.0126953125, |
| "loss": 0.2405, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.026289939880371, |
| "rewards/margins": 1.9990599155426025, |
| "rewards/rejected": -3.0253496170043945, |
| "step": 250 |
| }, |
| { |
| "epoch": 8.96551724137931, |
| "grad_norm": 6.396641731262207, |
| "learning_rate": 3.3691733481883693e-06, |
| "logits/chosen": -2.3716306686401367, |
| "logits/rejected": -2.3546042442321777, |
| "logps/chosen": -87.29267883300781, |
| "logps/rejected": -98.01856994628906, |
| "loss": 0.2546, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.2485218048095703, |
| "rewards/margins": 1.980831503868103, |
| "rewards/rejected": -3.229353427886963, |
| "step": 260 |
| }, |
| { |
| "epoch": 9.310344827586206, |
| "grad_norm": 5.644590377807617, |
| "learning_rate": 3.226611521064278e-06, |
| "logits/chosen": -2.2916407585144043, |
| "logits/rejected": -2.3004841804504395, |
| "logps/chosen": -79.30825805664062, |
| "logps/rejected": -113.4272689819336, |
| "loss": 0.1999, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -1.1944876909255981, |
| "rewards/margins": 2.365144729614258, |
| "rewards/rejected": -3.5596320629119873, |
| "step": 270 |
| }, |
| { |
| "epoch": 9.655172413793103, |
| "grad_norm": 6.177416801452637, |
| "learning_rate": 3.0814186389357765e-06, |
| "logits/chosen": -2.346484661102295, |
| "logits/rejected": -2.346285581588745, |
| "logps/chosen": -77.01927947998047, |
| "logps/rejected": -105.2061538696289, |
| "loss": 0.1875, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -1.260752558708191, |
| "rewards/margins": 2.249293565750122, |
| "rewards/rejected": -3.5100464820861816, |
| "step": 280 |
| }, |
| { |
| "epoch": 10.0, |
| "grad_norm": 6.281085014343262, |
| "learning_rate": 2.9341204441673267e-06, |
| "logits/chosen": -2.3005917072296143, |
| "logits/rejected": -2.3319716453552246, |
| "logps/chosen": -92.97987365722656, |
| "logps/rejected": -112.0716552734375, |
| "loss": 0.2036, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -1.6851022243499756, |
| "rewards/margins": 2.1508073806762695, |
| "rewards/rejected": -3.835909605026245, |
| "step": 290 |
| }, |
| { |
| "epoch": 10.344827586206897, |
| "grad_norm": 5.364060401916504, |
| "learning_rate": 2.785250302445062e-06, |
| "logits/chosen": -2.3305928707122803, |
| "logits/rejected": -2.3451201915740967, |
| "logps/chosen": -85.11759185791016, |
| "logps/rejected": -115.65193176269531, |
| "loss": 0.1664, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -1.5494135618209839, |
| "rewards/margins": 2.6376235485076904, |
| "rewards/rejected": -4.187037467956543, |
| "step": 300 |
| }, |
| { |
| "epoch": 10.689655172413794, |
| "grad_norm": 5.540137767791748, |
| "learning_rate": 2.6353472714635443e-06, |
| "logits/chosen": -2.3217194080352783, |
| "logits/rejected": -2.345512628555298, |
| "logps/chosen": -83.56015014648438, |
| "logps/rejected": -111.0335922241211, |
| "loss": 0.1446, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -1.6171716451644897, |
| "rewards/margins": 2.553410053253174, |
| "rewards/rejected": -4.170581340789795, |
| "step": 310 |
| }, |
| { |
| "epoch": 11.03448275862069, |
| "grad_norm": 5.321618556976318, |
| "learning_rate": 2.4849541490017868e-06, |
| "logits/chosen": -2.2711265087127686, |
| "logits/rejected": -2.274831533432007, |
| "logps/chosen": -93.46397399902344, |
| "logps/rejected": -138.03286743164062, |
| "loss": 0.1416, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -2.0557093620300293, |
| "rewards/margins": 2.809981346130371, |
| "rewards/rejected": -4.8656907081604, |
| "step": 320 |
| }, |
| { |
| "epoch": 11.379310344827585, |
| "grad_norm": 6.168019771575928, |
| "learning_rate": 2.3346155074564712e-06, |
| "logits/chosen": -2.2561862468719482, |
| "logits/rejected": -2.260565996170044, |
| "logps/chosen": -89.53536224365234, |
| "logps/rejected": -119.193603515625, |
| "loss": 0.1174, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.2866904735565186, |
| "rewards/margins": 2.866985321044922, |
| "rewards/rejected": -5.1536760330200195, |
| "step": 330 |
| }, |
| { |
| "epoch": 11.724137931034482, |
| "grad_norm": 4.970952033996582, |
| "learning_rate": 2.184875721949277e-06, |
| "logits/chosen": -2.204287052154541, |
| "logits/rejected": -2.2466628551483154, |
| "logps/chosen": -171.76095581054688, |
| "logps/rejected": -147.26907348632812, |
| "loss": 0.1152, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.4709744453430176, |
| "rewards/margins": 3.782508134841919, |
| "rewards/rejected": -6.253481864929199, |
| "step": 340 |
| }, |
| { |
| "epoch": 12.068965517241379, |
| "grad_norm": 4.031501770019531, |
| "learning_rate": 2.0362769991485514e-06, |
| "logits/chosen": -2.245434522628784, |
| "logits/rejected": -2.267470121383667, |
| "logps/chosen": -98.18807220458984, |
| "logps/rejected": -143.83383178710938, |
| "loss": 0.1104, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.5792040824890137, |
| "rewards/margins": 3.238723039627075, |
| "rewards/rejected": -5.817927360534668, |
| "step": 350 |
| }, |
| { |
| "epoch": 12.413793103448276, |
| "grad_norm": 4.009598731994629, |
| "learning_rate": 1.8893574139429226e-06, |
| "logits/chosen": -2.2339296340942383, |
| "logits/rejected": -2.254106283187866, |
| "logps/chosen": -104.34129333496094, |
| "logps/rejected": -130.74081420898438, |
| "loss": 0.0859, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -2.8974556922912598, |
| "rewards/margins": 3.275568723678589, |
| "rewards/rejected": -6.1730241775512695, |
| "step": 360 |
| }, |
| { |
| "epoch": 12.758620689655173, |
| "grad_norm": 6.944422721862793, |
| "learning_rate": 1.744648961076068e-06, |
| "logits/chosen": -2.252500295639038, |
| "logits/rejected": -2.25302791595459, |
| "logps/chosen": -97.11503601074219, |
| "logps/rejected": -135.61911010742188, |
| "loss": 0.0901, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.814434051513672, |
| "rewards/margins": 3.549976348876953, |
| "rewards/rejected": -6.364409923553467, |
| "step": 370 |
| }, |
| { |
| "epoch": 13.10344827586207, |
| "grad_norm": 5.268442630767822, |
| "learning_rate": 1.602675628797636e-06, |
| "logits/chosen": -2.2403993606567383, |
| "logits/rejected": -2.240546703338623, |
| "logps/chosen": -101.62408447265625, |
| "logps/rejected": -136.84268188476562, |
| "loss": 0.0902, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -2.9260125160217285, |
| "rewards/margins": 3.421342134475708, |
| "rewards/rejected": -6.347354888916016, |
| "step": 380 |
| }, |
| { |
| "epoch": 13.448275862068966, |
| "grad_norm": 4.1959404945373535, |
| "learning_rate": 1.4639515015056205e-06, |
| "logits/chosen": -2.2759053707122803, |
| "logits/rejected": -2.2657554149627686, |
| "logps/chosen": -103.3962631225586, |
| "logps/rejected": -138.56057739257812, |
| "loss": 0.0729, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.688096523284912, |
| "rewards/margins": 4.030136585235596, |
| "rewards/rejected": -6.718233585357666, |
| "step": 390 |
| }, |
| { |
| "epoch": 13.793103448275861, |
| "grad_norm": 4.1255574226379395, |
| "learning_rate": 1.328978898250525e-06, |
| "logits/chosen": -2.180371046066284, |
| "logits/rejected": -2.213881015777588, |
| "logps/chosen": -100.98771667480469, |
| "logps/rejected": -135.07044982910156, |
| "loss": 0.0747, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.0276432037353516, |
| "rewards/margins": 3.8850128650665283, |
| "rewards/rejected": -6.912655830383301, |
| "step": 400 |
| }, |
| { |
| "epoch": 14.137931034482758, |
| "grad_norm": 5.081278324127197, |
| "learning_rate": 1.198246553841744e-06, |
| "logits/chosen": -2.237619161605835, |
| "logits/rejected": -2.219052314758301, |
| "logps/chosen": -98.90690612792969, |
| "logps/rejected": -131.66647338867188, |
| "loss": 0.0576, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.216820478439331, |
| "rewards/margins": 3.5623440742492676, |
| "rewards/rejected": -6.779164791107178, |
| "step": 410 |
| }, |
| { |
| "epoch": 14.482758620689655, |
| "grad_norm": 4.580889701843262, |
| "learning_rate": 1.0722278491423998e-06, |
| "logits/chosen": -2.1594581604003906, |
| "logits/rejected": -2.183774471282959, |
| "logps/chosen": -106.17756652832031, |
| "logps/rejected": -163.53775024414062, |
| "loss": 0.0545, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.6679279804229736, |
| "rewards/margins": 4.358443260192871, |
| "rewards/rejected": -8.026371955871582, |
| "step": 420 |
| }, |
| { |
| "epoch": 14.827586206896552, |
| "grad_norm": 7.627413749694824, |
| "learning_rate": 9.513790969606926e-07, |
| "logits/chosen": -2.1865813732147217, |
| "logits/rejected": -2.2028133869171143, |
| "logps/chosen": -109.03575134277344, |
| "logps/rejected": -147.6363983154297, |
| "loss": 0.0552, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.6653411388397217, |
| "rewards/margins": 3.91802978515625, |
| "rewards/rejected": -7.583371162414551, |
| "step": 430 |
| }, |
| { |
| "epoch": 15.172413793103448, |
| "grad_norm": 4.102783679962158, |
| "learning_rate": 8.361378897445643e-07, |
| "logits/chosen": -2.18919038772583, |
| "logits/rejected": -2.2041501998901367, |
| "logps/chosen": -110.42228698730469, |
| "logps/rejected": -157.49429321289062, |
| "loss": 0.0596, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -3.86995005607605, |
| "rewards/margins": 4.187633037567139, |
| "rewards/rejected": -8.057581901550293, |
| "step": 440 |
| }, |
| { |
| "epoch": 15.517241379310345, |
| "grad_norm": 4.329521656036377, |
| "learning_rate": 7.269215150626391e-07, |
| "logits/chosen": -2.1742970943450928, |
| "logits/rejected": -2.1848294734954834, |
| "logps/chosen": -107.87260437011719, |
| "logps/rejected": -150.42959594726562, |
| "loss": 0.047, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.9351372718811035, |
| "rewards/margins": 4.330621242523193, |
| "rewards/rejected": -8.26575756072998, |
| "step": 450 |
| }, |
| { |
| "epoch": 15.862068965517242, |
| "grad_norm": 4.850955963134766, |
| "learning_rate": 6.241254446089942e-07, |
| "logits/chosen": -2.2147865295410156, |
| "logits/rejected": -2.2335610389709473, |
| "logps/chosen": -105.22979736328125, |
| "logps/rejected": -149.566162109375, |
| "loss": 0.0443, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.2662596702575684, |
| "rewards/margins": 4.226635932922363, |
| "rewards/rejected": -7.49289608001709, |
| "step": 460 |
| }, |
| { |
| "epoch": 16.20689655172414, |
| "grad_norm": 3.0453879833221436, |
| "learning_rate": 5.281219022030423e-07, |
| "logits/chosen": -2.158506155014038, |
| "logits/rejected": -2.1904215812683105, |
| "logps/chosen": -119.72633361816406, |
| "logps/rejected": -159.1144561767578, |
| "loss": 0.0409, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -3.4267520904541016, |
| "rewards/margins": 4.899367332458496, |
| "rewards/rejected": -8.326119422912598, |
| "step": 470 |
| }, |
| { |
| "epoch": 16.551724137931036, |
| "grad_norm": 3.7533819675445557, |
| "learning_rate": 4.392585159698087e-07, |
| "logits/chosen": -2.152306318283081, |
| "logits/rejected": -2.1774165630340576, |
| "logps/chosen": -112.7719955444336, |
| "logps/rejected": -165.19509887695312, |
| "loss": 0.0416, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.251755714416504, |
| "rewards/margins": 4.445396900177002, |
| "rewards/rejected": -8.697153091430664, |
| "step": 480 |
| }, |
| { |
| "epoch": 16.896551724137932, |
| "grad_norm": 7.166266918182373, |
| "learning_rate": 3.578570595810274e-07, |
| "logits/chosen": -2.1818134784698486, |
| "logits/rejected": -2.2025341987609863, |
| "logps/chosen": -96.93048095703125, |
| "logps/rejected": -171.3131103515625, |
| "loss": 0.0417, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.020156383514404, |
| "rewards/margins": 5.221116542816162, |
| "rewards/rejected": -9.241272926330566, |
| "step": 490 |
| }, |
| { |
| "epoch": 17.24137931034483, |
| "grad_norm": 3.491783618927002, |
| "learning_rate": 2.8421228711503127e-07, |
| "logits/chosen": -2.1831130981445312, |
| "logits/rejected": -2.1822221279144287, |
| "logps/chosen": -113.05125427246094, |
| "logps/rejected": -171.09835815429688, |
| "loss": 0.0369, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.272199630737305, |
| "rewards/margins": 4.791441440582275, |
| "rewards/rejected": -9.063641548156738, |
| "step": 500 |
| }, |
| { |
| "epoch": 17.586206896551722, |
| "grad_norm": 2.9002482891082764, |
| "learning_rate": 2.1859086575439225e-07, |
| "logits/chosen": -2.211790084838867, |
| "logits/rejected": -2.1946873664855957, |
| "logps/chosen": -129.7812042236328, |
| "logps/rejected": -168.2603302001953, |
| "loss": 0.0359, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.544430732727051, |
| "rewards/margins": 4.710877418518066, |
| "rewards/rejected": -9.2553071975708, |
| "step": 510 |
| }, |
| { |
| "epoch": 17.93103448275862, |
| "grad_norm": 3.7184576988220215, |
| "learning_rate": 1.6123041018599766e-07, |
| "logits/chosen": -2.1540164947509766, |
| "logits/rejected": -2.147338390350342, |
| "logps/chosen": -117.89579772949219, |
| "logps/rejected": -158.124755859375, |
| "loss": 0.0413, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.149048328399658, |
| "rewards/margins": 4.793323516845703, |
| "rewards/rejected": -8.94237232208252, |
| "step": 520 |
| }, |
| { |
| "epoch": 18.275862068965516, |
| "grad_norm": 3.723693370819092, |
| "learning_rate": 1.1233862220001168e-07, |
| "logits/chosen": -2.169210195541382, |
| "logits/rejected": -2.184584140777588, |
| "logps/chosen": -107.45458984375, |
| "logps/rejected": -161.4453582763672, |
| "loss": 0.0356, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -3.943101167678833, |
| "rewards/margins": 5.099206447601318, |
| "rewards/rejected": -9.04230785369873, |
| "step": 530 |
| }, |
| { |
| "epoch": 18.620689655172413, |
| "grad_norm": 2.663479804992676, |
| "learning_rate": 7.209253860320897e-08, |
| "logits/chosen": -2.141265392303467, |
| "logits/rejected": -2.1849865913391113, |
| "logps/chosen": -155.58615112304688, |
| "logps/rejected": -161.23976135253906, |
| "loss": 0.0346, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -2.961081027984619, |
| "rewards/margins": 6.033589839935303, |
| "rewards/rejected": -8.994670867919922, |
| "step": 540 |
| }, |
| { |
| "epoch": 18.96551724137931, |
| "grad_norm": 3.3965961933135986, |
| "learning_rate": 4.063789016999331e-08, |
| "logits/chosen": -2.2001423835754395, |
| "logits/rejected": -2.214458703994751, |
| "logps/chosen": -128.25711059570312, |
| "logps/rejected": -166.47296142578125, |
| "loss": 0.0394, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.388108730316162, |
| "rewards/margins": 4.831867694854736, |
| "rewards/rejected": -9.219977378845215, |
| "step": 550 |
| }, |
| { |
| "epoch": 19.310344827586206, |
| "grad_norm": 3.214250326156616, |
| "learning_rate": 1.808857395232788e-08, |
| "logits/chosen": -2.147885799407959, |
| "logits/rejected": -2.1538643836975098, |
| "logps/chosen": -119.9779281616211, |
| "logps/rejected": -163.17922973632812, |
| "loss": 0.0352, |
| "rewards/accuracies": 0.987500011920929, |
| "rewards/chosen": -4.499241828918457, |
| "rewards/margins": 4.586289882659912, |
| "rewards/rejected": -9.085531234741211, |
| "step": 560 |
| }, |
| { |
| "epoch": 19.655172413793103, |
| "grad_norm": 3.0123510360717773, |
| "learning_rate": 4.526240859345499e-09, |
| "logits/chosen": -2.1862876415252686, |
| "logits/rejected": -2.1756527423858643, |
| "logps/chosen": -135.414306640625, |
| "logps/rejected": -169.79623413085938, |
| "loss": 0.0344, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.599732398986816, |
| "rewards/margins": 4.528085231781006, |
| "rewards/rejected": -9.12781810760498, |
| "step": 570 |
| }, |
| { |
| "epoch": 20.0, |
| "grad_norm": 2.647522449493408, |
| "learning_rate": 0.0, |
| "logits/chosen": -2.175386428833008, |
| "logits/rejected": -2.183178424835205, |
| "logps/chosen": -106.87957763671875, |
| "logps/rejected": -163.1853790283203, |
| "loss": 0.036, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -4.439215183258057, |
| "rewards/margins": 4.813824653625488, |
| "rewards/rejected": -9.253039360046387, |
| "step": 580 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 580, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 20, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 2.1130760508114207e+18, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
