| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.99860529986053, | |
| "eval_steps": 500, | |
| "global_step": 358, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01394700139470014, | |
| "grad_norm": 8.443673133850098, | |
| "learning_rate": 3.1067090740782183e-06, | |
| "logits/chosen": 13.965856552124023, | |
| "logits/rejected": 15.354721069335938, | |
| "logps/chosen": -263.2093200683594, | |
| "logps/rejected": -286.745361328125, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.3083333373069763, | |
| "rewards/chosen": 0.001035079127177596, | |
| "rewards/margins": 0.004346395842730999, | |
| "rewards/rejected": -0.003311316715553403, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.02789400278940028, | |
| "grad_norm": 9.05903434753418, | |
| "learning_rate": 6.99009541667599e-06, | |
| "logits/chosen": 13.079957962036133, | |
| "logits/rejected": 15.2965726852417, | |
| "logps/chosen": -246.61495971679688, | |
| "logps/rejected": -292.19561767578125, | |
| "loss": 0.6512, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.004167428705841303, | |
| "rewards/margins": 0.10341107845306396, | |
| "rewards/rejected": -0.10757851600646973, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.04184100418410042, | |
| "grad_norm": 7.4054036140441895, | |
| "learning_rate": 1.0873481759273766e-05, | |
| "logits/chosen": 13.588569641113281, | |
| "logits/rejected": 14.664776802062988, | |
| "logps/chosen": -250.9985809326172, | |
| "logps/rejected": -271.63397216796875, | |
| "loss": 0.6306, | |
| "rewards/accuracies": 0.6750000715255737, | |
| "rewards/chosen": -0.07446910440921783, | |
| "rewards/margins": 0.2632213234901428, | |
| "rewards/rejected": -0.33769041299819946, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.05578800557880056, | |
| "grad_norm": 7.973468780517578, | |
| "learning_rate": 1.4756868101871536e-05, | |
| "logits/chosen": 10.776374816894531, | |
| "logits/rejected": 12.654977798461914, | |
| "logps/chosen": -219.0859375, | |
| "logps/rejected": -279.8912048339844, | |
| "loss": 0.5803, | |
| "rewards/accuracies": 0.7083333730697632, | |
| "rewards/chosen": -0.15387776494026184, | |
| "rewards/margins": 0.5580573678016663, | |
| "rewards/rejected": -0.7119351625442505, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.0697350069735007, | |
| "grad_norm": 8.493500709533691, | |
| "learning_rate": 1.864025444446931e-05, | |
| "logits/chosen": 11.028432846069336, | |
| "logits/rejected": 12.664867401123047, | |
| "logps/chosen": -269.8415832519531, | |
| "logps/rejected": -324.97613525390625, | |
| "loss": 0.6957, | |
| "rewards/accuracies": 0.6833333969116211, | |
| "rewards/chosen": -0.31670600175857544, | |
| "rewards/margins": 0.7376400232315063, | |
| "rewards/rejected": -1.0543458461761475, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.08368200836820083, | |
| "grad_norm": 6.57462739944458, | |
| "learning_rate": 2.2523640787067085e-05, | |
| "logits/chosen": 12.154653549194336, | |
| "logits/rejected": 13.542457580566406, | |
| "logps/chosen": -250.53341674804688, | |
| "logps/rejected": -290.4168395996094, | |
| "loss": 0.6559, | |
| "rewards/accuracies": 0.6750000715255737, | |
| "rewards/chosen": -0.11686629056930542, | |
| "rewards/margins": 0.5811147093772888, | |
| "rewards/rejected": -0.6979809999465942, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.09762900976290098, | |
| "grad_norm": 7.757661819458008, | |
| "learning_rate": 2.6407027129664858e-05, | |
| "logits/chosen": 11.735052108764648, | |
| "logits/rejected": 12.871235847473145, | |
| "logps/chosen": -196.1747589111328, | |
| "logps/rejected": -223.5957794189453, | |
| "loss": 0.6182, | |
| "rewards/accuracies": 0.6333333253860474, | |
| "rewards/chosen": -0.00016996636986732483, | |
| "rewards/margins": 0.3299906551837921, | |
| "rewards/rejected": -0.33016061782836914, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.11157601115760112, | |
| "grad_norm": 9.74497127532959, | |
| "learning_rate": 2.7182958819900885e-05, | |
| "logits/chosen": 10.667373657226562, | |
| "logits/rejected": 12.847865104675293, | |
| "logps/chosen": -208.2180633544922, | |
| "logps/rejected": -302.39971923828125, | |
| "loss": 0.5985, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.03828881308436394, | |
| "rewards/margins": 0.43971744179725647, | |
| "rewards/rejected": -0.4780062735080719, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.12552301255230125, | |
| "grad_norm": 9.502607345581055, | |
| "learning_rate": 2.7179930095042222e-05, | |
| "logits/chosen": 11.666415214538574, | |
| "logits/rejected": 13.399192810058594, | |
| "logps/chosen": -252.95516967773438, | |
| "logps/rejected": -297.4845275878906, | |
| "loss": 0.6683, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.2012624740600586, | |
| "rewards/margins": 0.4417993426322937, | |
| "rewards/rejected": -0.6430618166923523, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.1394700139470014, | |
| "grad_norm": 6.734437942504883, | |
| "learning_rate": 2.717457231667877e-05, | |
| "logits/chosen": 11.400947570800781, | |
| "logits/rejected": 12.905207633972168, | |
| "logps/chosen": -199.80596923828125, | |
| "logps/rejected": -254.4221954345703, | |
| "loss": 0.5718, | |
| "rewards/accuracies": 0.6416667103767395, | |
| "rewards/chosen": -0.022540345788002014, | |
| "rewards/margins": 0.5749102830886841, | |
| "rewards/rejected": -0.5974506139755249, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.15341701534170155, | |
| "grad_norm": 6.349542617797852, | |
| "learning_rate": 2.7166886709384802e-05, | |
| "logits/chosen": 9.220715522766113, | |
| "logits/rejected": 10.65298080444336, | |
| "logps/chosen": -187.50601196289062, | |
| "logps/rejected": -232.32241821289062, | |
| "loss": 0.6379, | |
| "rewards/accuracies": 0.6583333015441895, | |
| "rewards/chosen": -0.3121258318424225, | |
| "rewards/margins": 0.4689728617668152, | |
| "rewards/rejected": -0.7810987234115601, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.16736401673640167, | |
| "grad_norm": 6.804388046264648, | |
| "learning_rate": 2.715687502978336e-05, | |
| "logits/chosen": 10.728836059570312, | |
| "logits/rejected": 11.548182487487793, | |
| "logps/chosen": -220.2592010498047, | |
| "logps/rejected": -248.6510009765625, | |
| "loss": 0.5624, | |
| "rewards/accuracies": 0.6916667222976685, | |
| "rewards/chosen": -0.24719354510307312, | |
| "rewards/margins": 0.5707910656929016, | |
| "rewards/rejected": -0.8179847002029419, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.18131101813110181, | |
| "grad_norm": 8.308326721191406, | |
| "learning_rate": 2.714453956614478e-05, | |
| "logits/chosen": 9.632620811462402, | |
| "logits/rejected": 10.766716003417969, | |
| "logps/chosen": -234.59494018554688, | |
| "logps/rejected": -246.2835235595703, | |
| "loss": 0.6495, | |
| "rewards/accuracies": 0.6500000357627869, | |
| "rewards/chosen": -0.4380224347114563, | |
| "rewards/margins": 0.505369246006012, | |
| "rewards/rejected": -0.9433916211128235, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.19525801952580196, | |
| "grad_norm": 5.469903945922852, | |
| "learning_rate": 2.7129883137863668e-05, | |
| "logits/chosen": 10.187846183776855, | |
| "logits/rejected": 11.750052452087402, | |
| "logps/chosen": -201.72952270507812, | |
| "logps/rejected": -254.0623779296875, | |
| "loss": 0.6407, | |
| "rewards/accuracies": 0.6166666746139526, | |
| "rewards/chosen": -0.28986990451812744, | |
| "rewards/margins": 0.4815855026245117, | |
| "rewards/rejected": -0.7714553475379944, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.20920502092050208, | |
| "grad_norm": 6.822120189666748, | |
| "learning_rate": 2.7112909094814497e-05, | |
| "logits/chosen": 11.570058822631836, | |
| "logits/rejected": 12.092260360717773, | |
| "logps/chosen": -201.8399658203125, | |
| "logps/rejected": -255.96774291992188, | |
| "loss": 0.5926, | |
| "rewards/accuracies": 0.6500000357627869, | |
| "rewards/chosen": -0.17586487531661987, | |
| "rewards/margins": 0.4662502706050873, | |
| "rewards/rejected": -0.6421152353286743, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.22315202231520223, | |
| "grad_norm": 6.789912223815918, | |
| "learning_rate": 2.7093621316585976e-05, | |
| "logits/chosen": 9.428289413452148, | |
| "logits/rejected": 12.114514350891113, | |
| "logps/chosen": -199.76634216308594, | |
| "logps/rejected": -259.3163146972656, | |
| "loss": 0.6011, | |
| "rewards/accuracies": 0.7083333730697632, | |
| "rewards/chosen": -0.12371706962585449, | |
| "rewards/margins": 0.6005428433418274, | |
| "rewards/rejected": -0.7242598533630371, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.23709902370990238, | |
| "grad_norm": 8.106010437011719, | |
| "learning_rate": 2.7072024211594312e-05, | |
| "logits/chosen": 10.688722610473633, | |
| "logits/rejected": 11.625075340270996, | |
| "logps/chosen": -221.6556854248047, | |
| "logps/rejected": -274.34808349609375, | |
| "loss": 0.6676, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.237966850399971, | |
| "rewards/margins": 0.4688500761985779, | |
| "rewards/rejected": -0.7068168520927429, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.2510460251046025, | |
| "grad_norm": 7.263269901275635, | |
| "learning_rate": 2.7048122716075636e-05, | |
| "logits/chosen": 12.357452392578125, | |
| "logits/rejected": 14.314582824707031, | |
| "logps/chosen": -224.59451293945312, | |
| "logps/rejected": -291.54840087890625, | |
| "loss": 0.6033, | |
| "rewards/accuracies": 0.6666667461395264, | |
| "rewards/chosen": -0.07308916002511978, | |
| "rewards/margins": 0.504932165145874, | |
| "rewards/rejected": -0.5780213475227356, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.2649930264993027, | |
| "grad_norm": 8.925468444824219, | |
| "learning_rate": 2.7021922292957776e-05, | |
| "logits/chosen": 12.812413215637207, | |
| "logits/rejected": 13.796060562133789, | |
| "logps/chosen": -230.6768798828125, | |
| "logps/rejected": -276.98828125, | |
| "loss": 0.6054, | |
| "rewards/accuracies": 0.6916667222976685, | |
| "rewards/chosen": -0.03974637761712074, | |
| "rewards/margins": 0.5660965442657471, | |
| "rewards/rejected": -0.6058429479598999, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.2789400278940028, | |
| "grad_norm": 5.422610282897949, | |
| "learning_rate": 2.6993428930611634e-05, | |
| "logits/chosen": 12.078475952148438, | |
| "logits/rejected": 14.003347396850586, | |
| "logps/chosen": -229.6229705810547, | |
| "logps/rejected": -276.1091613769531, | |
| "loss": 0.5103, | |
| "rewards/accuracies": 0.7916666269302368, | |
| "rewards/chosen": 0.10478214919567108, | |
| "rewards/margins": 0.8226801156997681, | |
| "rewards/rejected": -0.7178980112075806, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.2928870292887029, | |
| "grad_norm": 7.437990188598633, | |
| "learning_rate": 2.69626491414825e-05, | |
| "logits/chosen": 11.215384483337402, | |
| "logits/rejected": 14.319795608520508, | |
| "logps/chosen": -253.2958221435547, | |
| "logps/rejected": -353.27728271484375, | |
| "loss": 0.4647, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.20068514347076416, | |
| "rewards/margins": 1.1105215549468994, | |
| "rewards/rejected": -1.3112064599990845, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.3068340306834031, | |
| "grad_norm": 4.438156604766846, | |
| "learning_rate": 2.6929589960601567e-05, | |
| "logits/chosen": 10.04852294921875, | |
| "logits/rejected": 11.834890365600586, | |
| "logps/chosen": -177.2984619140625, | |
| "logps/rejected": -235.3513946533203, | |
| "loss": 0.5941, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.14255599677562714, | |
| "rewards/margins": 1.0380289554595947, | |
| "rewards/rejected": -1.1805849075317383, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.3207810320781032, | |
| "grad_norm": 5.81158971786499, | |
| "learning_rate": 2.689425894397799e-05, | |
| "logits/chosen": 11.626302719116211, | |
| "logits/rejected": 13.641179084777832, | |
| "logps/chosen": -258.5274353027344, | |
| "logps/rejected": -290.1232604980469, | |
| "loss": 0.7585, | |
| "rewards/accuracies": 0.6916667222976685, | |
| "rewards/chosen": -0.3435746729373932, | |
| "rewards/margins": 0.7051381468772888, | |
| "rewards/rejected": -1.0487128496170044, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.33472803347280333, | |
| "grad_norm": 6.80866003036499, | |
| "learning_rate": 2.685666416687189e-05, | |
| "logits/chosen": 12.343037605285645, | |
| "logits/rejected": 14.431178092956543, | |
| "logps/chosen": -240.32119750976562, | |
| "logps/rejected": -306.0838623046875, | |
| "loss": 0.595, | |
| "rewards/accuracies": 0.7166666984558105, | |
| "rewards/chosen": 0.007438424974679947, | |
| "rewards/margins": 0.7371615171432495, | |
| "rewards/rejected": -0.7297230362892151, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.3486750348675035, | |
| "grad_norm": 5.887704372406006, | |
| "learning_rate": 2.6816814221948682e-05, | |
| "logits/chosen": 12.360528945922852, | |
| "logits/rejected": 13.354936599731445, | |
| "logps/chosen": -229.65194702148438, | |
| "logps/rejected": -286.4953918457031, | |
| "loss": 0.5958, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": 0.04780165106058121, | |
| "rewards/margins": 0.5758650898933411, | |
| "rewards/rejected": -0.5280634164810181, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.36262203626220363, | |
| "grad_norm": 6.528600215911865, | |
| "learning_rate": 2.6774718217315124e-05, | |
| "logits/chosen": 12.22896671295166, | |
| "logits/rejected": 14.30839729309082, | |
| "logps/chosen": -258.0196838378906, | |
| "logps/rejected": -312.25018310546875, | |
| "loss": 0.5731, | |
| "rewards/accuracies": 0.7166666984558105, | |
| "rewards/chosen": -0.1742248833179474, | |
| "rewards/margins": 0.7956671118736267, | |
| "rewards/rejected": -0.9698920249938965, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.37656903765690375, | |
| "grad_norm": 4.8263702392578125, | |
| "learning_rate": 2.6730385774437575e-05, | |
| "logits/chosen": 11.001718521118164, | |
| "logits/rejected": 12.369864463806152, | |
| "logps/chosen": -186.54306030273438, | |
| "logps/rejected": -229.1658935546875, | |
| "loss": 0.5747, | |
| "rewards/accuracies": 0.7333333492279053, | |
| "rewards/chosen": -0.10876262187957764, | |
| "rewards/margins": 0.783820390701294, | |
| "rewards/rejected": -0.8925830721855164, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.3905160390516039, | |
| "grad_norm": 6.88525915145874, | |
| "learning_rate": 2.668382702594289e-05, | |
| "logits/chosen": 12.428079605102539, | |
| "logits/rejected": 13.376548767089844, | |
| "logps/chosen": -244.35336303710938, | |
| "logps/rejected": -314.2310485839844, | |
| "loss": 0.5727, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.19110675156116486, | |
| "rewards/margins": 0.8208184242248535, | |
| "rewards/rejected": -1.011925220489502, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.40446304044630405, | |
| "grad_norm": 5.504042148590088, | |
| "learning_rate": 2.663505261330254e-05, | |
| "logits/chosen": 11.734379768371582, | |
| "logits/rejected": 13.473363876342773, | |
| "logps/chosen": -204.04251098632812, | |
| "logps/rejected": -258.86737060546875, | |
| "loss": 0.6128, | |
| "rewards/accuracies": 0.6666666865348816, | |
| "rewards/chosen": -0.17432288825511932, | |
| "rewards/margins": 0.4802930951118469, | |
| "rewards/rejected": -0.6546159982681274, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.41841004184100417, | |
| "grad_norm": 6.70412540435791, | |
| "learning_rate": 2.6584073684400373e-05, | |
| "logits/chosen": 13.045954704284668, | |
| "logits/rejected": 13.6720552444458, | |
| "logps/chosen": -220.8832550048828, | |
| "logps/rejected": -269.233642578125, | |
| "loss": 0.6032, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.04318929463624954, | |
| "rewards/margins": 0.618157684803009, | |
| "rewards/rejected": -0.6613470315933228, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.43235704323570434, | |
| "grad_norm": 12.566320419311523, | |
| "learning_rate": 2.653090189098466e-05, | |
| "logits/chosen": 11.808868408203125, | |
| "logits/rejected": 13.49694538116455, | |
| "logps/chosen": -248.18212890625, | |
| "logps/rejected": -286.3559265136719, | |
| "loss": 0.6096, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.15974543988704681, | |
| "rewards/margins": 0.6279758214950562, | |
| "rewards/rejected": -0.7877213954925537, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.44630404463040446, | |
| "grad_norm": 6.7717509269714355, | |
| "learning_rate": 2.647554938600497e-05, | |
| "logits/chosen": 11.906153678894043, | |
| "logits/rejected": 13.252202033996582, | |
| "logps/chosen": -230.92593383789062, | |
| "logps/rejected": -248.78488159179688, | |
| "loss": 0.6474, | |
| "rewards/accuracies": 0.6166666746139526, | |
| "rewards/chosen": -0.20924147963523865, | |
| "rewards/margins": 0.49998435378074646, | |
| "rewards/rejected": -0.7092257738113403, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.4602510460251046, | |
| "grad_norm": 7.113363742828369, | |
| "learning_rate": 2.6418028820834483e-05, | |
| "logits/chosen": 11.409849166870117, | |
| "logits/rejected": 13.500089645385742, | |
| "logps/chosen": -242.9384765625, | |
| "logps/rejected": -325.8098449707031, | |
| "loss": 0.5703, | |
| "rewards/accuracies": 0.7000000476837158, | |
| "rewards/chosen": -0.4405299127101898, | |
| "rewards/margins": 0.8166629076004028, | |
| "rewards/rejected": -1.257192850112915, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.47419804741980476, | |
| "grad_norm": 4.8473429679870605, | |
| "learning_rate": 2.6358353342378405e-05, | |
| "logits/chosen": 8.604534149169922, | |
| "logits/rejected": 11.609006881713867, | |
| "logps/chosen": -185.0061492919922, | |
| "logps/rejected": -256.83465576171875, | |
| "loss": 0.4518, | |
| "rewards/accuracies": 0.8166667222976685, | |
| "rewards/chosen": -0.5300418734550476, | |
| "rewards/margins": 1.1527934074401855, | |
| "rewards/rejected": -1.682835340499878, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.4881450488145049, | |
| "grad_norm": 6.59335994720459, | |
| "learning_rate": 2.6296536590069104e-05, | |
| "logits/chosen": 9.987462997436523, | |
| "logits/rejected": 11.572959899902344, | |
| "logps/chosen": -253.36050415039062, | |
| "logps/rejected": -306.0450439453125, | |
| "loss": 0.7439, | |
| "rewards/accuracies": 0.6666666269302368, | |
| "rewards/chosen": -1.0800180435180664, | |
| "rewards/margins": 0.8873661160469055, | |
| "rewards/rejected": -1.9673839807510376, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.502092050209205, | |
| "grad_norm": 7.2572808265686035, | |
| "learning_rate": 2.6232592692748676e-05, | |
| "logits/chosen": 9.321784019470215, | |
| "logits/rejected": 10.399984359741211, | |
| "logps/chosen": -234.3979949951172, | |
| "logps/rejected": -294.28961181640625, | |
| "loss": 0.489, | |
| "rewards/accuracies": 0.7916667461395264, | |
| "rewards/chosen": -0.8451215624809265, | |
| "rewards/margins": 1.1555068492889404, | |
| "rewards/rejected": -2.0006284713745117, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.5160390516039052, | |
| "grad_norm": 6.049355506896973, | |
| "learning_rate": 2.6166536265439664e-05, | |
| "logits/chosen": 11.809592247009277, | |
| "logits/rejected": 13.855669021606445, | |
| "logps/chosen": -235.9927215576172, | |
| "logps/rejected": -315.3490295410156, | |
| "loss": 0.5868, | |
| "rewards/accuracies": 0.6583333015441895, | |
| "rewards/chosen": -0.6016494631767273, | |
| "rewards/margins": 0.7112706899642944, | |
| "rewards/rejected": -1.3129202127456665, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.5299860529986054, | |
| "grad_norm": 11.038729667663574, | |
| "learning_rate": 2.609838240600464e-05, | |
| "logits/chosen": 11.31627082824707, | |
| "logits/rejected": 12.770919799804688, | |
| "logps/chosen": -221.79501342773438, | |
| "logps/rejected": -262.8851013183594, | |
| "loss": 0.5937, | |
| "rewards/accuracies": 0.6916667222976685, | |
| "rewards/chosen": -0.5558231472969055, | |
| "rewards/margins": 0.6753066778182983, | |
| "rewards/rejected": -1.2311298847198486, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.5439330543933054, | |
| "grad_norm": 4.267746925354004, | |
| "learning_rate": 2.602814669169543e-05, | |
| "logits/chosen": 11.312234878540039, | |
| "logits/rejected": 13.296246528625488, | |
| "logps/chosen": -221.83663940429688, | |
| "logps/rejected": -302.32623291015625, | |
| "loss": 0.5809, | |
| "rewards/accuracies": 0.7166666984558105, | |
| "rewards/chosen": -0.5700381994247437, | |
| "rewards/margins": 0.8077206611633301, | |
| "rewards/rejected": -1.3777590990066528, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.5578800557880056, | |
| "grad_norm": 5.395613193511963, | |
| "learning_rate": 2.5955845175592813e-05, | |
| "logits/chosen": 11.410247802734375, | |
| "logits/rejected": 13.515039443969727, | |
| "logps/chosen": -232.266845703125, | |
| "logps/rejected": -293.22467041015625, | |
| "loss": 0.5553, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.5373277068138123, | |
| "rewards/margins": 0.8851755857467651, | |
| "rewards/rejected": -1.4225032329559326, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.5718270571827058, | |
| "grad_norm": 5.884355545043945, | |
| "learning_rate": 2.5881494382937354e-05, | |
| "logits/chosen": 11.658515930175781, | |
| "logits/rejected": 13.375651359558105, | |
| "logps/chosen": -212.52932739257812, | |
| "logps/rejected": -286.71783447265625, | |
| "loss": 0.5126, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.6171377897262573, | |
| "rewards/margins": 1.1004387140274048, | |
| "rewards/rejected": -1.7175763845443726, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.5857740585774058, | |
| "grad_norm": 8.411831855773926, | |
| "learning_rate": 2.5805111307352483e-05, | |
| "logits/chosen": 11.734652519226074, | |
| "logits/rejected": 12.831028938293457, | |
| "logps/chosen": -247.45736694335938, | |
| "logps/rejected": -297.03485107421875, | |
| "loss": 0.5907, | |
| "rewards/accuracies": 0.7333332896232605, | |
| "rewards/chosen": -0.6340610384941101, | |
| "rewards/margins": 1.0966707468032837, | |
| "rewards/rejected": -1.7307319641113281, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.599721059972106, | |
| "grad_norm": 6.429418087005615, | |
| "learning_rate": 2.5726713406960365e-05, | |
| "logits/chosen": 9.269506454467773, | |
| "logits/rejected": 12.285856246948242, | |
| "logps/chosen": -187.17591857910156, | |
| "logps/rejected": -269.7003479003906, | |
| "loss": 0.5349, | |
| "rewards/accuracies": 0.7666667699813843, | |
| "rewards/chosen": -0.5631422400474548, | |
| "rewards/margins": 1.0805346965789795, | |
| "rewards/rejected": -1.6436771154403687, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.6136680613668062, | |
| "grad_norm": 8.299524307250977, | |
| "learning_rate": 2.5646318600391693e-05, | |
| "logits/chosen": 10.429685592651367, | |
| "logits/rejected": 11.87198543548584, | |
| "logps/chosen": -234.75802612304688, | |
| "logps/rejected": -284.33642578125, | |
| "loss": 0.5775, | |
| "rewards/accuracies": 0.6833333373069763, | |
| "rewards/chosen": -0.5350970029830933, | |
| "rewards/margins": 1.0643174648284912, | |
| "rewards/rejected": -1.599414348602295, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.6276150627615062, | |
| "grad_norm": 3.9937915802001953, | |
| "learning_rate": 2.556394526269021e-05, | |
| "logits/chosen": 11.182074546813965, | |
| "logits/rejected": 12.9561767578125, | |
| "logps/chosen": -209.6283721923828, | |
| "logps/rejected": -261.5816345214844, | |
| "loss": 0.7124, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.651893675327301, | |
| "rewards/margins": 0.631694495677948, | |
| "rewards/rejected": -1.283588171005249, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.6415620641562064, | |
| "grad_norm": 8.43625259399414, | |
| "learning_rate": 2.5479612221112888e-05, | |
| "logits/chosen": 11.575922966003418, | |
| "logits/rejected": 12.958845138549805, | |
| "logps/chosen": -241.3083953857422, | |
| "logps/rejected": -291.8955993652344, | |
| "loss": 0.6766, | |
| "rewards/accuracies": 0.6666666269302368, | |
| "rewards/chosen": -0.6825530529022217, | |
| "rewards/margins": 0.6574732661247253, | |
| "rewards/rejected": -1.3400263786315918, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.6555090655509066, | |
| "grad_norm": 4.0093817710876465, | |
| "learning_rate": 2.5393338750826796e-05, | |
| "logits/chosen": 12.776809692382812, | |
| "logits/rejected": 14.406654357910156, | |
| "logps/chosen": -229.2129669189453, | |
| "logps/rejected": -300.2899475097656, | |
| "loss": 0.5466, | |
| "rewards/accuracies": 0.7000000476837158, | |
| "rewards/chosen": -0.5617952346801758, | |
| "rewards/margins": 0.752461850643158, | |
| "rewards/rejected": -1.314257264137268, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.6694560669456067, | |
| "grad_norm": 6.296603679656982, | |
| "learning_rate": 2.5305144570503554e-05, | |
| "logits/chosen": 12.071104049682617, | |
| "logits/rejected": 14.155171394348145, | |
| "logps/chosen": -226.9972381591797, | |
| "logps/rejected": -299.86993408203125, | |
| "loss": 0.5546, | |
| "rewards/accuracies": 0.7166667580604553, | |
| "rewards/chosen": -0.5309011936187744, | |
| "rewards/margins": 0.8765324354171753, | |
| "rewards/rejected": -1.4074336290359497, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.6834030683403068, | |
| "grad_norm": 6.713784694671631, | |
| "learning_rate": 2.5215049837812413e-05, | |
| "logits/chosen": 11.046672821044922, | |
| "logits/rejected": 13.329177856445312, | |
| "logps/chosen": -222.3082275390625, | |
| "logps/rejected": -302.4698181152344, | |
| "loss": 0.4944, | |
| "rewards/accuracies": 0.7583333253860474, | |
| "rewards/chosen": -0.6500851511955261, | |
| "rewards/margins": 1.216412901878357, | |
| "rewards/rejected": -1.8664979934692383, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.697350069735007, | |
| "grad_norm": 3.64483904838562, | |
| "learning_rate": 2.5123075144813044e-05, | |
| "logits/chosen": 9.95425796508789, | |
| "logits/rejected": 12.83178997039795, | |
| "logps/chosen": -245.83987426757812, | |
| "logps/rejected": -347.21807861328125, | |
| "loss": 0.4497, | |
| "rewards/accuracies": 0.8083333969116211, | |
| "rewards/chosen": -0.7453610897064209, | |
| "rewards/margins": 1.5202277898788452, | |
| "rewards/rejected": -2.2655892372131348, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.7112970711297071, | |
| "grad_norm": 6.323943138122559, | |
| "learning_rate": 2.5029241513248992e-05, | |
| "logits/chosen": 10.718851089477539, | |
| "logits/rejected": 11.787945747375488, | |
| "logps/chosen": -243.6724853515625, | |
| "logps/rejected": -275.3341064453125, | |
| "loss": 0.7227, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.8060539364814758, | |
| "rewards/margins": 0.8095922470092773, | |
| "rewards/rejected": -1.6156460046768188, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.7252440725244073, | |
| "grad_norm": 7.24354887008667, | |
| "learning_rate": 2.4933570389742975e-05, | |
| "logits/chosen": 10.517350196838379, | |
| "logits/rejected": 12.290593147277832, | |
| "logps/chosen": -217.3165740966797, | |
| "logps/rejected": -277.9285888671875, | |
| "loss": 0.5759, | |
| "rewards/accuracies": 0.7333332896232605, | |
| "rewards/chosen": -0.4403415620326996, | |
| "rewards/margins": 0.91583251953125, | |
| "rewards/rejected": -1.3561739921569824, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.7391910739191074, | |
| "grad_norm": 5.851767063140869, | |
| "learning_rate": 2.4836083640895016e-05, | |
| "logits/chosen": 11.360766410827637, | |
| "logits/rejected": 13.0468111038208, | |
| "logps/chosen": -253.7987518310547, | |
| "logps/rejected": -321.97967529296875, | |
| "loss": 0.552, | |
| "rewards/accuracies": 0.73333340883255, | |
| "rewards/chosen": -0.37671542167663574, | |
| "rewards/margins": 1.0203909873962402, | |
| "rewards/rejected": -1.3971065282821655, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.7531380753138075, | |
| "grad_norm": 5.513597011566162, | |
| "learning_rate": 2.473680354828461e-05, | |
| "logits/chosen": 10.999284744262695, | |
| "logits/rejected": 12.660871505737305, | |
| "logps/chosen": -218.94735717773438, | |
| "logps/rejected": -259.43133544921875, | |
| "loss": 0.5662, | |
| "rewards/accuracies": 0.7083333730697632, | |
| "rewards/chosen": -0.4751812517642975, | |
| "rewards/margins": 0.7894707322120667, | |
| "rewards/rejected": -1.264651894569397, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.7670850767085077, | |
| "grad_norm": 5.750617504119873, | |
| "learning_rate": 2.4635752803378063e-05, | |
| "logits/chosen": 11.961132049560547, | |
| "logits/rejected": 14.314129829406738, | |
| "logps/chosen": -221.6836700439453, | |
| "logps/rejected": -307.97467041015625, | |
| "loss": 0.5598, | |
| "rewards/accuracies": 0.7166666984558105, | |
| "rewards/chosen": -0.42271748185157776, | |
| "rewards/margins": 0.7973464727401733, | |
| "rewards/rejected": -1.2200638055801392, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.7810320781032078, | |
| "grad_norm": 6.243062496185303, | |
| "learning_rate": 2.453295450234211e-05, | |
| "logits/chosen": 11.414163589477539, | |
| "logits/rejected": 13.276272773742676, | |
| "logps/chosen": -223.52529907226562, | |
| "logps/rejected": -324.9430236816406, | |
| "loss": 0.516, | |
| "rewards/accuracies": 0.7833333015441895, | |
| "rewards/chosen": -0.5223814249038696, | |
| "rewards/margins": 1.0250855684280396, | |
| "rewards/rejected": -1.5474669933319092, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.7949790794979079, | |
| "grad_norm": 7.279259204864502, | |
| "learning_rate": 2.442843214076507e-05, | |
| "logits/chosen": 11.585288047790527, | |
| "logits/rejected": 12.664071083068848, | |
| "logps/chosen": -244.99154663085938, | |
| "logps/rejected": -254.84158325195312, | |
| "loss": 0.5448, | |
| "rewards/accuracies": 0.6916667222976685, | |
| "rewards/chosen": -0.7385099530220032, | |
| "rewards/margins": 0.9241247177124023, | |
| "rewards/rejected": -1.6626346111297607, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.8089260808926081, | |
| "grad_norm": 4.353116512298584, | |
| "learning_rate": 2.4322209608286686e-05, | |
| "logits/chosen": 8.986726760864258, | |
| "logits/rejected": 12.175418853759766, | |
| "logps/chosen": -206.93594360351562, | |
| "logps/rejected": -292.4324645996094, | |
| "loss": 0.5075, | |
| "rewards/accuracies": 0.7166666984558105, | |
| "rewards/chosen": -0.7221769094467163, | |
| "rewards/margins": 1.2281521558761597, | |
| "rewards/rejected": -1.9503291845321655, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.8228730822873083, | |
| "grad_norm": 8.33502197265625, | |
| "learning_rate": 2.421431118313789e-05, | |
| "logits/chosen": 10.475484848022461, | |
| "logits/rejected": 11.844476699829102, | |
| "logps/chosen": -240.02206420898438, | |
| "logps/rejected": -279.66064453125, | |
| "loss": 0.7693, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -1.0663460493087769, | |
| "rewards/margins": 0.940434455871582, | |
| "rewards/rejected": -2.0067803859710693, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 0.8368200836820083, | |
| "grad_norm": 7.870750904083252, | |
| "learning_rate": 2.41047615265918e-05, | |
| "logits/chosen": 11.179079055786133, | |
| "logits/rejected": 13.722526550292969, | |
| "logps/chosen": -243.4619140625, | |
| "logps/rejected": -338.6166687011719, | |
| "loss": 0.5564, | |
| "rewards/accuracies": 0.7333333492279053, | |
| "rewards/chosen": -0.5845211744308472, | |
| "rewards/margins": 1.252018690109253, | |
| "rewards/rejected": -1.8365398645401, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.8507670850767085, | |
| "grad_norm": 5.815814018249512, | |
| "learning_rate": 2.3993585677327107e-05, | |
| "logits/chosen": 13.206197738647461, | |
| "logits/rejected": 13.943652153015137, | |
| "logps/chosen": -276.7415466308594, | |
| "logps/rejected": -329.16949462890625, | |
| "loss": 0.5341, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.4414283335208893, | |
| "rewards/margins": 0.9674088358879089, | |
| "rewards/rejected": -1.408837080001831, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 0.8647140864714087, | |
| "grad_norm": 8.953941345214844, | |
| "learning_rate": 2.3880809045705262e-05, | |
| "logits/chosen": 12.374226570129395, | |
| "logits/rejected": 13.989924430847168, | |
| "logps/chosen": -244.1103515625, | |
| "logps/rejected": -294.4126281738281, | |
| "loss": 0.5197, | |
| "rewards/accuracies": 0.7416666746139526, | |
| "rewards/chosen": -0.3111744523048401, | |
| "rewards/margins": 1.076432228088379, | |
| "rewards/rejected": -1.3876066207885742, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.8786610878661087, | |
| "grad_norm": 5.921610355377197, | |
| "learning_rate": 2.3766457407962654e-05, | |
| "logits/chosen": 11.312451362609863, | |
| "logits/rejected": 13.527206420898438, | |
| "logps/chosen": -234.7072296142578, | |
| "logps/rejected": -290.8281555175781, | |
| "loss": 0.6344, | |
| "rewards/accuracies": 0.6583333611488342, | |
| "rewards/chosen": -0.5945696234703064, | |
| "rewards/margins": 0.7386828660964966, | |
| "rewards/rejected": -1.3332524299621582, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 0.8926080892608089, | |
| "grad_norm": 7.662965774536133, | |
| "learning_rate": 2.3650556900319204e-05, | |
| "logits/chosen": 11.144214630126953, | |
| "logits/rejected": 13.931811332702637, | |
| "logps/chosen": -198.17422485351562, | |
| "logps/rejected": -286.37152099609375, | |
| "loss": 0.5295, | |
| "rewards/accuracies": 0.7166666388511658, | |
| "rewards/chosen": -0.3581700325012207, | |
| "rewards/margins": 1.0556398630142212, | |
| "rewards/rejected": -1.4138100147247314, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.9065550906555091, | |
| "grad_norm": 6.2693071365356445, | |
| "learning_rate": 2.3533134013004666e-05, | |
| "logits/chosen": 11.203069686889648, | |
| "logits/rejected": 11.721672058105469, | |
| "logps/chosen": -192.05380249023438, | |
| "logps/rejected": -221.40042114257812, | |
| "loss": 0.6338, | |
| "rewards/accuracies": 0.7083333730697632, | |
| "rewards/chosen": -0.56825852394104, | |
| "rewards/margins": 0.6851181983947754, | |
| "rewards/rejected": -1.253376841545105, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 0.9205020920502092, | |
| "grad_norm": 4.502702236175537, | |
| "learning_rate": 2.341421558420403e-05, | |
| "logits/chosen": 10.94641399383545, | |
| "logits/rejected": 13.1921968460083, | |
| "logps/chosen": -214.18911743164062, | |
| "logps/rejected": -298.7375183105469, | |
| "loss": 0.4812, | |
| "rewards/accuracies": 0.7416666746139526, | |
| "rewards/chosen": -0.6049180030822754, | |
| "rewards/margins": 1.2558748722076416, | |
| "rewards/rejected": -1.860793113708496, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.9344490934449093, | |
| "grad_norm": 6.939857482910156, | |
| "learning_rate": 2.3293828793923365e-05, | |
| "logits/chosen": 11.944158554077148, | |
| "logits/rejected": 13.312121391296387, | |
| "logps/chosen": -258.62774658203125, | |
| "logps/rejected": -314.3907165527344, | |
| "loss": 0.619, | |
| "rewards/accuracies": 0.6583333015441895, | |
| "rewards/chosen": -0.5666269063949585, | |
| "rewards/margins": 1.0656505823135376, | |
| "rewards/rejected": -1.632277488708496, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 0.9483960948396095, | |
| "grad_norm": 4.783708572387695, | |
| "learning_rate": 2.3172001157777566e-05, | |
| "logits/chosen": 11.259064674377441, | |
| "logits/rejected": 12.700227737426758, | |
| "logps/chosen": -222.77395629882812, | |
| "logps/rejected": -291.24884033203125, | |
| "loss": 0.6497, | |
| "rewards/accuracies": 0.658333420753479, | |
| "rewards/chosen": -0.5113255977630615, | |
| "rewards/margins": 0.866672158241272, | |
| "rewards/rejected": -1.377997636795044, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.9623430962343096, | |
| "grad_norm": 4.230240821838379, | |
| "learning_rate": 2.3048760520701374e-05, | |
| "logits/chosen": 11.741998672485352, | |
| "logits/rejected": 13.120841979980469, | |
| "logps/chosen": -246.1442108154297, | |
| "logps/rejected": -311.65972900390625, | |
| "loss": 0.485, | |
| "rewards/accuracies": 0.7416666746139526, | |
| "rewards/chosen": -0.4931299090385437, | |
| "rewards/margins": 1.0040347576141357, | |
| "rewards/rejected": -1.4971646070480347, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 0.9762900976290098, | |
| "grad_norm": 4.901912212371826, | |
| "learning_rate": 2.2924135050585152e-05, | |
| "logits/chosen": 11.310202598571777, | |
| "logits/rejected": 13.040916442871094, | |
| "logps/chosen": -223.5089111328125, | |
| "logps/rejected": -253.26953125, | |
| "loss": 0.5638, | |
| "rewards/accuracies": 0.7416666746139526, | |
| "rewards/chosen": -0.6479132771492004, | |
| "rewards/margins": 0.7726107835769653, | |
| "rewards/rejected": -1.4205242395401, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.9902370990237099, | |
| "grad_norm": 4.653822898864746, | |
| "learning_rate": 2.2798153231836813e-05, | |
| "logits/chosen": 12.201348304748535, | |
| "logits/rejected": 13.601194381713867, | |
| "logps/chosen": -268.86328125, | |
| "logps/rejected": -299.11578369140625, | |
| "loss": 0.5367, | |
| "rewards/accuracies": 0.7416666746139526, | |
| "rewards/chosen": -0.5867849588394165, | |
| "rewards/margins": 1.1348934173583984, | |
| "rewards/rejected": -1.721678376197815, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 0.99860529986053, | |
| "eval_logits/chosen": 11.91541862487793, | |
| "eval_logits/rejected": 12.808130264282227, | |
| "eval_logps/chosen": -229.4397430419922, | |
| "eval_logps/rejected": -286.59234619140625, | |
| "eval_loss": 0.6298339366912842, | |
| "eval_rewards/accuracies": 0.7099999785423279, | |
| "eval_rewards/chosen": -0.7923385500907898, | |
| "eval_rewards/margins": 0.9493054151535034, | |
| "eval_rewards/rejected": -1.7416436672210693, | |
| "eval_runtime": 24.5973, | |
| "eval_samples_per_second": 8.131, | |
| "eval_steps_per_second": 8.131, | |
| "step": 358 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 1074, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 12, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |