| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.0, |
| "eval_steps": 500, |
| "global_step": 1407, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.010666666666666666, |
| "grad_norm": 3.0458550453186035, |
| "learning_rate": 4.651162790697675e-07, |
| "logits/chosen": -1.1378610134124756, |
| "logits/rejected": -1.1131585836410522, |
| "logps/chosen": -220.4049530029297, |
| "logps/rejected": -234.8083953857422, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.3375000059604645, |
| "rewards/chosen": 0.0011935376096516848, |
| "rewards/margins": 0.0005433893529698253, |
| "rewards/rejected": 0.0006501483730971813, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021333333333333333, |
| "grad_norm": 2.8662004470825195, |
| "learning_rate": 1.0465116279069768e-06, |
| "logits/chosen": -1.122249722480774, |
| "logits/rejected": -1.0652912855148315, |
| "logps/chosen": -223.3093719482422, |
| "logps/rejected": -215.4896697998047, |
| "loss": 0.6878, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": 0.010744137689471245, |
| "rewards/margins": 0.010764424689114094, |
| "rewards/rejected": -2.028709241130855e-05, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.032, |
| "grad_norm": 3.1073992252349854, |
| "learning_rate": 1.6279069767441862e-06, |
| "logits/chosen": -1.103829026222229, |
| "logits/rejected": -1.021059274673462, |
| "logps/chosen": -230.15866088867188, |
| "logps/rejected": -228.9940948486328, |
| "loss": 0.6777, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": 0.02945018745958805, |
| "rewards/margins": 0.032092902809381485, |
| "rewards/rejected": -0.002642714884132147, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.042666666666666665, |
| "grad_norm": 2.8836920261383057, |
| "learning_rate": 2.2093023255813954e-06, |
| "logits/chosen": -1.131155252456665, |
| "logits/rejected": -1.1667766571044922, |
| "logps/chosen": -224.6150665283203, |
| "logps/rejected": -239.52145385742188, |
| "loss": 0.6578, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 0.06944439560174942, |
| "rewards/margins": 0.07668272405862808, |
| "rewards/rejected": -0.0072383270598948, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.05333333333333334, |
| "grad_norm": 2.933941602706909, |
| "learning_rate": 2.790697674418605e-06, |
| "logits/chosen": -1.182265043258667, |
| "logits/rejected": -1.1075844764709473, |
| "logps/chosen": -224.00808715820312, |
| "logps/rejected": -253.9156951904297, |
| "loss": 0.6349, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.11455950886011124, |
| "rewards/margins": 0.13011686503887177, |
| "rewards/rejected": -0.015557368285953999, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.064, |
| "grad_norm": 3.208721399307251, |
| "learning_rate": 3.372093023255814e-06, |
| "logits/chosen": -1.1079695224761963, |
| "logits/rejected": -1.1115219593048096, |
| "logps/chosen": -214.77395629882812, |
| "logps/rejected": -233.01913452148438, |
| "loss": 0.6115, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.15914058685302734, |
| "rewards/margins": 0.19624650478363037, |
| "rewards/rejected": -0.03710593655705452, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07466666666666667, |
| "grad_norm": 3.0986759662628174, |
| "learning_rate": 3.953488372093024e-06, |
| "logits/chosen": -1.2299989461898804, |
| "logits/rejected": -1.0735408067703247, |
| "logps/chosen": -226.4742889404297, |
| "logps/rejected": -225.53738403320312, |
| "loss": 0.5806, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.2632412314414978, |
| "rewards/margins": 0.3024232089519501, |
| "rewards/rejected": -0.03918201103806496, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08533333333333333, |
| "grad_norm": 2.538855791091919, |
| "learning_rate": 4.5348837209302326e-06, |
| "logits/chosen": -1.2544300556182861, |
| "logits/rejected": -1.1668885946273804, |
| "logps/chosen": -217.96572875976562, |
| "logps/rejected": -240.23434448242188, |
| "loss": 0.543, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 0.3376604914665222, |
| "rewards/margins": 0.44354838132858276, |
| "rewards/rejected": -0.10588787496089935, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.096, |
| "grad_norm": 2.8498013019561768, |
| "learning_rate": 4.99999336897255e-06, |
| "logits/chosen": -1.205204963684082, |
| "logits/rejected": -1.1860382556915283, |
| "logps/chosen": -222.1924591064453, |
| "logps/rejected": -227.74472045898438, |
| "loss": 0.5477, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": 0.33458584547042847, |
| "rewards/margins": 0.4299747347831726, |
| "rewards/rejected": -0.09538889676332474, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10666666666666667, |
| "grad_norm": 2.4346165657043457, |
| "learning_rate": 4.999761286705285e-06, |
| "logits/chosen": -1.158801794052124, |
| "logits/rejected": -1.1945750713348389, |
| "logps/chosen": -219.235107421875, |
| "logps/rejected": -242.387451171875, |
| "loss": 0.5045, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": 0.4660046100616455, |
| "rewards/margins": 0.6082803010940552, |
| "rewards/rejected": -0.14227569103240967, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11733333333333333, |
| "grad_norm": 2.311579704284668, |
| "learning_rate": 4.999197688241076e-06, |
| "logits/chosen": -1.2277817726135254, |
| "logits/rejected": -1.1361002922058105, |
| "logps/chosen": -204.9770050048828, |
| "logps/rejected": -241.1087188720703, |
| "loss": 0.4901, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": 0.47566550970077515, |
| "rewards/margins": 0.7237527966499329, |
| "rewards/rejected": -0.24808719754219055, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.128, |
| "grad_norm": 2.3512964248657227, |
| "learning_rate": 4.9983026483238685e-06, |
| "logits/chosen": -1.2030401229858398, |
| "logits/rejected": -1.1904528141021729, |
| "logps/chosen": -228.5006561279297, |
| "logps/rejected": -251.4223175048828, |
| "loss": 0.4376, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.6135213971138, |
| "rewards/margins": 0.9777547717094421, |
| "rewards/rejected": -0.36423343420028687, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13866666666666666, |
| "grad_norm": 2.3212287425994873, |
| "learning_rate": 4.997076285653087e-06, |
| "logits/chosen": -1.2506873607635498, |
| "logits/rejected": -1.196251392364502, |
| "logps/chosen": -240.0591583251953, |
| "logps/rejected": -230.9004669189453, |
| "loss": 0.4769, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": 0.5166982412338257, |
| "rewards/margins": 0.8954504728317261, |
| "rewards/rejected": -0.3787522315979004, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14933333333333335, |
| "grad_norm": 2.683375120162964, |
| "learning_rate": 4.995518762867897e-06, |
| "logits/chosen": -1.383887529373169, |
| "logits/rejected": -1.2323031425476074, |
| "logps/chosen": -209.1799774169922, |
| "logps/rejected": -238.6314239501953, |
| "loss": 0.4814, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": 0.5157665014266968, |
| "rewards/margins": 0.9230127334594727, |
| "rewards/rejected": -0.4072461724281311, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.16, |
| "grad_norm": 2.596590280532837, |
| "learning_rate": 4.993630286525634e-06, |
| "logits/chosen": -1.3620352745056152, |
| "logits/rejected": -1.3067903518676758, |
| "logps/chosen": -200.5843505859375, |
| "logps/rejected": -233.67300415039062, |
| "loss": 0.4318, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.5101022124290466, |
| "rewards/margins": 0.9610568284988403, |
| "rewards/rejected": -0.4509546160697937, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.17066666666666666, |
| "grad_norm": 2.6825714111328125, |
| "learning_rate": 4.99141110707441e-06, |
| "logits/chosen": -1.312425136566162, |
| "logits/rejected": -1.2870181798934937, |
| "logps/chosen": -230.2716064453125, |
| "logps/rejected": -238.5307159423828, |
| "loss": 0.4464, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": 0.5834903717041016, |
| "rewards/margins": 1.1369092464447021, |
| "rewards/rejected": -0.5534188747406006, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.18133333333333335, |
| "grad_norm": 3.115396738052368, |
| "learning_rate": 4.9888615188199e-06, |
| "logits/chosen": -1.2816749811172485, |
| "logits/rejected": -1.2261050939559937, |
| "logps/chosen": -227.169189453125, |
| "logps/rejected": -247.8328857421875, |
| "loss": 0.3964, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.7111604809761047, |
| "rewards/margins": 1.3225681781768799, |
| "rewards/rejected": -0.6114078164100647, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.192, |
| "grad_norm": 2.20292329788208, |
| "learning_rate": 4.9859818598863104e-06, |
| "logits/chosen": -1.2855154275894165, |
| "logits/rejected": -1.2214223146438599, |
| "logps/chosen": -207.7841796875, |
| "logps/rejected": -247.1141815185547, |
| "loss": 0.3909, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.6553568243980408, |
| "rewards/margins": 1.384448766708374, |
| "rewards/rejected": -0.729092001914978, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20266666666666666, |
| "grad_norm": 3.6834919452667236, |
| "learning_rate": 4.982772512171538e-06, |
| "logits/chosen": -1.3217604160308838, |
| "logits/rejected": -1.3131191730499268, |
| "logps/chosen": -211.2960662841797, |
| "logps/rejected": -242.7036590576172, |
| "loss": 0.459, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.3602835536003113, |
| "rewards/margins": 1.075905680656433, |
| "rewards/rejected": -0.7156221866607666, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21333333333333335, |
| "grad_norm": 2.337843179702759, |
| "learning_rate": 4.979233901296523e-06, |
| "logits/chosen": -1.4003416299819946, |
| "logits/rejected": -1.3594415187835693, |
| "logps/chosen": -263.1855773925781, |
| "logps/rejected": -250.721435546875, |
| "loss": 0.3688, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.5528009533882141, |
| "rewards/margins": 1.4668482542037964, |
| "rewards/rejected": -0.9140474200248718, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.224, |
| "grad_norm": 2.303995132446289, |
| "learning_rate": 4.975366496548804e-06, |
| "logits/chosen": -1.319817066192627, |
| "logits/rejected": -1.2342548370361328, |
| "logps/chosen": -213.7783966064453, |
| "logps/rejected": -256.16680908203125, |
| "loss": 0.3626, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.6676830053329468, |
| "rewards/margins": 1.5440034866333008, |
| "rewards/rejected": -0.876320481300354, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23466666666666666, |
| "grad_norm": 2.2961652278900146, |
| "learning_rate": 4.971170810820279e-06, |
| "logits/chosen": -1.34084153175354, |
| "logits/rejected": -1.3188228607177734, |
| "logps/chosen": -226.8044891357422, |
| "logps/rejected": -235.49752807617188, |
| "loss": 0.4056, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.4525774419307709, |
| "rewards/margins": 1.4416332244873047, |
| "rewards/rejected": -0.9890557527542114, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24533333333333332, |
| "grad_norm": 2.7958457469940186, |
| "learning_rate": 4.96664740053919e-06, |
| "logits/chosen": -1.3018507957458496, |
| "logits/rejected": -1.2472938299179077, |
| "logps/chosen": -214.3036651611328, |
| "logps/rejected": -240.1237030029297, |
| "loss": 0.3764, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 0.5986575484275818, |
| "rewards/margins": 1.6609241962432861, |
| "rewards/rejected": -1.0622665882110596, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.256, |
| "grad_norm": 3.2483537197113037, |
| "learning_rate": 4.961796865596328e-06, |
| "logits/chosen": -1.2724711894989014, |
| "logits/rejected": -1.3170382976531982, |
| "logps/chosen": -205.23342895507812, |
| "logps/rejected": -252.8673095703125, |
| "loss": 0.3533, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.7337419390678406, |
| "rewards/margins": 1.7917591333389282, |
| "rewards/rejected": -1.0580171346664429, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.26666666666666666, |
| "grad_norm": 2.846489191055298, |
| "learning_rate": 4.956619849265473e-06, |
| "logits/chosen": -1.4343109130859375, |
| "logits/rejected": -1.3631373643875122, |
| "logps/chosen": -198.54171752929688, |
| "logps/rejected": -262.36981201171875, |
| "loss": 0.37, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.6829321384429932, |
| "rewards/margins": 1.8867518901824951, |
| "rewards/rejected": -1.2038198709487915, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.2773333333333333, |
| "grad_norm": 1.6181107759475708, |
| "learning_rate": 4.951117038118091e-06, |
| "logits/chosen": -1.4376757144927979, |
| "logits/rejected": -1.4121671915054321, |
| "logps/chosen": -207.24801635742188, |
| "logps/rejected": -277.3598327636719, |
| "loss": 0.3248, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.5676002502441406, |
| "rewards/margins": 1.9428207874298096, |
| "rewards/rejected": -1.375220537185669, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.288, |
| "grad_norm": 3.0397274494171143, |
| "learning_rate": 4.945289161932273e-06, |
| "logits/chosen": -1.4408540725708008, |
| "logits/rejected": -1.3692197799682617, |
| "logps/chosen": -222.3744659423828, |
| "logps/rejected": -261.72308349609375, |
| "loss": 0.41, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 0.29623842239379883, |
| "rewards/margins": 1.554736614227295, |
| "rewards/rejected": -1.2584980726242065, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.2986666666666667, |
| "grad_norm": 3.019113779067993, |
| "learning_rate": 4.93913699359596e-06, |
| "logits/chosen": -1.4328057765960693, |
| "logits/rejected": -1.3377001285552979, |
| "logps/chosen": -203.7570343017578, |
| "logps/rejected": -245.04714965820312, |
| "loss": 0.3437, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.4516337513923645, |
| "rewards/margins": 1.7678340673446655, |
| "rewards/rejected": -1.3162004947662354, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.30933333333333335, |
| "grad_norm": 2.6861343383789062, |
| "learning_rate": 4.9326613490044364e-06, |
| "logits/chosen": -1.4812839031219482, |
| "logits/rejected": -1.4032024145126343, |
| "logps/chosen": -213.575439453125, |
| "logps/rejected": -232.4373016357422, |
| "loss": 0.3738, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.5343422293663025, |
| "rewards/margins": 1.5936319828033447, |
| "rewards/rejected": -1.0592896938323975, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 3.3076558113098145, |
| "learning_rate": 4.925863086952132e-06, |
| "logits/chosen": -1.5180320739746094, |
| "logits/rejected": -1.362023115158081, |
| "logps/chosen": -237.8686065673828, |
| "logps/rejected": -263.8629455566406, |
| "loss": 0.3664, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.5287350416183472, |
| "rewards/margins": 2.087376356124878, |
| "rewards/rejected": -1.5586414337158203, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.33066666666666666, |
| "grad_norm": 3.5192840099334717, |
| "learning_rate": 4.918743109018727e-06, |
| "logits/chosen": -1.5185763835906982, |
| "logits/rejected": -1.4548755884170532, |
| "logps/chosen": -215.61984252929688, |
| "logps/rejected": -258.0099792480469, |
| "loss": 0.385, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.38530558347702026, |
| "rewards/margins": 1.934962511062622, |
| "rewards/rejected": -1.5496571063995361, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.3413333333333333, |
| "grad_norm": 4.284816741943359, |
| "learning_rate": 4.911302359449585e-06, |
| "logits/chosen": -1.4868816137313843, |
| "logits/rejected": -1.4484461545944214, |
| "logps/chosen": -203.9530487060547, |
| "logps/rejected": -259.6980895996094, |
| "loss": 0.321, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.456205278635025, |
| "rewards/margins": 2.133808135986328, |
| "rewards/rejected": -1.6776031255722046, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.352, |
| "grad_norm": 2.7927303314208984, |
| "learning_rate": 4.9035418250305314e-06, |
| "logits/chosen": -1.3723137378692627, |
| "logits/rejected": -1.3202406167984009, |
| "logps/chosen": -216.36062622070312, |
| "logps/rejected": -239.44772338867188, |
| "loss": 0.3563, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.6298683881759644, |
| "rewards/margins": 2.070138692855835, |
| "rewards/rejected": -1.4402703046798706, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3626666666666667, |
| "grad_norm": 3.5610151290893555, |
| "learning_rate": 4.895462534956981e-06, |
| "logits/chosen": -1.4355902671813965, |
| "logits/rejected": -1.4587870836257935, |
| "logps/chosen": -225.1851806640625, |
| "logps/rejected": -241.0689239501953, |
| "loss": 0.3507, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.5874021053314209, |
| "rewards/margins": 1.9395253658294678, |
| "rewards/rejected": -1.352123498916626, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.37333333333333335, |
| "grad_norm": 2.739126443862915, |
| "learning_rate": 4.887065560697449e-06, |
| "logits/chosen": -1.4393802881240845, |
| "logits/rejected": -1.340178370475769, |
| "logps/chosen": -218.3380584716797, |
| "logps/rejected": -254.26895141601562, |
| "loss": 0.3761, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.48171377182006836, |
| "rewards/margins": 1.7582696676254272, |
| "rewards/rejected": -1.2765560150146484, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.384, |
| "grad_norm": 3.747058391571045, |
| "learning_rate": 4.878352015851459e-06, |
| "logits/chosen": -1.4342701435089111, |
| "logits/rejected": -1.4054113626480103, |
| "logps/chosen": -238.86172485351562, |
| "logps/rejected": -251.49441528320312, |
| "loss": 0.3594, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.5655156373977661, |
| "rewards/margins": 2.102660655975342, |
| "rewards/rejected": -1.5371451377868652, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.39466666666666667, |
| "grad_norm": 2.4808998107910156, |
| "learning_rate": 4.869323056001852e-06, |
| "logits/chosen": -1.4115904569625854, |
| "logits/rejected": -1.3571182489395142, |
| "logps/chosen": -208.3723602294922, |
| "logps/rejected": -235.1692657470703, |
| "loss": 0.3642, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": 0.47888022661209106, |
| "rewards/margins": 1.8599073886871338, |
| "rewards/rejected": -1.3810272216796875, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.4053333333333333, |
| "grad_norm": 3.6028048992156982, |
| "learning_rate": 4.859979878561536e-06, |
| "logits/chosen": -1.3889703750610352, |
| "logits/rejected": -1.3901216983795166, |
| "logps/chosen": -204.32888793945312, |
| "logps/rejected": -242.7096710205078, |
| "loss": 0.3413, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": 0.5300948023796082, |
| "rewards/margins": 2.064892530441284, |
| "rewards/rejected": -1.5347976684570312, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.416, |
| "grad_norm": 2.390690326690674, |
| "learning_rate": 4.850323722614685e-06, |
| "logits/chosen": -1.3985008001327515, |
| "logits/rejected": -1.384414553642273, |
| "logps/chosen": -201.96835327148438, |
| "logps/rejected": -274.15191650390625, |
| "loss": 0.2784, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.9769827127456665, |
| "rewards/margins": 2.9426965713500977, |
| "rewards/rejected": -1.9657137393951416, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.4266666666666667, |
| "grad_norm": 3.0497686862945557, |
| "learning_rate": 4.840355868752416e-06, |
| "logits/chosen": -1.3238798379898071, |
| "logits/rejected": -1.3403199911117554, |
| "logps/chosen": -263.2410583496094, |
| "logps/rejected": -245.2187957763672, |
| "loss": 0.3321, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 0.7236034870147705, |
| "rewards/margins": 2.1606106758117676, |
| "rewards/rejected": -1.4370070695877075, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.43733333333333335, |
| "grad_norm": 3.0012412071228027, |
| "learning_rate": 4.830077638902955e-06, |
| "logits/chosen": -1.5537317991256714, |
| "logits/rejected": -1.4608463048934937, |
| "logps/chosen": -200.6154022216797, |
| "logps/rejected": -248.28836059570312, |
| "loss": 0.3128, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.4444311261177063, |
| "rewards/margins": 2.3631651401519775, |
| "rewards/rejected": -1.918733835220337, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.448, |
| "grad_norm": 2.235403060913086, |
| "learning_rate": 4.8194903961563255e-06, |
| "logits/chosen": -1.5298961400985718, |
| "logits/rejected": -1.4516569375991821, |
| "logps/chosen": -218.44970703125, |
| "logps/rejected": -259.55633544921875, |
| "loss": 0.2824, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.8634427785873413, |
| "rewards/margins": 2.612311840057373, |
| "rewards/rejected": -1.7488689422607422, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.45866666666666667, |
| "grad_norm": 2.519658088684082, |
| "learning_rate": 4.808595544583578e-06, |
| "logits/chosen": -1.4698635339736938, |
| "logits/rejected": -1.380286455154419, |
| "logps/chosen": -204.63861083984375, |
| "logps/rejected": -256.0403137207031, |
| "loss": 0.2968, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.7937697172164917, |
| "rewards/margins": 2.663573980331421, |
| "rewards/rejected": -1.8698041439056396, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.4693333333333333, |
| "grad_norm": 4.095766544342041, |
| "learning_rate": 4.797394529050577e-06, |
| "logits/chosen": -1.463175892829895, |
| "logits/rejected": -1.4339282512664795, |
| "logps/chosen": -244.7183837890625, |
| "logps/rejected": -261.1028747558594, |
| "loss": 0.3653, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.38948220014572144, |
| "rewards/margins": 2.2226202487945557, |
| "rewards/rejected": -1.8331377506256104, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.48, |
| "grad_norm": 2.851942300796509, |
| "learning_rate": 4.785888835026393e-06, |
| "logits/chosen": -1.3653357028961182, |
| "logits/rejected": -1.330588936805725, |
| "logps/chosen": -217.13760375976562, |
| "logps/rejected": -240.1661376953125, |
| "loss": 0.2874, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.732596755027771, |
| "rewards/margins": 2.40860915184021, |
| "rewards/rejected": -1.676012396812439, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.49066666666666664, |
| "grad_norm": 3.4217729568481445, |
| "learning_rate": 4.7740799883862966e-06, |
| "logits/chosen": -1.4823631048202515, |
| "logits/rejected": -1.495508074760437, |
| "logps/chosen": -212.22323608398438, |
| "logps/rejected": -263.6670837402344, |
| "loss": 0.2746, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.5829408168792725, |
| "rewards/margins": 2.398766040802002, |
| "rewards/rejected": -1.8158252239227295, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.5013333333333333, |
| "grad_norm": 2.7907536029815674, |
| "learning_rate": 4.761969555209394e-06, |
| "logits/chosen": -1.5069305896759033, |
| "logits/rejected": -1.464449167251587, |
| "logps/chosen": -193.39453125, |
| "logps/rejected": -249.33828735351562, |
| "loss": 0.3033, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.5029724836349487, |
| "rewards/margins": 2.535928726196289, |
| "rewards/rejected": -2.03295636177063, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.512, |
| "grad_norm": 2.6872572898864746, |
| "learning_rate": 4.7495591415709435e-06, |
| "logits/chosen": -1.4873557090759277, |
| "logits/rejected": -1.4141255617141724, |
| "logps/chosen": -218.90084838867188, |
| "logps/rejected": -248.9842071533203, |
| "loss": 0.2922, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.41843342781066895, |
| "rewards/margins": 2.512903928756714, |
| "rewards/rejected": -2.094470739364624, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5226666666666666, |
| "grad_norm": 2.2702150344848633, |
| "learning_rate": 4.736850393329353e-06, |
| "logits/chosen": -1.500101923942566, |
| "logits/rejected": -1.3862520456314087, |
| "logps/chosen": -211.1094970703125, |
| "logps/rejected": -270.4104919433594, |
| "loss": 0.2471, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.5343133211135864, |
| "rewards/margins": 2.90330171585083, |
| "rewards/rejected": -2.368988513946533, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5333333333333333, |
| "grad_norm": 3.828376054763794, |
| "learning_rate": 4.7238449959079095e-06, |
| "logits/chosen": -1.538053274154663, |
| "logits/rejected": -1.4550931453704834, |
| "logps/chosen": -237.7978973388672, |
| "logps/rejected": -277.4515686035156, |
| "loss": 0.2882, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.445631206035614, |
| "rewards/margins": 3.2756295204162598, |
| "rewards/rejected": -2.829998254776001, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.544, |
| "grad_norm": 7.073206901550293, |
| "learning_rate": 4.710544674071259e-06, |
| "logits/chosen": -1.6128956079483032, |
| "logits/rejected": -1.5589414834976196, |
| "logps/chosen": -218.8146514892578, |
| "logps/rejected": -247.97824096679688, |
| "loss": 0.2747, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.433823823928833, |
| "rewards/margins": 2.5846328735351562, |
| "rewards/rejected": -2.150808811187744, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5546666666666666, |
| "grad_norm": 3.671692132949829, |
| "learning_rate": 4.696951191696673e-06, |
| "logits/chosen": -1.5363198518753052, |
| "logits/rejected": -1.432539939880371, |
| "logps/chosen": -194.21780395507812, |
| "logps/rejected": -259.4532470703125, |
| "loss": 0.2959, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.29416778683662415, |
| "rewards/margins": 2.8601794242858887, |
| "rewards/rejected": -2.566011905670166, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5653333333333334, |
| "grad_norm": 4.124833583831787, |
| "learning_rate": 4.6830663515401196e-06, |
| "logits/chosen": -1.5194003582000732, |
| "logits/rejected": -1.4758837223052979, |
| "logps/chosen": -222.5775146484375, |
| "logps/rejected": -270.2529296875, |
| "loss": 0.2939, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.4511001706123352, |
| "rewards/margins": 2.7838187217712402, |
| "rewards/rejected": -2.3327183723449707, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.576, |
| "grad_norm": 3.6710762977600098, |
| "learning_rate": 4.6688919949971855e-06, |
| "logits/chosen": -1.5958110094070435, |
| "logits/rejected": -1.4789632558822632, |
| "logps/chosen": -229.09042358398438, |
| "logps/rejected": -255.7130889892578, |
| "loss": 0.2994, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": 0.42495328187942505, |
| "rewards/margins": 2.7729554176330566, |
| "rewards/rejected": -2.3480019569396973, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5866666666666667, |
| "grad_norm": 2.481130838394165, |
| "learning_rate": 4.654430001858874e-06, |
| "logits/chosen": -1.3712882995605469, |
| "logits/rejected": -1.3942594528198242, |
| "logps/chosen": -221.78616333007812, |
| "logps/rejected": -271.4743347167969, |
| "loss": 0.2772, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.5858339071273804, |
| "rewards/margins": 2.6469860076904297, |
| "rewards/rejected": -2.0611519813537598, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5973333333333334, |
| "grad_norm": 3.2142515182495117, |
| "learning_rate": 4.6396822900623075e-06, |
| "logits/chosen": -1.5062901973724365, |
| "logits/rejected": -1.4870845079421997, |
| "logps/chosen": -219.1357421875, |
| "logps/rejected": -263.96307373046875, |
| "loss": 0.2826, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.28349483013153076, |
| "rewards/margins": 2.623389482498169, |
| "rewards/rejected": -2.3398945331573486, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.608, |
| "grad_norm": 2.7220044136047363, |
| "learning_rate": 4.624650815436368e-06, |
| "logits/chosen": -1.4841105937957764, |
| "logits/rejected": -1.4876569509506226, |
| "logps/chosen": -227.7067108154297, |
| "logps/rejected": -277.1427917480469, |
| "loss": 0.2772, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.5623529553413391, |
| "rewards/margins": 2.996119260787964, |
| "rewards/rejected": -2.4337663650512695, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6186666666666667, |
| "grad_norm": 3.9186594486236572, |
| "learning_rate": 4.609337571442324e-06, |
| "logits/chosen": -1.5781495571136475, |
| "logits/rejected": -1.5577192306518555, |
| "logps/chosen": -215.00234985351562, |
| "logps/rejected": -272.5433044433594, |
| "loss": 0.3279, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 0.29643192887306213, |
| "rewards/margins": 2.8020668029785156, |
| "rewards/rejected": -2.5056347846984863, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6293333333333333, |
| "grad_norm": 10.114188194274902, |
| "learning_rate": 4.5937445889094535e-06, |
| "logits/chosen": -1.5302282571792603, |
| "logits/rejected": -1.520778775215149, |
| "logps/chosen": -238.6151123046875, |
| "logps/rejected": -278.87847900390625, |
| "loss": 0.3095, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.2077063024044037, |
| "rewards/margins": 2.5661838054656982, |
| "rewards/rejected": -2.3584771156311035, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.64, |
| "grad_norm": 2.399756669998169, |
| "learning_rate": 4.577873935765722e-06, |
| "logits/chosen": -1.4977320432662964, |
| "logits/rejected": -1.389096975326538, |
| "logps/chosen": -215.07699584960938, |
| "logps/rejected": -257.64459228515625, |
| "loss": 0.267, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.5605412125587463, |
| "rewards/margins": 3.175250291824341, |
| "rewards/rejected": -2.614708662033081, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.6506666666666666, |
| "grad_norm": 2.632978916168213, |
| "learning_rate": 4.561727716763532e-06, |
| "logits/chosen": -1.5880491733551025, |
| "logits/rejected": -1.4870240688323975, |
| "logps/chosen": -235.69680786132812, |
| "logps/rejected": -260.10711669921875, |
| "loss": 0.3318, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": 0.39901071786880493, |
| "rewards/margins": 2.604020118713379, |
| "rewards/rejected": -2.2050094604492188, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.6613333333333333, |
| "grad_norm": 2.3961174488067627, |
| "learning_rate": 4.545308073200591e-06, |
| "logits/chosen": -1.4467146396636963, |
| "logits/rejected": -1.3576264381408691, |
| "logps/chosen": -224.96505737304688, |
| "logps/rejected": -259.876953125, |
| "loss": 0.2734, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.3828810751438141, |
| "rewards/margins": 2.681600570678711, |
| "rewards/rejected": -2.298719644546509, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.672, |
| "grad_norm": 3.1757161617279053, |
| "learning_rate": 4.528617182635943e-06, |
| "logits/chosen": -1.527538537979126, |
| "logits/rejected": -1.528403639793396, |
| "logps/chosen": -222.27877807617188, |
| "logps/rejected": -258.99517822265625, |
| "loss": 0.3181, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": 0.3306906521320343, |
| "rewards/margins": 2.67979097366333, |
| "rewards/rejected": -2.349100351333618, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6826666666666666, |
| "grad_norm": 2.3359498977661133, |
| "learning_rate": 4.511657258601171e-06, |
| "logits/chosen": -1.5962450504302979, |
| "logits/rejected": -1.4786200523376465, |
| "logps/chosen": -220.0579376220703, |
| "logps/rejected": -273.56915283203125, |
| "loss": 0.2359, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.6050397157669067, |
| "rewards/margins": 3.208775758743286, |
| "rewards/rejected": -2.60373592376709, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6933333333333334, |
| "grad_norm": 4.228616237640381, |
| "learning_rate": 4.494430550306848e-06, |
| "logits/chosen": -1.4205467700958252, |
| "logits/rejected": -1.4639966487884521, |
| "logps/chosen": -235.7682647705078, |
| "logps/rejected": -263.72576904296875, |
| "loss": 0.2699, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.4260522723197937, |
| "rewards/margins": 2.8981242179870605, |
| "rewards/rejected": -2.472072124481201, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.704, |
| "grad_norm": 3.994901180267334, |
| "learning_rate": 4.476939342344246e-06, |
| "logits/chosen": -1.6341301202774048, |
| "logits/rejected": -1.5283631086349487, |
| "logps/chosen": -211.6797332763672, |
| "logps/rejected": -266.39544677734375, |
| "loss": 0.2507, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.3721843361854553, |
| "rewards/margins": 3.0854640007019043, |
| "rewards/rejected": -2.7132792472839355, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7146666666666667, |
| "grad_norm": 2.890618085861206, |
| "learning_rate": 4.45918595438236e-06, |
| "logits/chosen": -1.5078494548797607, |
| "logits/rejected": -1.4805063009262085, |
| "logps/chosen": -214.09115600585938, |
| "logps/rejected": -252.56002807617188, |
| "loss": 0.2518, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.32611769437789917, |
| "rewards/margins": 2.888943910598755, |
| "rewards/rejected": -2.562826633453369, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7253333333333334, |
| "grad_norm": 4.099746227264404, |
| "learning_rate": 4.441172740860267e-06, |
| "logits/chosen": -1.5950417518615723, |
| "logits/rejected": -1.467929482460022, |
| "logps/chosen": -230.2890167236328, |
| "logps/rejected": -252.35507202148438, |
| "loss": 0.3413, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.19502702355384827, |
| "rewards/margins": 2.6124227046966553, |
| "rewards/rejected": -2.417395830154419, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.736, |
| "grad_norm": 2.731096029281616, |
| "learning_rate": 4.422902090674886e-06, |
| "logits/chosen": -1.4848092794418335, |
| "logits/rejected": -1.4913749694824219, |
| "logps/chosen": -222.9447784423828, |
| "logps/rejected": -275.4829406738281, |
| "loss": 0.2407, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.09350375831127167, |
| "rewards/margins": 3.030643939971924, |
| "rewards/rejected": -2.9371399879455566, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7466666666666667, |
| "grad_norm": 2.716784954071045, |
| "learning_rate": 4.4043764268641705e-06, |
| "logits/chosen": -1.5931785106658936, |
| "logits/rejected": -1.445983648300171, |
| "logps/chosen": -231.0179443359375, |
| "logps/rejected": -247.4789276123047, |
| "loss": 0.2612, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.10627200454473495, |
| "rewards/margins": 2.9888815879821777, |
| "rewards/rejected": -2.8826098442077637, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7573333333333333, |
| "grad_norm": 4.22883939743042, |
| "learning_rate": 4.385598206285758e-06, |
| "logits/chosen": -1.6081180572509766, |
| "logits/rejected": -1.557462215423584, |
| "logps/chosen": -217.7015838623047, |
| "logps/rejected": -258.1307678222656, |
| "loss": 0.2923, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.25779181718826294, |
| "rewards/margins": 2.9773926734924316, |
| "rewards/rejected": -2.7196006774902344, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.768, |
| "grad_norm": 3.020744562149048, |
| "learning_rate": 4.3665699192911495e-06, |
| "logits/chosen": -1.454756259918213, |
| "logits/rejected": -1.4177753925323486, |
| "logps/chosen": -209.04257202148438, |
| "logps/rejected": -252.439208984375, |
| "loss": 0.2649, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.3628079891204834, |
| "rewards/margins": 2.9099552631378174, |
| "rewards/rejected": -2.547147274017334, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7786666666666666, |
| "grad_norm": 2.0658082962036133, |
| "learning_rate": 4.347294089395438e-06, |
| "logits/chosen": -1.460191011428833, |
| "logits/rejected": -1.4749881029129028, |
| "logps/chosen": -223.18679809570312, |
| "logps/rejected": -285.80413818359375, |
| "loss": 0.1976, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.4657130241394043, |
| "rewards/margins": 3.4711146354675293, |
| "rewards/rejected": -3.005401611328125, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7893333333333333, |
| "grad_norm": 3.1469435691833496, |
| "learning_rate": 4.327773272942647e-06, |
| "logits/chosen": -1.5563991069793701, |
| "logits/rejected": -1.5385240316390991, |
| "logps/chosen": -225.7266387939453, |
| "logps/rejected": -277.9988708496094, |
| "loss": 0.2847, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": 0.30648013949394226, |
| "rewards/margins": 2.885929822921753, |
| "rewards/rejected": -2.5794498920440674, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.8, |
| "grad_norm": 3.111003875732422, |
| "learning_rate": 4.308010058766703e-06, |
| "logits/chosen": -1.6107202768325806, |
| "logits/rejected": -1.537246823310852, |
| "logps/chosen": -238.1480255126953, |
| "logps/rejected": -271.89373779296875, |
| "loss": 0.2056, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.4036385118961334, |
| "rewards/margins": 3.371934413909912, |
| "rewards/rejected": -2.9682960510253906, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8106666666666666, |
| "grad_norm": 3.8910281658172607, |
| "learning_rate": 4.2880070678481135e-06, |
| "logits/chosen": -1.5961246490478516, |
| "logits/rejected": -1.5346485376358032, |
| "logps/chosen": -212.40737915039062, |
| "logps/rejected": -242.66879272460938, |
| "loss": 0.2987, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.1651727855205536, |
| "rewards/margins": 2.9011754989624023, |
| "rewards/rejected": -2.7360026836395264, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8213333333333334, |
| "grad_norm": 3.2986905574798584, |
| "learning_rate": 4.267766952966369e-06, |
| "logits/chosen": -1.5123292207717896, |
| "logits/rejected": -1.4317362308502197, |
| "logps/chosen": -215.16653442382812, |
| "logps/rejected": -250.494873046875, |
| "loss": 0.297, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.2533227205276489, |
| "rewards/margins": 3.0472066402435303, |
| "rewards/rejected": -2.793883800506592, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.832, |
| "grad_norm": 2.390202283859253, |
| "learning_rate": 4.24729239834814e-06, |
| "logits/chosen": -1.4423052072525024, |
| "logits/rejected": -1.4100033044815063, |
| "logps/chosen": -220.01803588867188, |
| "logps/rejected": -267.92913818359375, |
| "loss": 0.273, |
| "rewards/accuracies": 0.8812500238418579, |
| "rewards/chosen": 0.3290650248527527, |
| "rewards/margins": 3.225865125656128, |
| "rewards/rejected": -2.8968000411987305, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8426666666666667, |
| "grad_norm": 2.8150224685668945, |
| "learning_rate": 4.226586119311291e-06, |
| "logits/chosen": -1.522770643234253, |
| "logits/rejected": -1.5387157201766968, |
| "logps/chosen": -221.63082885742188, |
| "logps/rejected": -273.49322509765625, |
| "loss": 0.2344, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.5276051163673401, |
| "rewards/margins": 3.0984864234924316, |
| "rewards/rejected": -2.5708811283111572, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8533333333333334, |
| "grad_norm": 3.9126734733581543, |
| "learning_rate": 4.205650861904782e-06, |
| "logits/chosen": -1.5304027795791626, |
| "logits/rejected": -1.4529463052749634, |
| "logps/chosen": -212.8784942626953, |
| "logps/rejected": -272.4510803222656, |
| "loss": 0.2645, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": 0.4774355888366699, |
| "rewards/margins": 3.302903413772583, |
| "rewards/rejected": -2.825467348098755, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.864, |
| "grad_norm": 1.941848635673523, |
| "learning_rate": 4.184489402544486e-06, |
| "logits/chosen": -1.5869189500808716, |
| "logits/rejected": -1.520146131515503, |
| "logps/chosen": -206.26156616210938, |
| "logps/rejected": -257.34173583984375, |
| "loss": 0.1988, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.7204893827438354, |
| "rewards/margins": 3.477137327194214, |
| "rewards/rejected": -2.756648063659668, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.8746666666666667, |
| "grad_norm": 3.2625303268432617, |
| "learning_rate": 4.1631045476449865e-06, |
| "logits/chosen": -1.5185470581054688, |
| "logits/rejected": -1.4453086853027344, |
| "logps/chosen": -204.9614715576172, |
| "logps/rejected": -272.30706787109375, |
| "loss": 0.2353, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.33646291494369507, |
| "rewards/margins": 3.2520689964294434, |
| "rewards/rejected": -2.9156060218811035, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.8853333333333333, |
| "grad_norm": 2.795285224914551, |
| "learning_rate": 4.141499133247391e-06, |
| "logits/chosen": -1.4114835262298584, |
| "logits/rejected": -1.4512608051300049, |
| "logps/chosen": -217.20480346679688, |
| "logps/rejected": -261.76727294921875, |
| "loss": 0.2568, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.555108904838562, |
| "rewards/margins": 3.5297584533691406, |
| "rewards/rejected": -2.9746499061584473, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.896, |
| "grad_norm": 4.065403461456299, |
| "learning_rate": 4.119676024643217e-06, |
| "logits/chosen": -1.4818923473358154, |
| "logits/rejected": -1.4995462894439697, |
| "logps/chosen": -209.0481719970703, |
| "logps/rejected": -255.38623046875, |
| "loss": 0.2527, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.5125702023506165, |
| "rewards/margins": 2.971860647201538, |
| "rewards/rejected": -2.459290027618408, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.9066666666666666, |
| "grad_norm": 4.261399745941162, |
| "learning_rate": 4.0976381159944055e-06, |
| "logits/chosen": -1.6032196283340454, |
| "logits/rejected": -1.5087716579437256, |
| "logps/chosen": -229.303955078125, |
| "logps/rejected": -273.9316711425781, |
| "loss": 0.2773, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.056284308433532715, |
| "rewards/margins": 2.942106246948242, |
| "rewards/rejected": -2.885822057723999, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9173333333333333, |
| "grad_norm": 1.7086869478225708, |
| "learning_rate": 4.07538832994949e-06, |
| "logits/chosen": -1.5353028774261475, |
| "logits/rejected": -1.4393786191940308, |
| "logps/chosen": -208.2356719970703, |
| "logps/rejected": -272.1566467285156, |
| "loss": 0.23, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.37963634729385376, |
| "rewards/margins": 3.723524808883667, |
| "rewards/rejected": -3.343888521194458, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.928, |
| "grad_norm": 2.9886105060577393, |
| "learning_rate": 4.0529296172560055e-06, |
| "logits/chosen": -1.6221126317977905, |
| "logits/rejected": -1.5396804809570312, |
| "logps/chosen": -219.42343139648438, |
| "logps/rejected": -275.2545471191406, |
| "loss": 0.2345, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.3662927746772766, |
| "rewards/margins": 3.762205123901367, |
| "rewards/rejected": -3.3959121704101562, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9386666666666666, |
| "grad_norm": 2.806347370147705, |
| "learning_rate": 4.030264956369158e-06, |
| "logits/chosen": -1.5537915229797363, |
| "logits/rejected": -1.4640367031097412, |
| "logps/chosen": -222.9738006591797, |
| "logps/rejected": -288.2947692871094, |
| "loss": 0.244, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": 0.5269616842269897, |
| "rewards/margins": 3.6222381591796875, |
| "rewards/rejected": -3.095276355743408, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9493333333333334, |
| "grad_norm": 3.4051554203033447, |
| "learning_rate": 4.0073973530568274e-06, |
| "logits/chosen": -1.5768224000930786, |
| "logits/rejected": -1.4694640636444092, |
| "logps/chosen": -222.8614959716797, |
| "logps/rejected": -276.8932189941406, |
| "loss": 0.2644, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": 0.48596447706222534, |
| "rewards/margins": 3.459862232208252, |
| "rewards/rejected": -2.9738974571228027, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.96, |
| "grad_norm": 1.5982437133789062, |
| "learning_rate": 3.9843298400009435e-06, |
| "logits/chosen": -1.5265693664550781, |
| "logits/rejected": -1.4454200267791748, |
| "logps/chosen": -248.08175659179688, |
| "logps/rejected": -263.53564453125, |
| "loss": 0.2418, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.9819803237915039, |
| "rewards/margins": 3.853539228439331, |
| "rewards/rejected": -2.8715591430664062, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9706666666666667, |
| "grad_norm": 3.993597984313965, |
| "learning_rate": 3.961065476395296e-06, |
| "logits/chosen": -1.6176679134368896, |
| "logits/rejected": -1.5645582675933838, |
| "logps/chosen": -229.76318359375, |
| "logps/rejected": -262.1435852050781, |
| "loss": 0.29, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": 0.5866170525550842, |
| "rewards/margins": 3.056516408920288, |
| "rewards/rejected": -2.4698996543884277, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9813333333333333, |
| "grad_norm": 2.713717222213745, |
| "learning_rate": 3.937607347539823e-06, |
| "logits/chosen": -1.4770532846450806, |
| "logits/rejected": -1.4442005157470703, |
| "logps/chosen": -208.3784942626953, |
| "logps/rejected": -270.6184387207031, |
| "loss": 0.2425, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.6007565855979919, |
| "rewards/margins": 3.5067782402038574, |
| "rewards/rejected": -2.9060218334198, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.992, |
| "grad_norm": 2.6341986656188965, |
| "learning_rate": 3.913958564431447e-06, |
| "logits/chosen": -1.5352776050567627, |
| "logits/rejected": -1.5059245824813843, |
| "logps/chosen": -231.0626678466797, |
| "logps/rejected": -271.16015625, |
| "loss": 0.2402, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.5198360681533813, |
| "rewards/margins": 3.263561964035034, |
| "rewards/rejected": -2.7437267303466797, |
| "step": 465 |
| }, |
| { |
| "epoch": 1.0021333333333333, |
| "grad_norm": 2.800222396850586, |
| "learning_rate": 3.89012226335149e-06, |
| "logits/chosen": -1.507896900177002, |
| "logits/rejected": -1.4542078971862793, |
| "logps/chosen": -223.8075408935547, |
| "logps/rejected": -264.8511657714844, |
| "loss": 0.2531, |
| "rewards/accuracies": 0.8881579041481018, |
| "rewards/chosen": 0.7271106839179993, |
| "rewards/margins": 3.1902239322662354, |
| "rewards/rejected": -2.4631130695343018, |
| "step": 470 |
| }, |
| { |
| "epoch": 1.0128, |
| "grad_norm": 1.8111785650253296, |
| "learning_rate": 3.86610160544975e-06, |
| "logits/chosen": -1.5138390064239502, |
| "logits/rejected": -1.5100157260894775, |
| "logps/chosen": -222.42367553710938, |
| "logps/rejected": -270.0904235839844, |
| "loss": 0.1977, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.9269617199897766, |
| "rewards/margins": 3.785017728805542, |
| "rewards/rejected": -2.8580563068389893, |
| "step": 475 |
| }, |
| { |
| "epoch": 1.0234666666666667, |
| "grad_norm": 2.3462116718292236, |
| "learning_rate": 3.841899776325267e-06, |
| "logits/chosen": -1.4097557067871094, |
| "logits/rejected": -1.2989996671676636, |
| "logps/chosen": -242.18115234375, |
| "logps/rejected": -254.24459838867188, |
| "loss": 0.2003, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.9647325277328491, |
| "rewards/margins": 3.434415102005005, |
| "rewards/rejected": -2.4696826934814453, |
| "step": 480 |
| }, |
| { |
| "epoch": 1.0341333333333333, |
| "grad_norm": 2.4273829460144043, |
| "learning_rate": 3.817519985603855e-06, |
| "logits/chosen": -1.6104650497436523, |
| "logits/rejected": -1.5876798629760742, |
| "logps/chosen": -199.7315673828125, |
| "logps/rejected": -262.48309326171875, |
| "loss": 0.2409, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.648854672908783, |
| "rewards/margins": 3.4328675270080566, |
| "rewards/rejected": -2.78401255607605, |
| "step": 485 |
| }, |
| { |
| "epoch": 1.0448, |
| "grad_norm": 2.23551344871521, |
| "learning_rate": 3.792965466512437e-06, |
| "logits/chosen": -1.4702277183532715, |
| "logits/rejected": -1.3995940685272217, |
| "logps/chosen": -241.27835083007812, |
| "logps/rejected": -275.19061279296875, |
| "loss": 0.1657, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.8960878252983093, |
| "rewards/margins": 3.913990020751953, |
| "rewards/rejected": -3.01790189743042, |
| "step": 490 |
| }, |
| { |
| "epoch": 1.0554666666666668, |
| "grad_norm": 2.930630922317505, |
| "learning_rate": 3.7682394754502687e-06, |
| "logits/chosen": -1.5484654903411865, |
| "logits/rejected": -1.5531933307647705, |
| "logps/chosen": -212.1646728515625, |
| "logps/rejected": -272.5528564453125, |
| "loss": 0.2003, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.4871641993522644, |
| "rewards/margins": 3.573843002319336, |
| "rewards/rejected": -3.0866787433624268, |
| "step": 495 |
| }, |
| { |
| "epoch": 1.0661333333333334, |
| "grad_norm": 1.9930554628372192, |
| "learning_rate": 3.7433452915570685e-06, |
| "logits/chosen": -1.5165655612945557, |
| "logits/rejected": -1.48264479637146, |
| "logps/chosen": -225.77340698242188, |
| "logps/rejected": -268.895751953125, |
| "loss": 0.1915, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.714760959148407, |
| "rewards/margins": 3.824763059616089, |
| "rewards/rejected": -3.110002040863037, |
| "step": 500 |
| }, |
| { |
| "epoch": 1.0768, |
| "grad_norm": 3.423259973526001, |
| "learning_rate": 3.718286216278143e-06, |
| "logits/chosen": -1.5718601942062378, |
| "logits/rejected": -1.503844976425171, |
| "logps/chosen": -208.6433563232422, |
| "logps/rejected": -278.9520263671875, |
| "loss": 0.2557, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": 0.4504665434360504, |
| "rewards/margins": 3.5247490406036377, |
| "rewards/rejected": -3.0742828845977783, |
| "step": 505 |
| }, |
| { |
| "epoch": 1.0874666666666666, |
| "grad_norm": 2.5199198722839355, |
| "learning_rate": 3.6930655729265557e-06, |
| "logits/chosen": -1.4899485111236572, |
| "logits/rejected": -1.4122374057769775, |
| "logps/chosen": -227.7731170654297, |
| "logps/rejected": -277.1492004394531, |
| "loss": 0.1901, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.5795149207115173, |
| "rewards/margins": 4.235302925109863, |
| "rewards/rejected": -3.6557884216308594, |
| "step": 510 |
| }, |
| { |
| "epoch": 1.0981333333333334, |
| "grad_norm": 3.217991590499878, |
| "learning_rate": 3.667686706242386e-06, |
| "logits/chosen": -1.4657047986984253, |
| "logits/rejected": -1.423705816268921, |
| "logps/chosen": -208.4857177734375, |
| "logps/rejected": -274.12799072265625, |
| "loss": 0.1808, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.6760324239730835, |
| "rewards/margins": 3.973620653152466, |
| "rewards/rejected": -3.2975876331329346, |
| "step": 515 |
| }, |
| { |
| "epoch": 1.1088, |
| "grad_norm": 2.835545301437378, |
| "learning_rate": 3.6421529819491576e-06, |
| "logits/chosen": -1.5454612970352173, |
| "logits/rejected": -1.5034112930297852, |
| "logps/chosen": -229.1444854736328, |
| "logps/rejected": -275.30047607421875, |
| "loss": 0.1639, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.3288564383983612, |
| "rewards/margins": 3.9869868755340576, |
| "rewards/rejected": -3.658130168914795, |
| "step": 520 |
| }, |
| { |
| "epoch": 1.1194666666666666, |
| "grad_norm": 2.960411787033081, |
| "learning_rate": 3.6164677863074783e-06, |
| "logits/chosen": -1.5239708423614502, |
| "logits/rejected": -1.500842571258545, |
| "logps/chosen": -209.4385528564453, |
| "logps/rejected": -283.56854248046875, |
| "loss": 0.1454, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.5432895421981812, |
| "rewards/margins": 4.694332599639893, |
| "rewards/rejected": -4.15104341506958, |
| "step": 525 |
| }, |
| { |
| "epoch": 1.1301333333333332, |
| "grad_norm": 2.7328364849090576, |
| "learning_rate": 3.590634525665955e-06, |
| "logits/chosen": -1.6704280376434326, |
| "logits/rejected": -1.5336025953292847, |
| "logps/chosen": -218.2663116455078, |
| "logps/rejected": -287.0753479003906, |
| "loss": 0.1882, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.1658170372247696, |
| "rewards/margins": 4.253686428070068, |
| "rewards/rejected": -4.087869167327881, |
| "step": 530 |
| }, |
| { |
| "epoch": 1.1408, |
| "grad_norm": 3.2963521480560303, |
| "learning_rate": 3.564656626009446e-06, |
| "logits/chosen": -1.605804443359375, |
| "logits/rejected": -1.5619615316390991, |
| "logps/chosen": -203.71310424804688, |
| "logps/rejected": -267.4145812988281, |
| "loss": 0.2009, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.14755108952522278, |
| "rewards/margins": 4.140759468078613, |
| "rewards/rejected": -4.288310527801514, |
| "step": 535 |
| }, |
| { |
| "epoch": 1.1514666666666666, |
| "grad_norm": 4.603735446929932, |
| "learning_rate": 3.5385375325047167e-06, |
| "logits/chosen": -1.6345436573028564, |
| "logits/rejected": -1.5837303400039673, |
| "logps/chosen": -223.50537109375, |
| "logps/rejected": -281.343017578125, |
| "loss": 0.183, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.307804673910141, |
| "rewards/margins": 4.121958255767822, |
| "rewards/rejected": -4.429762840270996, |
| "step": 540 |
| }, |
| { |
| "epoch": 1.1621333333333332, |
| "grad_norm": 3.5765788555145264, |
| "learning_rate": 3.512280709043534e-06, |
| "logits/chosen": -1.5483214855194092, |
| "logits/rejected": -1.451832890510559, |
| "logps/chosen": -228.17013549804688, |
| "logps/rejected": -270.2801818847656, |
| "loss": 0.2415, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.13486400246620178, |
| "rewards/margins": 3.71563982963562, |
| "rewards/rejected": -3.850503921508789, |
| "step": 545 |
| }, |
| { |
| "epoch": 1.1728, |
| "grad_norm": 2.7986321449279785, |
| "learning_rate": 3.4858896377832966e-06, |
| "logits/chosen": -1.5429253578186035, |
| "logits/rejected": -1.5843229293823242, |
| "logps/chosen": -225.6952362060547, |
| "logps/rejected": -317.2574157714844, |
| "loss": 0.1625, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.02160305716097355, |
| "rewards/margins": 5.061923027038574, |
| "rewards/rejected": -5.083526611328125, |
| "step": 550 |
| }, |
| { |
| "epoch": 1.1834666666666667, |
| "grad_norm": 3.046375274658203, |
| "learning_rate": 3.459367818685231e-06, |
| "logits/chosen": -1.5679842233657837, |
| "logits/rejected": -1.565364956855774, |
| "logps/chosen": -221.58963012695312, |
| "logps/rejected": -277.75091552734375, |
| "loss": 0.2252, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.12510250508785248, |
| "rewards/margins": 3.9496397972106934, |
| "rewards/rejected": -4.074742317199707, |
| "step": 555 |
| }, |
| { |
| "epoch": 1.1941333333333333, |
| "grad_norm": 2.593656301498413, |
| "learning_rate": 3.432718769050228e-06, |
| "logits/chosen": -1.3937714099884033, |
| "logits/rejected": -1.4195549488067627, |
| "logps/chosen": -221.0165252685547, |
| "logps/rejected": -310.3327331542969, |
| "loss": 0.1942, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.2540460228919983, |
| "rewards/margins": 4.681498050689697, |
| "rewards/rejected": -4.935544490814209, |
| "step": 560 |
| }, |
| { |
| "epoch": 1.2048, |
| "grad_norm": 4.23615026473999, |
| "learning_rate": 3.405946023052386e-06, |
| "logits/chosen": -1.5223419666290283, |
| "logits/rejected": -1.4661571979522705, |
| "logps/chosen": -217.5615234375, |
| "logps/rejected": -292.19683837890625, |
| "loss": 0.1682, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.1407851129770279, |
| "rewards/margins": 4.647849082946777, |
| "rewards/rejected": -4.507063865661621, |
| "step": 565 |
| }, |
| { |
| "epoch": 1.2154666666666667, |
| "grad_norm": 3.9983882904052734, |
| "learning_rate": 3.3790531312703055e-06, |
| "logits/chosen": -1.5181207656860352, |
| "logits/rejected": -1.4914724826812744, |
| "logps/chosen": -210.10433959960938, |
| "logps/rejected": -273.66107177734375, |
| "loss": 0.1985, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.14912539720535278, |
| "rewards/margins": 4.561871528625488, |
| "rewards/rejected": -4.412745952606201, |
| "step": 570 |
| }, |
| { |
| "epoch": 1.2261333333333333, |
| "grad_norm": 4.649803161621094, |
| "learning_rate": 3.35204366021622e-06, |
| "logits/chosen": -1.4583940505981445, |
| "logits/rejected": -1.4670298099517822, |
| "logps/chosen": -239.1728057861328, |
| "logps/rejected": -291.33392333984375, |
| "loss": 0.2066, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.08136308193206787, |
| "rewards/margins": 4.34789514541626, |
| "rewards/rejected": -4.429257869720459, |
| "step": 575 |
| }, |
| { |
| "epoch": 1.2368000000000001, |
| "grad_norm": 2.3416860103607178, |
| "learning_rate": 3.3249211918630055e-06, |
| "logits/chosen": -1.5698027610778809, |
| "logits/rejected": -1.491105318069458, |
| "logps/chosen": -220.2606658935547, |
| "logps/rejected": -281.61572265625, |
| "loss": 0.1992, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.2473183125257492, |
| "rewards/margins": 4.019981384277344, |
| "rewards/rejected": -4.267299175262451, |
| "step": 580 |
| }, |
| { |
| "epoch": 1.2474666666666667, |
| "grad_norm": 2.0436906814575195, |
| "learning_rate": 3.2976893231691424e-06, |
| "logits/chosen": -1.5137239694595337, |
| "logits/rejected": -1.465408444404602, |
| "logps/chosen": -221.51223754882812, |
| "logps/rejected": -292.5458679199219, |
| "loss": 0.1629, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.05579204484820366, |
| "rewards/margins": 4.571835517883301, |
| "rewards/rejected": -4.516043663024902, |
| "step": 585 |
| }, |
| { |
| "epoch": 1.2581333333333333, |
| "grad_norm": 5.584434509277344, |
| "learning_rate": 3.270351665601691e-06, |
| "logits/chosen": -1.53299880027771, |
| "logits/rejected": -1.477111577987671, |
| "logps/chosen": -277.4779968261719, |
| "logps/rejected": -267.1672668457031, |
| "loss": 0.2395, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.21413402259349823, |
| "rewards/margins": 4.064342021942139, |
| "rewards/rejected": -3.850208282470703, |
| "step": 590 |
| }, |
| { |
| "epoch": 1.2688, |
| "grad_norm": 3.1149520874023438, |
| "learning_rate": 3.2429118446573417e-06, |
| "logits/chosen": -1.569455862045288, |
| "logits/rejected": -1.5134979486465454, |
| "logps/chosen": -219.28079223632812, |
| "logps/rejected": -292.9593811035156, |
| "loss": 0.1917, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.11000633239746094, |
| "rewards/margins": 4.244585037231445, |
| "rewards/rejected": -4.354591369628906, |
| "step": 595 |
| }, |
| { |
| "epoch": 1.2794666666666665, |
| "grad_norm": 3.55899977684021, |
| "learning_rate": 3.215373499381602e-06, |
| "logits/chosen": -1.4652760028839111, |
| "logits/rejected": -1.428091287612915, |
| "logps/chosen": -225.0402069091797, |
| "logps/rejected": -285.8553466796875, |
| "loss": 0.183, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.03203168511390686, |
| "rewards/margins": 4.701508522033691, |
| "rewards/rejected": -4.7335405349731445, |
| "step": 600 |
| }, |
| { |
| "epoch": 1.2901333333333334, |
| "grad_norm": 3.1927192211151123, |
| "learning_rate": 3.1877402818861954e-06, |
| "logits/chosen": -1.5257372856140137, |
| "logits/rejected": -1.502838134765625, |
| "logps/chosen": -232.27536010742188, |
| "logps/rejected": -305.02679443359375, |
| "loss": 0.155, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.1784450113773346, |
| "rewards/margins": 5.102679252624512, |
| "rewards/rejected": -5.281124114990234, |
| "step": 605 |
| }, |
| { |
| "epoch": 1.3008, |
| "grad_norm": 2.593752861022949, |
| "learning_rate": 3.1600158568647145e-06, |
| "logits/chosen": -1.5107977390289307, |
| "logits/rejected": -1.5205315351486206, |
| "logps/chosen": -229.0679473876953, |
| "logps/rejected": -298.5066223144531, |
| "loss": 0.1543, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.29753953218460083, |
| "rewards/margins": 4.581008434295654, |
| "rewards/rejected": -4.8785481452941895, |
| "step": 610 |
| }, |
| { |
| "epoch": 1.3114666666666666, |
| "grad_norm": 3.3902955055236816, |
| "learning_rate": 3.1322039011066172e-06, |
| "logits/chosen": -1.6671390533447266, |
| "logits/rejected": -1.50319504737854, |
| "logps/chosen": -244.96578979492188, |
| "logps/rejected": -264.3321228027344, |
| "loss": 0.2094, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.17131295800209045, |
| "rewards/margins": 3.743359088897705, |
| "rewards/rejected": -3.9146721363067627, |
| "step": 615 |
| }, |
| { |
| "epoch": 1.3221333333333334, |
| "grad_norm": 1.5384914875030518, |
| "learning_rate": 3.1043081030096125e-06, |
| "logits/chosen": -1.521968126296997, |
| "logits/rejected": -1.4370750188827515, |
| "logps/chosen": -223.0912628173828, |
| "logps/rejected": -272.32012939453125, |
| "loss": 0.1685, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.06771224737167358, |
| "rewards/margins": 4.779266834259033, |
| "rewards/rejected": -4.711554527282715, |
| "step": 620 |
| }, |
| { |
| "epoch": 1.3328, |
| "grad_norm": 4.687516212463379, |
| "learning_rate": 3.0763321620905072e-06, |
| "logits/chosen": -1.4829010963439941, |
| "logits/rejected": -1.4064357280731201, |
| "logps/chosen": -237.6384735107422, |
| "logps/rejected": -287.4407043457031, |
| "loss": 0.1972, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.37526464462280273, |
| "rewards/margins": 4.6708197593688965, |
| "rewards/rejected": -5.046084880828857, |
| "step": 625 |
| }, |
| { |
| "epoch": 1.3434666666666666, |
| "grad_norm": 3.4469211101531982, |
| "learning_rate": 3.048279788494583e-06, |
| "logits/chosen": -1.5600812435150146, |
| "logits/rejected": -1.5160925388336182, |
| "logps/chosen": -219.6197967529297, |
| "logps/rejected": -277.3929748535156, |
| "loss": 0.1902, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.6968265771865845, |
| "rewards/margins": 5.039797782897949, |
| "rewards/rejected": -5.736624240875244, |
| "step": 630 |
| }, |
| { |
| "epoch": 1.3541333333333334, |
| "grad_norm": 2.8947951793670654, |
| "learning_rate": 3.0201547025035553e-06, |
| "logits/chosen": -1.5564571619033813, |
| "logits/rejected": -1.4424645900726318, |
| "logps/chosen": -253.28347778320312, |
| "logps/rejected": -278.94183349609375, |
| "loss": 0.2277, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.6225299835205078, |
| "rewards/margins": 4.102585792541504, |
| "rewards/rejected": -4.725115776062012, |
| "step": 635 |
| }, |
| { |
| "epoch": 1.3648, |
| "grad_norm": 1.9876494407653809, |
| "learning_rate": 2.9919606340421963e-06, |
| "logits/chosen": -1.5709205865859985, |
| "logits/rejected": -1.5445259809494019, |
| "logps/chosen": -215.19046020507812, |
| "logps/rejected": -280.66961669921875, |
| "loss": 0.2121, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.4023857116699219, |
| "rewards/margins": 4.108145236968994, |
| "rewards/rejected": -4.510531425476074, |
| "step": 640 |
| }, |
| { |
| "epoch": 1.3754666666666666, |
| "grad_norm": 5.166397571563721, |
| "learning_rate": 2.9637013221836723e-06, |
| "logits/chosen": -1.5972050428390503, |
| "logits/rejected": -1.5269817113876343, |
| "logps/chosen": -239.1528778076172, |
| "logps/rejected": -287.1815490722656, |
| "loss": 0.2613, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.5880186557769775, |
| "rewards/margins": 4.12716007232666, |
| "rewards/rejected": -4.715178489685059, |
| "step": 645 |
| }, |
| { |
| "epoch": 1.3861333333333334, |
| "grad_norm": 3.5479846000671387, |
| "learning_rate": 2.9353805146536762e-06, |
| "logits/chosen": -1.521431803703308, |
| "logits/rejected": -1.463780403137207, |
| "logps/chosen": -232.4151611328125, |
| "logps/rejected": -268.7264099121094, |
| "loss": 0.2158, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.15852993726730347, |
| "rewards/margins": 4.376145839691162, |
| "rewards/rejected": -4.534675598144531, |
| "step": 650 |
| }, |
| { |
| "epoch": 1.3968, |
| "grad_norm": 4.420405387878418, |
| "learning_rate": 2.907001967333401e-06, |
| "logits/chosen": -1.6656224727630615, |
| "logits/rejected": -1.5566421747207642, |
| "logps/chosen": -215.8498992919922, |
| "logps/rejected": -274.35357666015625, |
| "loss": 0.2414, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.14214067161083221, |
| "rewards/margins": 4.248762130737305, |
| "rewards/rejected": -4.390902519226074, |
| "step": 655 |
| }, |
| { |
| "epoch": 1.4074666666666666, |
| "grad_norm": 2.908594846725464, |
| "learning_rate": 2.878569443761442e-06, |
| "logits/chosen": -1.4593592882156372, |
| "logits/rejected": -1.4189205169677734, |
| "logps/chosen": -204.3805694580078, |
| "logps/rejected": -258.9472351074219, |
| "loss": 0.1519, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.021609745919704437, |
| "rewards/margins": 4.725183963775635, |
| "rewards/rejected": -4.746793270111084, |
| "step": 660 |
| }, |
| { |
| "epoch": 1.4181333333333335, |
| "grad_norm": 5.427767276763916, |
| "learning_rate": 2.85008671463468e-06, |
| "logits/chosen": -1.5551220178604126, |
| "logits/rejected": -1.5175118446350098, |
| "logps/chosen": -219.32080078125, |
| "logps/rejected": -286.12615966796875, |
| "loss": 0.1962, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3829983174800873, |
| "rewards/margins": 4.652920722961426, |
| "rewards/rejected": -5.035919189453125, |
| "step": 665 |
| }, |
| { |
| "epoch": 1.4288, |
| "grad_norm": 2.0883243083953857, |
| "learning_rate": 2.8215575573082127e-06, |
| "logits/chosen": -1.5865761041641235, |
| "logits/rejected": -1.5060150623321533, |
| "logps/chosen": -211.5963592529297, |
| "logps/rejected": -292.7962951660156, |
| "loss": 0.1581, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.3247772455215454, |
| "rewards/margins": 5.109382152557373, |
| "rewards/rejected": -5.434159278869629, |
| "step": 670 |
| }, |
| { |
| "epoch": 1.4394666666666667, |
| "grad_norm": 4.16923189163208, |
| "learning_rate": 2.792985755294404e-06, |
| "logits/chosen": -1.5806046724319458, |
| "logits/rejected": -1.4857282638549805, |
| "logps/chosen": -213.30587768554688, |
| "logps/rejected": -269.56365966796875, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5577279329299927, |
| "rewards/margins": 4.701187610626221, |
| "rewards/rejected": -5.258915901184082, |
| "step": 675 |
| }, |
| { |
| "epoch": 1.4501333333333333, |
| "grad_norm": 4.474754810333252, |
| "learning_rate": 2.764375097761122e-06, |
| "logits/chosen": -1.4321134090423584, |
| "logits/rejected": -1.477826714515686, |
| "logps/chosen": -216.34506225585938, |
| "logps/rejected": -296.2746887207031, |
| "loss": 0.2041, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.43209022283554077, |
| "rewards/margins": 4.764926433563232, |
| "rewards/rejected": -5.197016716003418, |
| "step": 680 |
| }, |
| { |
| "epoch": 1.4607999999999999, |
| "grad_norm": 3.623042583465576, |
| "learning_rate": 2.73572937902922e-06, |
| "logits/chosen": -1.5734399557113647, |
| "logits/rejected": -1.5528452396392822, |
| "logps/chosen": -245.42086791992188, |
| "logps/rejected": -285.8833923339844, |
| "loss": 0.2116, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.6044676899909973, |
| "rewards/margins": 4.195218563079834, |
| "rewards/rejected": -4.799685955047607, |
| "step": 685 |
| }, |
| { |
| "epoch": 1.4714666666666667, |
| "grad_norm": 3.855921506881714, |
| "learning_rate": 2.7070523980693362e-06, |
| "logits/chosen": -1.385648488998413, |
| "logits/rejected": -1.407254934310913, |
| "logps/chosen": -238.00991821289062, |
| "logps/rejected": -293.28619384765625, |
| "loss": 0.2153, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.36002954840660095, |
| "rewards/margins": 4.525365352630615, |
| "rewards/rejected": -4.885395050048828, |
| "step": 690 |
| }, |
| { |
| "epoch": 1.4821333333333333, |
| "grad_norm": 3.164611339569092, |
| "learning_rate": 2.678347957998081e-06, |
| "logits/chosen": -1.5446698665618896, |
| "logits/rejected": -1.4527463912963867, |
| "logps/chosen": -229.75595092773438, |
| "logps/rejected": -275.39825439453125, |
| "loss": 0.1813, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.625076413154602, |
| "rewards/margins": 4.960640907287598, |
| "rewards/rejected": -5.585717678070068, |
| "step": 695 |
| }, |
| { |
| "epoch": 1.4928, |
| "grad_norm": 2.971756935119629, |
| "learning_rate": 2.649619865573666e-06, |
| "logits/chosen": -1.620855689048767, |
| "logits/rejected": -1.5859421491622925, |
| "logps/chosen": -232.5053253173828, |
| "logps/rejected": -301.48516845703125, |
| "loss": 0.1653, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6067575216293335, |
| "rewards/margins": 4.746621131896973, |
| "rewards/rejected": -5.353378772735596, |
| "step": 700 |
| }, |
| { |
| "epoch": 1.5034666666666667, |
| "grad_norm": 5.307252883911133, |
| "learning_rate": 2.620871930691058e-06, |
| "logits/chosen": -1.542690396308899, |
| "logits/rejected": -1.5111274719238281, |
| "logps/chosen": -229.7939453125, |
| "logps/rejected": -281.5523986816406, |
| "loss": 0.2582, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.8791757822036743, |
| "rewards/margins": 4.178490161895752, |
| "rewards/rejected": -5.057665824890137, |
| "step": 705 |
| }, |
| { |
| "epoch": 1.5141333333333333, |
| "grad_norm": 3.5124833583831787, |
| "learning_rate": 2.5921079658767152e-06, |
| "logits/chosen": -1.54134202003479, |
| "logits/rejected": -1.527388572692871, |
| "logps/chosen": -221.4246368408203, |
| "logps/rejected": -286.0513000488281, |
| "loss": 0.1538, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.41137829422950745, |
| "rewards/margins": 5.438130855560303, |
| "rewards/rejected": -5.849509239196777, |
| "step": 710 |
| }, |
| { |
| "epoch": 1.5248, |
| "grad_norm": 5.473639011383057, |
| "learning_rate": 2.56333178578297e-06, |
| "logits/chosen": -1.5290729999542236, |
| "logits/rejected": -1.4904533624649048, |
| "logps/chosen": -212.76895141601562, |
| "logps/rejected": -281.6219482421875, |
| "loss": 0.1965, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.6981285810470581, |
| "rewards/margins": 4.48247766494751, |
| "rewards/rejected": -5.180605888366699, |
| "step": 715 |
| }, |
| { |
| "epoch": 1.5354666666666668, |
| "grad_norm": 5.93931770324707, |
| "learning_rate": 2.534547206682136e-06, |
| "logits/chosen": -1.6086270809173584, |
| "logits/rejected": -1.5720148086547852, |
| "logps/chosen": -228.1181182861328, |
| "logps/rejected": -307.33563232421875, |
| "loss": 0.1537, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6191433668136597, |
| "rewards/margins": 4.839142322540283, |
| "rewards/rejected": -5.458285331726074, |
| "step": 720 |
| }, |
| { |
| "epoch": 1.5461333333333334, |
| "grad_norm": 2.0792360305786133, |
| "learning_rate": 2.5057580459603947e-06, |
| "logits/chosen": -1.6214183568954468, |
| "logits/rejected": -1.5774202346801758, |
| "logps/chosen": -213.54150390625, |
| "logps/rejected": -274.8948974609375, |
| "loss": 0.1982, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.48994430899620056, |
| "rewards/margins": 4.390619277954102, |
| "rewards/rejected": -4.880563259124756, |
| "step": 725 |
| }, |
| { |
| "epoch": 1.5568, |
| "grad_norm": 3.505744218826294, |
| "learning_rate": 2.4769681216115384e-06, |
| "logits/chosen": -1.6032994985580444, |
| "logits/rejected": -1.5804897546768188, |
| "logps/chosen": -221.6490020751953, |
| "logps/rejected": -285.50909423828125, |
| "loss": 0.2008, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.7238569259643555, |
| "rewards/margins": 4.416735649108887, |
| "rewards/rejected": -5.1405930519104, |
| "step": 730 |
| }, |
| { |
| "epoch": 1.5674666666666668, |
| "grad_norm": 2.8359549045562744, |
| "learning_rate": 2.448181251730632e-06, |
| "logits/chosen": -1.519239068031311, |
| "logits/rejected": -1.4421329498291016, |
| "logps/chosen": -217.1107177734375, |
| "logps/rejected": -258.64642333984375, |
| "loss": 0.1843, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.4088948667049408, |
| "rewards/margins": 4.174046516418457, |
| "rewards/rejected": -4.582940101623535, |
| "step": 735 |
| }, |
| { |
| "epoch": 1.5781333333333334, |
| "grad_norm": 4.682807922363281, |
| "learning_rate": 2.419401254007658e-06, |
| "logits/chosen": -1.4647693634033203, |
| "logits/rejected": -1.4171350002288818, |
| "logps/chosen": -225.9655303955078, |
| "logps/rejected": -299.84405517578125, |
| "loss": 0.1921, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.16728904843330383, |
| "rewards/margins": 5.242563724517822, |
| "rewards/rejected": -5.409852981567383, |
| "step": 740 |
| }, |
| { |
| "epoch": 1.5888, |
| "grad_norm": 5.701550006866455, |
| "learning_rate": 2.390631945221221e-06, |
| "logits/chosen": -1.5880542993545532, |
| "logits/rejected": -1.547324776649475, |
| "logps/chosen": -235.06039428710938, |
| "logps/rejected": -297.7355651855469, |
| "loss": 0.1892, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6305644512176514, |
| "rewards/margins": 5.0092668533325195, |
| "rewards/rejected": -5.63983154296875, |
| "step": 745 |
| }, |
| { |
| "epoch": 1.5994666666666668, |
| "grad_norm": 4.782139301300049, |
| "learning_rate": 2.361877140732366e-06, |
| "logits/chosen": -1.5697643756866455, |
| "logits/rejected": -1.510378122329712, |
| "logps/chosen": -224.08065795898438, |
| "logps/rejected": -284.9275207519531, |
| "loss": 0.1929, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.6623788475990295, |
| "rewards/margins": 4.567178726196289, |
| "rewards/rejected": -5.229557514190674, |
| "step": 750 |
| }, |
| { |
| "epoch": 1.6101333333333332, |
| "grad_norm": 4.09246826171875, |
| "learning_rate": 2.333140653978592e-06, |
| "logits/chosen": -1.6691592931747437, |
| "logits/rejected": -1.568692922592163, |
| "logps/chosen": -226.2360382080078, |
| "logps/rejected": -297.37799072265625, |
| "loss": 0.1745, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.5370773077011108, |
| "rewards/margins": 4.940179347991943, |
| "rewards/rejected": -5.47725772857666, |
| "step": 755 |
| }, |
| { |
| "epoch": 1.6208, |
| "grad_norm": 4.379272937774658, |
| "learning_rate": 2.3044262959681156e-06, |
| "logits/chosen": -1.4959796667099, |
| "logits/rejected": -1.51832115650177, |
| "logps/chosen": -216.44430541992188, |
| "logps/rejected": -308.85015869140625, |
| "loss": 0.1948, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.460176557302475, |
| "rewards/margins": 4.9603471755981445, |
| "rewards/rejected": -5.420523166656494, |
| "step": 760 |
| }, |
| { |
| "epoch": 1.6314666666666666, |
| "grad_norm": 2.3544809818267822, |
| "learning_rate": 2.2757378747744575e-06, |
| "logits/chosen": -1.636636734008789, |
| "logits/rejected": -1.5793819427490234, |
| "logps/chosen": -216.9700927734375, |
| "logps/rejected": -287.65386962890625, |
| "loss": 0.1965, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.5221351385116577, |
| "rewards/margins": 4.529210090637207, |
| "rewards/rejected": -5.0513458251953125, |
| "step": 765 |
| }, |
| { |
| "epoch": 1.6421333333333332, |
| "grad_norm": 3.5676753520965576, |
| "learning_rate": 2.24707919503142e-06, |
| "logits/chosen": -1.660280466079712, |
| "logits/rejected": -1.5980972051620483, |
| "logps/chosen": -239.9665985107422, |
| "logps/rejected": -295.49224853515625, |
| "loss": 0.2126, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.6387029886245728, |
| "rewards/margins": 4.475255966186523, |
| "rewards/rejected": -5.113959312438965, |
| "step": 770 |
| }, |
| { |
| "epoch": 1.6528, |
| "grad_norm": 4.009433746337891, |
| "learning_rate": 2.2184540574285213e-06, |
| "logits/chosen": -1.5477678775787354, |
| "logits/rejected": -1.5038559436798096, |
| "logps/chosen": -233.2089080810547, |
| "logps/rejected": -275.3874816894531, |
| "loss": 0.1616, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.1705014854669571, |
| "rewards/margins": 4.591641426086426, |
| "rewards/rejected": -4.762143135070801, |
| "step": 775 |
| }, |
| { |
| "epoch": 1.6634666666666666, |
| "grad_norm": 4.829162120819092, |
| "learning_rate": 2.1898662582069486e-06, |
| "logits/chosen": -1.6277942657470703, |
| "logits/rejected": -1.541800856590271, |
| "logps/chosen": -264.6813659667969, |
| "logps/rejected": -298.3739318847656, |
| "loss": 0.1844, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.132328063249588, |
| "rewards/margins": 5.129775047302246, |
| "rewards/rejected": -4.997447490692139, |
| "step": 780 |
| }, |
| { |
| "epoch": 1.6741333333333333, |
| "grad_norm": 2.8724241256713867, |
| "learning_rate": 2.1613195886561044e-06, |
| "logits/chosen": -1.5807163715362549, |
| "logits/rejected": -1.5272471904754639, |
| "logps/chosen": -212.9057159423828, |
| "logps/rejected": -302.0938720703125, |
| "loss": 0.1395, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.03779119253158569, |
| "rewards/margins": 5.289394378662109, |
| "rewards/rejected": -5.251603603363037, |
| "step": 785 |
| }, |
| { |
| "epoch": 1.6848, |
| "grad_norm": 4.156414985656738, |
| "learning_rate": 2.132817834610811e-06, |
| "logits/chosen": -1.6008613109588623, |
| "logits/rejected": -1.5462461709976196, |
| "logps/chosen": -221.5519256591797, |
| "logps/rejected": -276.48651123046875, |
| "loss": 0.2132, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.037129394710063934, |
| "rewards/margins": 4.2190446853637695, |
| "rewards/rejected": -4.256174087524414, |
| "step": 790 |
| }, |
| { |
| "epoch": 1.6954666666666667, |
| "grad_norm": 4.604877948760986, |
| "learning_rate": 2.1043647759492338e-06, |
| "logits/chosen": -1.4992469549179077, |
| "logits/rejected": -1.588596224784851, |
| "logps/chosen": -237.80368041992188, |
| "logps/rejected": -276.7908630371094, |
| "loss": 0.2331, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.19210034608840942, |
| "rewards/margins": 4.0293097496032715, |
| "rewards/rejected": -4.221410274505615, |
| "step": 795 |
| }, |
| { |
| "epoch": 1.7061333333333333, |
| "grad_norm": 4.77518367767334, |
| "learning_rate": 2.075964186091599e-06, |
| "logits/chosen": -1.5367120504379272, |
| "logits/rejected": -1.518836259841919, |
| "logps/chosen": -216.94961547851562, |
| "logps/rejected": -297.82269287109375, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.05688552185893059, |
| "rewards/margins": 4.8048095703125, |
| "rewards/rejected": -4.747923851013184, |
| "step": 800 |
| }, |
| { |
| "epoch": 1.7168, |
| "grad_norm": 2.7294278144836426, |
| "learning_rate": 2.047619831499765e-06, |
| "logits/chosen": -1.5205106735229492, |
| "logits/rejected": -1.4168894290924072, |
| "logps/chosen": -214.5801239013672, |
| "logps/rejected": -279.34320068359375, |
| "loss": 0.164, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.3186998963356018, |
| "rewards/margins": 4.727145195007324, |
| "rewards/rejected": -5.045844078063965, |
| "step": 805 |
| }, |
| { |
| "epoch": 1.7274666666666667, |
| "grad_norm": 3.494635820388794, |
| "learning_rate": 2.0193354711777155e-06, |
| "logits/chosen": -1.668074369430542, |
| "logits/rejected": -1.6056506633758545, |
| "logps/chosen": -226.056396484375, |
| "logps/rejected": -283.64312744140625, |
| "loss": 0.1962, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.24879607558250427, |
| "rewards/margins": 4.313138961791992, |
| "rewards/rejected": -4.561934947967529, |
| "step": 810 |
| }, |
| { |
| "epoch": 1.7381333333333333, |
| "grad_norm": 4.093262195587158, |
| "learning_rate": 1.991114856173048e-06, |
| "logits/chosen": -1.6196622848510742, |
| "logits/rejected": -1.5892953872680664, |
| "logps/chosen": -211.94473266601562, |
| "logps/rejected": -296.87890625, |
| "loss": 0.1735, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.1287282407283783, |
| "rewards/margins": 4.94359827041626, |
| "rewards/rejected": -4.814869403839111, |
| "step": 815 |
| }, |
| { |
| "epoch": 1.7488000000000001, |
| "grad_norm": 4.43931770324707, |
| "learning_rate": 1.9629617290795115e-06, |
| "logits/chosen": -1.5054540634155273, |
| "logits/rejected": -1.5589126348495483, |
| "logps/chosen": -200.23751831054688, |
| "logps/rejected": -271.1064453125, |
| "loss": 0.2093, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.019808894023299217, |
| "rewards/margins": 4.327120780944824, |
| "rewards/rejected": -4.34692907333374, |
| "step": 820 |
| }, |
| { |
| "epoch": 1.7594666666666665, |
| "grad_norm": 4.095510959625244, |
| "learning_rate": 1.934879823540663e-06, |
| "logits/chosen": -1.5728521347045898, |
| "logits/rejected": -1.4718952178955078, |
| "logps/chosen": -210.8157196044922, |
| "logps/rejected": -272.39410400390625, |
| "loss": 0.1988, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.16639301180839539, |
| "rewards/margins": 4.377795219421387, |
| "rewards/rejected": -4.21140193939209, |
| "step": 825 |
| }, |
| { |
| "epoch": 1.7701333333333333, |
| "grad_norm": 4.527327060699463, |
| "learning_rate": 1.9068728637547195e-06, |
| "logits/chosen": -1.5970168113708496, |
| "logits/rejected": -1.4815723896026611, |
| "logps/chosen": -223.53250122070312, |
| "logps/rejected": -286.47918701171875, |
| "loss": 0.1972, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.09205605089664459, |
| "rewards/margins": 4.707822322845459, |
| "rewards/rejected": -4.615766525268555, |
| "step": 830 |
| }, |
| { |
| "epoch": 1.7808000000000002, |
| "grad_norm": 2.798776388168335, |
| "learning_rate": 1.878944563980658e-06, |
| "logits/chosen": -1.5575376749038696, |
| "logits/rejected": -1.4680263996124268, |
| "logps/chosen": -229.95352172851562, |
| "logps/rejected": -277.1444396972656, |
| "loss": 0.1885, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.11531858146190643, |
| "rewards/margins": 4.542990684509277, |
| "rewards/rejected": -4.427672386169434, |
| "step": 835 |
| }, |
| { |
| "epoch": 1.7914666666666665, |
| "grad_norm": 2.6367621421813965, |
| "learning_rate": 1.85109862804563e-06, |
| "logits/chosen": -1.5009130239486694, |
| "logits/rejected": -1.52145254611969, |
| "logps/chosen": -233.2731170654297, |
| "logps/rejected": -285.61663818359375, |
| "loss": 0.1508, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.020412441343069077, |
| "rewards/margins": 4.928297996520996, |
| "rewards/rejected": -4.9487104415893555, |
| "step": 840 |
| }, |
| { |
| "epoch": 1.8021333333333334, |
| "grad_norm": 3.9212377071380615, |
| "learning_rate": 1.823338748853766e-06, |
| "logits/chosen": -1.6089658737182617, |
| "logits/rejected": -1.5349115133285522, |
| "logps/chosen": -217.51895141601562, |
| "logps/rejected": -281.0561218261719, |
| "loss": 0.1702, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.24485504627227783, |
| "rewards/margins": 4.494772434234619, |
| "rewards/rejected": -4.739627361297607, |
| "step": 845 |
| }, |
| { |
| "epoch": 1.8128, |
| "grad_norm": 2.247929334640503, |
| "learning_rate": 1.7956686078964257e-06, |
| "logits/chosen": -1.560617208480835, |
| "logits/rejected": -1.529205083847046, |
| "logps/chosen": -226.0928192138672, |
| "logps/rejected": -268.5011901855469, |
| "loss": 0.1853, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.20438656210899353, |
| "rewards/margins": 4.4868550300598145, |
| "rewards/rejected": -4.691241264343262, |
| "step": 850 |
| }, |
| { |
| "epoch": 1.8234666666666666, |
| "grad_norm": 3.934013843536377, |
| "learning_rate": 1.7680918747639608e-06, |
| "logits/chosen": -1.6019423007965088, |
| "logits/rejected": -1.5354797840118408, |
| "logps/chosen": -223.7849578857422, |
| "logps/rejected": -286.2995300292969, |
| "loss": 0.2091, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.15956279635429382, |
| "rewards/margins": 4.305161952972412, |
| "rewards/rejected": -4.464724540710449, |
| "step": 855 |
| }, |
| { |
| "epoch": 1.8341333333333334, |
| "grad_norm": 3.362903594970703, |
| "learning_rate": 1.740612206659057e-06, |
| "logits/chosen": -1.541973352432251, |
| "logits/rejected": -1.5166317224502563, |
| "logps/chosen": -221.7763214111328, |
| "logps/rejected": -293.17388916015625, |
| "loss": 0.2248, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.11231374740600586, |
| "rewards/margins": 4.609333038330078, |
| "rewards/rejected": -4.721646308898926, |
| "step": 860 |
| }, |
| { |
| "epoch": 1.8448, |
| "grad_norm": 1.669443964958191, |
| "learning_rate": 1.7132332479117204e-06, |
| "logits/chosen": -1.6339985132217407, |
| "logits/rejected": -1.5085113048553467, |
| "logps/chosen": -234.7068328857422, |
| "logps/rejected": -285.39642333984375, |
| "loss": 0.1391, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.4002878665924072, |
| "rewards/margins": 4.612551689147949, |
| "rewards/rejected": -4.212264060974121, |
| "step": 865 |
| }, |
| { |
| "epoch": 1.8554666666666666, |
| "grad_norm": 4.4884934425354, |
| "learning_rate": 1.6859586294959666e-06, |
| "logits/chosen": -1.591619610786438, |
| "logits/rejected": -1.495373010635376, |
| "logps/chosen": -237.3191375732422, |
| "logps/rejected": -295.4523010253906, |
| "loss": 0.1871, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.10502872616052628, |
| "rewards/margins": 5.0281901359558105, |
| "rewards/rejected": -4.923161506652832, |
| "step": 870 |
| }, |
| { |
| "epoch": 1.8661333333333334, |
| "grad_norm": 3.515310049057007, |
| "learning_rate": 1.6587919685482884e-06, |
| "logits/chosen": -1.6238123178482056, |
| "logits/rejected": -1.586525321006775, |
| "logps/chosen": -206.5242156982422, |
| "logps/rejected": -278.3238220214844, |
| "loss": 0.1752, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.16240589320659637, |
| "rewards/margins": 4.887397766113281, |
| "rewards/rejected": -4.724991798400879, |
| "step": 875 |
| }, |
| { |
| "epoch": 1.8768, |
| "grad_norm": 5.287433624267578, |
| "learning_rate": 1.6317368678879497e-06, |
| "logits/chosen": -1.540462851524353, |
| "logits/rejected": -1.5053167343139648, |
| "logps/chosen": -222.8974609375, |
| "logps/rejected": -283.7711486816406, |
| "loss": 0.231, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.02352920174598694, |
| "rewards/margins": 4.16703987121582, |
| "rewards/rejected": -4.143510341644287, |
| "step": 880 |
| }, |
| { |
| "epoch": 1.8874666666666666, |
| "grad_norm": 4.371232509613037, |
| "learning_rate": 1.6047969155391856e-06, |
| "logits/chosen": -1.4189904928207397, |
| "logits/rejected": -1.4196317195892334, |
| "logps/chosen": -238.34396362304688, |
| "logps/rejected": -286.05682373046875, |
| "loss": 0.2134, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.4680125117301941, |
| "rewards/margins": 5.1263604164123535, |
| "rewards/rejected": -4.6583476066589355, |
| "step": 885 |
| }, |
| { |
| "epoch": 1.8981333333333335, |
| "grad_norm": 4.04213285446167, |
| "learning_rate": 1.5779756842553617e-06, |
| "logits/chosen": -1.5660136938095093, |
| "logits/rejected": -1.5414934158325195, |
| "logps/chosen": -226.266845703125, |
| "logps/rejected": -281.00823974609375, |
| "loss": 0.1493, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.38566893339157104, |
| "rewards/margins": 4.783738136291504, |
| "rewards/rejected": -4.398069858551025, |
| "step": 890 |
| }, |
| { |
| "epoch": 1.9088, |
| "grad_norm": 2.9223861694335938, |
| "learning_rate": 1.551276731045157e-06, |
| "logits/chosen": -1.53744637966156, |
| "logits/rejected": -1.5588454008102417, |
| "logps/chosen": -236.10806274414062, |
| "logps/rejected": -289.128173828125, |
| "loss": 0.1751, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.26428985595703125, |
| "rewards/margins": 4.615136623382568, |
| "rewards/rejected": -4.350847244262695, |
| "step": 895 |
| }, |
| { |
| "epoch": 1.9194666666666667, |
| "grad_norm": 4.002009391784668, |
| "learning_rate": 1.5247035967008369e-06, |
| "logits/chosen": -1.439851999282837, |
| "logits/rejected": -1.4446804523468018, |
| "logps/chosen": -225.57601928710938, |
| "logps/rejected": -275.29644775390625, |
| "loss": 0.1965, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": 0.3654697835445404, |
| "rewards/margins": 4.579410076141357, |
| "rewards/rejected": -4.213940143585205, |
| "step": 900 |
| }, |
| { |
| "epoch": 1.9301333333333335, |
| "grad_norm": 3.070713996887207, |
| "learning_rate": 1.498259805328678e-06, |
| "logits/chosen": -1.5961030721664429, |
| "logits/rejected": -1.5069525241851807, |
| "logps/chosen": -216.4861297607422, |
| "logps/rejected": -267.57098388671875, |
| "loss": 0.1618, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.414588987827301, |
| "rewards/margins": 4.7993268966674805, |
| "rewards/rejected": -4.384737491607666, |
| "step": 905 |
| }, |
| { |
| "epoch": 1.9407999999999999, |
| "grad_norm": 3.228595495223999, |
| "learning_rate": 1.471948863881601e-06, |
| "logits/chosen": -1.6398388147354126, |
| "logits/rejected": -1.5302976369857788, |
| "logps/chosen": -241.460693359375, |
| "logps/rejected": -305.2004699707031, |
| "loss": 0.2167, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": 0.2526926100254059, |
| "rewards/margins": 4.873023509979248, |
| "rewards/rejected": -4.620330810546875, |
| "step": 910 |
| }, |
| { |
| "epoch": 1.9514666666666667, |
| "grad_norm": 4.4869794845581055, |
| "learning_rate": 1.4457742616940864e-06, |
| "logits/chosen": -1.5678088665008545, |
| "logits/rejected": -1.4262385368347168, |
| "logps/chosen": -232.7917022705078, |
| "logps/rejected": -277.34649658203125, |
| "loss": 0.174, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.3906717002391815, |
| "rewards/margins": 4.650841236114502, |
| "rewards/rejected": -4.26016902923584, |
| "step": 915 |
| }, |
| { |
| "epoch": 1.9621333333333333, |
| "grad_norm": 4.238879203796387, |
| "learning_rate": 1.4197394700194165e-06, |
| "logits/chosen": -1.5569572448730469, |
| "logits/rejected": -1.573560118675232, |
| "logps/chosen": -209.18460083007812, |
| "logps/rejected": -279.4632263183594, |
| "loss": 0.1466, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.6271043419837952, |
| "rewards/margins": 4.994255065917969, |
| "rewards/rejected": -4.367150783538818, |
| "step": 920 |
| }, |
| { |
| "epoch": 1.9727999999999999, |
| "grad_norm": 3.870699882507324, |
| "learning_rate": 1.3938479415693244e-06, |
| "logits/chosen": -1.6715972423553467, |
| "logits/rejected": -1.5435243844985962, |
| "logps/chosen": -223.7204132080078, |
| "logps/rejected": -261.2884826660156, |
| "loss": 0.1914, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.28375479578971863, |
| "rewards/margins": 4.4150848388671875, |
| "rewards/rejected": -4.131329536437988, |
| "step": 925 |
| }, |
| { |
| "epoch": 1.9834666666666667, |
| "grad_norm": 3.798652172088623, |
| "learning_rate": 1.368103110056096e-06, |
| "logits/chosen": -1.6140682697296143, |
| "logits/rejected": -1.5190401077270508, |
| "logps/chosen": -224.6747283935547, |
| "logps/rejected": -268.4491882324219, |
| "loss": 0.202, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": 0.05197082832455635, |
| "rewards/margins": 4.320896148681641, |
| "rewards/rejected": -4.268925666809082, |
| "step": 930 |
| }, |
| { |
| "epoch": 1.9941333333333333, |
| "grad_norm": 2.9648468494415283, |
| "learning_rate": 1.3425083897371983e-06, |
| "logits/chosen": -1.473844289779663, |
| "logits/rejected": -1.479618787765503, |
| "logps/chosen": -232.2730712890625, |
| "logps/rejected": -298.5815124511719, |
| "loss": 0.1892, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.10708019882440567, |
| "rewards/margins": 4.55968713760376, |
| "rewards/rejected": -4.452606678009033, |
| "step": 935 |
| }, |
| { |
| "epoch": 2.0042666666666666, |
| "grad_norm": 3.558039665222168, |
| "learning_rate": 1.317067174962478e-06, |
| "logits/chosen": -1.501671314239502, |
| "logits/rejected": -1.4494259357452393, |
| "logps/chosen": -232.77017211914062, |
| "logps/rejected": -291.8146057128906, |
| "loss": 0.1702, |
| "rewards/accuracies": 0.9473684430122375, |
| "rewards/chosen": 0.30579400062561035, |
| "rewards/margins": 4.907108306884766, |
| "rewards/rejected": -4.601314544677734, |
| "step": 940 |
| }, |
| { |
| "epoch": 2.0149333333333335, |
| "grad_norm": 1.816678762435913, |
| "learning_rate": 1.291782839724009e-06, |
| "logits/chosen": -1.5644961595535278, |
| "logits/rejected": -1.4842809438705444, |
| "logps/chosen": -237.2871551513672, |
| "logps/rejected": -296.8790588378906, |
| "loss": 0.13, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.1960347443819046, |
| "rewards/margins": 4.91409969329834, |
| "rewards/rejected": -4.7180657386779785, |
| "step": 945 |
| }, |
| { |
| "epoch": 2.0256, |
| "grad_norm": 3.761232376098633, |
| "learning_rate": 1.2666587372086352e-06, |
| "logits/chosen": -1.4631893634796143, |
| "logits/rejected": -1.4054863452911377, |
| "logps/chosen": -219.4536590576172, |
| "logps/rejected": -298.73504638671875, |
| "loss": 0.1734, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.10820744186639786, |
| "rewards/margins": 4.628209114074707, |
| "rewards/rejected": -4.520001411437988, |
| "step": 950 |
| }, |
| { |
| "epoch": 2.0362666666666667, |
| "grad_norm": 3.467888832092285, |
| "learning_rate": 1.241698199353275e-06, |
| "logits/chosen": -1.4637609720230103, |
| "logits/rejected": -1.4368250370025635, |
| "logps/chosen": -214.6084442138672, |
| "logps/rejected": -276.13189697265625, |
| "loss": 0.1424, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.2766164243221283, |
| "rewards/margins": 4.959458351135254, |
| "rewards/rejected": -4.682842254638672, |
| "step": 955 |
| }, |
| { |
| "epoch": 2.0469333333333335, |
| "grad_norm": 2.6543424129486084, |
| "learning_rate": 1.2169045364030404e-06, |
| "logits/chosen": -1.6459102630615234, |
| "logits/rejected": -1.6207349300384521, |
| "logps/chosen": -210.13259887695312, |
| "logps/rejected": -287.6831359863281, |
| "loss": 0.1244, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.011042666621506214, |
| "rewards/margins": 5.0912251472473145, |
| "rewards/rejected": -5.080182075500488, |
| "step": 960 |
| }, |
| { |
| "epoch": 2.0576, |
| "grad_norm": 3.120337963104248, |
| "learning_rate": 1.1922810364722402e-06, |
| "logits/chosen": -1.600803017616272, |
| "logits/rejected": -1.5503169298171997, |
| "logps/chosen": -233.22439575195312, |
| "logps/rejected": -283.6503601074219, |
| "loss": 0.1614, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.14875979721546173, |
| "rewards/margins": 4.319775581359863, |
| "rewards/rejected": -4.171015739440918, |
| "step": 965 |
| }, |
| { |
| "epoch": 2.0682666666666667, |
| "grad_norm": 3.577122211456299, |
| "learning_rate": 1.1678309651083055e-06, |
| "logits/chosen": -1.6564123630523682, |
| "logits/rejected": -1.5476027727127075, |
| "logps/chosen": -229.00778198242188, |
| "logps/rejected": -290.4595642089844, |
| "loss": 0.1642, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.0114586828276515, |
| "rewards/margins": 5.043142795562744, |
| "rewards/rejected": -5.031683921813965, |
| "step": 970 |
| }, |
| { |
| "epoch": 2.0789333333333335, |
| "grad_norm": 3.0231740474700928, |
| "learning_rate": 1.1435575648587224e-06, |
| "logits/chosen": -1.5095252990722656, |
| "logits/rejected": -1.5051429271697998, |
| "logps/chosen": -215.7372589111328, |
| "logps/rejected": -279.8727722167969, |
| "loss": 0.1494, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": 0.09145411103963852, |
| "rewards/margins": 4.958231449127197, |
| "rewards/rejected": -4.8667778968811035, |
| "step": 975 |
| }, |
| { |
| "epoch": 2.0896, |
| "grad_norm": 2.384561538696289, |
| "learning_rate": 1.1194640548410039e-06, |
| "logits/chosen": -1.5030790567398071, |
| "logits/rejected": -1.5039005279541016, |
| "logps/chosen": -223.53341674804688, |
| "logps/rejected": -268.07305908203125, |
| "loss": 0.1692, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.3087685704231262, |
| "rewards/margins": 4.652369499206543, |
| "rewards/rejected": -4.343601226806641, |
| "step": 980 |
| }, |
| { |
| "epoch": 2.1002666666666667, |
| "grad_norm": 2.826698064804077, |
| "learning_rate": 1.095553630315774e-06, |
| "logits/chosen": -1.5461299419403076, |
| "logits/rejected": -1.5500050783157349, |
| "logps/chosen": -219.19961547851562, |
| "logps/rejected": -297.0096740722656, |
| "loss": 0.1482, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.03134971857070923, |
| "rewards/margins": 4.71282958984375, |
| "rewards/rejected": -4.744178771972656, |
| "step": 985 |
| }, |
| { |
| "epoch": 2.1109333333333336, |
| "grad_norm": 4.161277770996094, |
| "learning_rate": 1.0718294622630188e-06, |
| "logits/chosen": -1.6473701000213623, |
| "logits/rejected": -1.5392096042633057, |
| "logps/chosen": -237.41329956054688, |
| "logps/rejected": -287.1451110839844, |
| "loss": 0.1731, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.3596492409706116, |
| "rewards/margins": 4.376633644104004, |
| "rewards/rejected": -4.7362823486328125, |
| "step": 990 |
| }, |
| { |
| "epoch": 2.1216, |
| "grad_norm": 2.4758927822113037, |
| "learning_rate": 1.0482946969615495e-06, |
| "logits/chosen": -1.5916746854782104, |
| "logits/rejected": -1.5499130487442017, |
| "logps/chosen": -200.91793823242188, |
| "logps/rejected": -293.1402282714844, |
| "loss": 0.1315, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.13916119933128357, |
| "rewards/margins": 5.298827648162842, |
| "rewards/rejected": -5.437989234924316, |
| "step": 995 |
| }, |
| { |
| "epoch": 2.1322666666666668, |
| "grad_norm": 2.656404972076416, |
| "learning_rate": 1.0249524555717461e-06, |
| "logits/chosen": -1.6521761417388916, |
| "logits/rejected": -1.4956820011138916, |
| "logps/chosen": -221.94546508789062, |
| "logps/rejected": -280.0649719238281, |
| "loss": 0.1546, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.021595388650894165, |
| "rewards/margins": 4.696625709533691, |
| "rewards/rejected": -4.718221187591553, |
| "step": 1000 |
| }, |
| { |
| "epoch": 2.142933333333333, |
| "grad_norm": 3.276386260986328, |
| "learning_rate": 1.0018058337216327e-06, |
| "logits/chosen": -1.5643367767333984, |
| "logits/rejected": -1.5389833450317383, |
| "logps/chosen": -232.3899383544922, |
| "logps/rejected": -296.09600830078125, |
| "loss": 0.1472, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.1014956459403038, |
| "rewards/margins": 4.947221755981445, |
| "rewards/rejected": -5.048717021942139, |
| "step": 1005 |
| }, |
| { |
| "epoch": 2.1536, |
| "grad_norm": 4.7639923095703125, |
| "learning_rate": 9.788579010963387e-07, |
| "logits/chosen": -1.6109592914581299, |
| "logits/rejected": -1.52823007106781, |
| "logps/chosen": -238.28677368164062, |
| "logps/rejected": -277.99713134765625, |
| "loss": 0.1226, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.02549382485449314, |
| "rewards/margins": 4.942860126495361, |
| "rewards/rejected": -4.968354225158691, |
| "step": 1010 |
| }, |
| { |
| "epoch": 2.164266666666667, |
| "grad_norm": 2.7891688346862793, |
| "learning_rate": 9.56111701030998e-07, |
| "logits/chosen": -1.5980685949325562, |
| "logits/rejected": -1.5289149284362793, |
| "logps/chosen": -217.5404510498047, |
| "logps/rejected": -278.1165771484375, |
| "loss": 0.1493, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": 0.06327895075082779, |
| "rewards/margins": 4.962241172790527, |
| "rewards/rejected": -4.898962020874023, |
| "step": 1015 |
| }, |
| { |
| "epoch": 2.174933333333333, |
| "grad_norm": 3.5160892009735107, |
| "learning_rate": 9.335702501071459e-07, |
| "logits/chosen": -1.5579856634140015, |
| "logits/rejected": -1.478334665298462, |
| "logps/chosen": -227.519287109375, |
| "logps/rejected": -287.3228454589844, |
| "loss": 0.1968, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.20410621166229248, |
| "rewards/margins": 4.94499397277832, |
| "rewards/rejected": -5.149099826812744, |
| "step": 1020 |
| }, |
| { |
| "epoch": 2.1856, |
| "grad_norm": 3.181624174118042, |
| "learning_rate": 9.112365377526636e-07, |
| "logits/chosen": -1.5951579809188843, |
| "logits/rejected": -1.5237982273101807, |
| "logps/chosen": -208.18765258789062, |
| "logps/rejected": -279.5474548339844, |
| "loss": 0.1626, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": 0.01243979949504137, |
| "rewards/margins": 5.2108306884765625, |
| "rewards/rejected": -5.198391437530518, |
| "step": 1025 |
| }, |
| { |
| "epoch": 2.196266666666667, |
| "grad_norm": 3.6655056476593018, |
| "learning_rate": 8.891135258453195e-07, |
| "logits/chosen": -1.4442431926727295, |
| "logits/rejected": -1.4688308238983154, |
| "logps/chosen": -217.04135131835938, |
| "logps/rejected": -294.37237548828125, |
| "loss": 0.1154, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.18168587982654572, |
| "rewards/margins": 4.986029624938965, |
| "rewards/rejected": -5.167714595794678, |
| "step": 1030 |
| }, |
| { |
| "epoch": 2.206933333333333, |
| "grad_norm": 2.8012073040008545, |
| "learning_rate": 8.672041483199709e-07, |
| "logits/chosen": -1.598311424255371, |
| "logits/rejected": -1.5263886451721191, |
| "logps/chosen": -225.3389892578125, |
| "logps/rejected": -280.2146911621094, |
| "loss": 0.1635, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.08202177286148071, |
| "rewards/margins": 4.779571056365967, |
| "rewards/rejected": -4.861592769622803, |
| "step": 1035 |
| }, |
| { |
| "epoch": 2.2176, |
| "grad_norm": 1.4678078889846802, |
| "learning_rate": 8.455113107794652e-07, |
| "logits/chosen": -1.5057955980300903, |
| "logits/rejected": -1.5607259273529053, |
| "logps/chosen": -228.93185424804688, |
| "logps/rejected": -289.180419921875, |
| "loss": 0.1137, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.21716734766960144, |
| "rewards/margins": 5.549811363220215, |
| "rewards/rejected": -5.7669782638549805, |
| "step": 1040 |
| }, |
| { |
| "epoch": 2.228266666666667, |
| "grad_norm": 3.0782907009124756, |
| "learning_rate": 8.240378901093035e-07, |
| "logits/chosen": -1.5692188739776611, |
| "logits/rejected": -1.4577603340148926, |
| "logps/chosen": -224.8596649169922, |
| "logps/rejected": -299.32440185546875, |
| "loss": 0.1343, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.03337126225233078, |
| "rewards/margins": 5.490138053894043, |
| "rewards/rejected": -5.5235090255737305, |
| "step": 1045 |
| }, |
| { |
| "epoch": 2.238933333333333, |
| "grad_norm": 2.948732376098633, |
| "learning_rate": 8.027867340961113e-07, |
| "logits/chosen": -1.5691108703613281, |
| "logits/rejected": -1.5023016929626465, |
| "logps/chosen": -221.1350555419922, |
| "logps/rejected": -278.23333740234375, |
| "loss": 0.1583, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.0295710451900959, |
| "rewards/margins": 5.049849510192871, |
| "rewards/rejected": -5.020279407501221, |
| "step": 1050 |
| }, |
| { |
| "epoch": 2.2496, |
| "grad_norm": 3.2476890087127686, |
| "learning_rate": 7.817606610499639e-07, |
| "logits/chosen": -1.5953521728515625, |
| "logits/rejected": -1.5542570352554321, |
| "logps/chosen": -232.19564819335938, |
| "logps/rejected": -282.22723388671875, |
| "loss": 0.1667, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.024515807628631592, |
| "rewards/margins": 5.146022319793701, |
| "rewards/rejected": -5.170538425445557, |
| "step": 1055 |
| }, |
| { |
| "epoch": 2.2602666666666664, |
| "grad_norm": 2.909899950027466, |
| "learning_rate": 7.609624594306278e-07, |
| "logits/chosen": -1.6917043924331665, |
| "logits/rejected": -1.5641858577728271, |
| "logps/chosen": -229.15185546875, |
| "logps/rejected": -308.77288818359375, |
| "loss": 0.1287, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.030151885002851486, |
| "rewards/margins": 5.283860206604004, |
| "rewards/rejected": -5.31401252746582, |
| "step": 1060 |
| }, |
| { |
| "epoch": 2.2709333333333332, |
| "grad_norm": 3.3673746585845947, |
| "learning_rate": 7.403948874777556e-07, |
| "logits/chosen": -1.5426510572433472, |
| "logits/rejected": -1.4823468923568726, |
| "logps/chosen": -231.9100341796875, |
| "logps/rejected": -267.9788818359375, |
| "loss": 0.1257, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.08240224421024323, |
| "rewards/margins": 4.7699127197265625, |
| "rewards/rejected": -4.8523149490356445, |
| "step": 1065 |
| }, |
| { |
| "epoch": 2.2816, |
| "grad_norm": 1.933462142944336, |
| "learning_rate": 7.20060672845091e-07, |
| "logits/chosen": -1.5066808462142944, |
| "logits/rejected": -1.4589513540267944, |
| "logps/chosen": -220.457763671875, |
| "logps/rejected": -277.82281494140625, |
| "loss": 0.1569, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.21246495842933655, |
| "rewards/margins": 5.227828025817871, |
| "rewards/rejected": -5.015363693237305, |
| "step": 1070 |
| }, |
| { |
| "epoch": 2.2922666666666665, |
| "grad_norm": 3.799119710922241, |
| "learning_rate": 6.99962512238733e-07, |
| "logits/chosen": -1.4951709508895874, |
| "logits/rejected": -1.448192834854126, |
| "logps/chosen": -224.44320678710938, |
| "logps/rejected": -273.17864990234375, |
| "loss": 0.1707, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": -0.21202436089515686, |
| "rewards/margins": 4.784801483154297, |
| "rewards/rejected": -4.996825218200684, |
| "step": 1075 |
| }, |
| { |
| "epoch": 2.3029333333333333, |
| "grad_norm": 3.391683340072632, |
| "learning_rate": 6.801030710594949e-07, |
| "logits/chosen": -1.4559767246246338, |
| "logits/rejected": -1.3820351362228394, |
| "logps/chosen": -224.79928588867188, |
| "logps/rejected": -291.6627197265625, |
| "loss": 0.1311, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": 0.032825786620378494, |
| "rewards/margins": 5.525381565093994, |
| "rewards/rejected": -5.492555618286133, |
| "step": 1080 |
| }, |
| { |
| "epoch": 2.3136, |
| "grad_norm": 2.596304416656494, |
| "learning_rate": 6.604849830494264e-07, |
| "logits/chosen": -1.6544069051742554, |
| "logits/rejected": -1.7071607112884521, |
| "logps/chosen": -212.39675903320312, |
| "logps/rejected": -295.53192138671875, |
| "loss": 0.1484, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.38105496764183044, |
| "rewards/margins": 5.09371280670166, |
| "rewards/rejected": -5.474766731262207, |
| "step": 1085 |
| }, |
| { |
| "epoch": 2.3242666666666665, |
| "grad_norm": 3.5445754528045654, |
| "learning_rate": 6.411108499425253e-07, |
| "logits/chosen": -1.6285476684570312, |
| "logits/rejected": -1.4937007427215576, |
| "logps/chosen": -231.6444091796875, |
| "logps/rejected": -286.3141174316406, |
| "loss": 0.1976, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.24693647027015686, |
| "rewards/margins": 4.544675827026367, |
| "rewards/rejected": -4.791612148284912, |
| "step": 1090 |
| }, |
| { |
| "epoch": 2.3349333333333333, |
| "grad_norm": 6.524362087249756, |
| "learning_rate": 6.219832411197013e-07, |
| "logits/chosen": -1.4855129718780518, |
| "logits/rejected": -1.4172195196151733, |
| "logps/chosen": -240.4997100830078, |
| "logps/rejected": -289.3425598144531, |
| "loss": 0.1824, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.5645937323570251, |
| "rewards/margins": 4.782386302947998, |
| "rewards/rejected": -5.346980094909668, |
| "step": 1095 |
| }, |
| { |
| "epoch": 2.3456, |
| "grad_norm": 2.2315993309020996, |
| "learning_rate": 6.031046932680229e-07, |
| "logits/chosen": -1.5639643669128418, |
| "logits/rejected": -1.477234125137329, |
| "logps/chosen": -211.5214080810547, |
| "logps/rejected": -280.4677429199219, |
| "loss": 0.1117, |
| "rewards/accuracies": 0.981249988079071, |
| "rewards/chosen": -0.0826307162642479, |
| "rewards/margins": 5.693845748901367, |
| "rewards/rejected": -5.776476860046387, |
| "step": 1100 |
| }, |
| { |
| "epoch": 2.3562666666666665, |
| "grad_norm": 2.92634654045105, |
| "learning_rate": 5.844777100443091e-07, |
| "logits/chosen": -1.5262413024902344, |
| "logits/rejected": -1.5272846221923828, |
| "logps/chosen": -213.702880859375, |
| "logps/rejected": -303.1562194824219, |
| "loss": 0.1217, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.05086277052760124, |
| "rewards/margins": 5.797705173492432, |
| "rewards/rejected": -5.848567485809326, |
| "step": 1105 |
| }, |
| { |
| "epoch": 2.3669333333333333, |
| "grad_norm": 6.668047904968262, |
| "learning_rate": 5.66104761743092e-07, |
| "logits/chosen": -1.673953652381897, |
| "logits/rejected": -1.703916311264038, |
| "logps/chosen": -208.412109375, |
| "logps/rejected": -313.27484130859375, |
| "loss": 0.1454, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.5052535533905029, |
| "rewards/margins": 5.835505485534668, |
| "rewards/rejected": -6.34075927734375, |
| "step": 1110 |
| }, |
| { |
| "epoch": 2.3776, |
| "grad_norm": 3.2992238998413086, |
| "learning_rate": 5.479882849690102e-07, |
| "logits/chosen": -1.5450654029846191, |
| "logits/rejected": -1.51485276222229, |
| "logps/chosen": -216.7915802001953, |
| "logps/rejected": -273.28741455078125, |
| "loss": 0.1854, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.23610785603523254, |
| "rewards/margins": 4.69701623916626, |
| "rewards/rejected": -4.933123588562012, |
| "step": 1115 |
| }, |
| { |
| "epoch": 2.3882666666666665, |
| "grad_norm": 2.530855417251587, |
| "learning_rate": 5.301306823136687e-07, |
| "logits/chosen": -1.5250041484832764, |
| "logits/rejected": -1.4885377883911133, |
| "logps/chosen": -264.8410339355469, |
| "logps/rejected": -289.5926818847656, |
| "loss": 0.1345, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": 0.18493278324604034, |
| "rewards/margins": 5.43831205368042, |
| "rewards/rejected": -5.253379821777344, |
| "step": 1120 |
| }, |
| { |
| "epoch": 2.3989333333333334, |
| "grad_norm": 2.140273332595825, |
| "learning_rate": 5.125343220370088e-07, |
| "logits/chosen": -1.499107837677002, |
| "logits/rejected": -1.499398946762085, |
| "logps/chosen": -222.88674926757812, |
| "logps/rejected": -291.7001037597656, |
| "loss": 0.1099, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.3723798394203186, |
| "rewards/margins": 5.258582592010498, |
| "rewards/rejected": -5.630963325500488, |
| "step": 1125 |
| }, |
| { |
| "epoch": 2.4096, |
| "grad_norm": 1.6813573837280273, |
| "learning_rate": 4.952015377532329e-07, |
| "logits/chosen": -1.64093017578125, |
| "logits/rejected": -1.5066547393798828, |
| "logps/chosen": -230.73257446289062, |
| "logps/rejected": -288.6161804199219, |
| "loss": 0.1134, |
| "rewards/accuracies": 0.9750000238418579, |
| "rewards/chosen": -0.4509183466434479, |
| "rewards/margins": 5.539261817932129, |
| "rewards/rejected": -5.990180015563965, |
| "step": 1130 |
| }, |
| { |
| "epoch": 2.4202666666666666, |
| "grad_norm": 3.6021177768707275, |
| "learning_rate": 4.781346281213204e-07, |
| "logits/chosen": -1.5829105377197266, |
| "logits/rejected": -1.5672820806503296, |
| "logps/chosen": -222.74728393554688, |
| "logps/rejected": -302.9520263671875, |
| "loss": 0.1154, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.12925612926483154, |
| "rewards/margins": 5.923072814941406, |
| "rewards/rejected": -6.052329063415527, |
| "step": 1135 |
| }, |
| { |
| "epoch": 2.4309333333333334, |
| "grad_norm": 4.091590881347656, |
| "learning_rate": 4.613358565401818e-07, |
| "logits/chosen": -1.4938812255859375, |
| "logits/rejected": -1.484735131263733, |
| "logps/chosen": -238.08212280273438, |
| "logps/rejected": -304.36090087890625, |
| "loss": 0.1727, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.22507795691490173, |
| "rewards/margins": 5.176184177398682, |
| "rewards/rejected": -5.401261806488037, |
| "step": 1140 |
| }, |
| { |
| "epoch": 2.4416, |
| "grad_norm": 4.568892002105713, |
| "learning_rate": 4.448074508484906e-07, |
| "logits/chosen": -1.6104847192764282, |
| "logits/rejected": -1.5651620626449585, |
| "logps/chosen": -220.1466827392578, |
| "logps/rejected": -282.4197998046875, |
| "loss": 0.1564, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.18282730877399445, |
| "rewards/margins": 5.292811393737793, |
| "rewards/rejected": -5.4756388664245605, |
| "step": 1145 |
| }, |
| { |
| "epoch": 2.4522666666666666, |
| "grad_norm": 3.958512306213379, |
| "learning_rate": 4.2855160302922777e-07, |
| "logits/chosen": -1.5375100374221802, |
| "logits/rejected": -1.5117011070251465, |
| "logps/chosen": -217.5662078857422, |
| "logps/rejected": -295.0578918457031, |
| "loss": 0.141, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.3425526022911072, |
| "rewards/margins": 5.050057888031006, |
| "rewards/rejected": -5.392610549926758, |
| "step": 1150 |
| }, |
| { |
| "epoch": 2.4629333333333334, |
| "grad_norm": 4.313801288604736, |
| "learning_rate": 4.125704689189819e-07, |
| "logits/chosen": -1.474077582359314, |
| "logits/rejected": -1.488755702972412, |
| "logps/chosen": -222.3016815185547, |
| "logps/rejected": -293.5660095214844, |
| "loss": 0.2337, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4544387459754944, |
| "rewards/margins": 4.519167900085449, |
| "rewards/rejected": -4.973607063293457, |
| "step": 1155 |
| }, |
| { |
| "epoch": 2.4736000000000002, |
| "grad_norm": 2.2440829277038574, |
| "learning_rate": 3.9686616792204677e-07, |
| "logits/chosen": -1.5298980474472046, |
| "logits/rejected": -1.526539921760559, |
| "logps/chosen": -223.37890625, |
| "logps/rejected": -300.4910583496094, |
| "loss": 0.132, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.316823810338974, |
| "rewards/margins": 5.748261451721191, |
| "rewards/rejected": -6.065085411071777, |
| "step": 1160 |
| }, |
| { |
| "epoch": 2.4842666666666666, |
| "grad_norm": 3.3815062046051025, |
| "learning_rate": 3.8144078272934676e-07, |
| "logits/chosen": -1.5895965099334717, |
| "logits/rejected": -1.5554959774017334, |
| "logps/chosen": -239.8270721435547, |
| "logps/rejected": -320.05853271484375, |
| "loss": 0.1482, |
| "rewards/accuracies": 0.925000011920929, |
| "rewards/chosen": 0.015445577912032604, |
| "rewards/margins": 5.61312198638916, |
| "rewards/rejected": -5.597676753997803, |
| "step": 1165 |
| }, |
| { |
| "epoch": 2.4949333333333334, |
| "grad_norm": 2.022261381149292, |
| "learning_rate": 3.6629635904222943e-07, |
| "logits/chosen": -1.5600824356079102, |
| "logits/rejected": -1.5448620319366455, |
| "logps/chosen": -228.0321044921875, |
| "logps/rejected": -286.53924560546875, |
| "loss": 0.1514, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.341894268989563, |
| "rewards/margins": 4.940691947937012, |
| "rewards/rejected": -5.282585620880127, |
| "step": 1170 |
| }, |
| { |
| "epoch": 2.5056000000000003, |
| "grad_norm": 3.8453714847564697, |
| "learning_rate": 3.514349053011698e-07, |
| "logits/chosen": -1.5347708463668823, |
| "logits/rejected": -1.5214083194732666, |
| "logps/chosen": -216.1919403076172, |
| "logps/rejected": -290.57806396484375, |
| "loss": 0.1768, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.2583093047142029, |
| "rewards/margins": 4.864516735076904, |
| "rewards/rejected": -5.122825622558594, |
| "step": 1175 |
| }, |
| { |
| "epoch": 2.5162666666666667, |
| "grad_norm": 3.6349446773529053, |
| "learning_rate": 3.3685839241941133e-07, |
| "logits/chosen": -1.4968597888946533, |
| "logits/rejected": -1.4725162982940674, |
| "logps/chosen": -223.86300659179688, |
| "logps/rejected": -276.0819396972656, |
| "loss": 0.1417, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.19201938807964325, |
| "rewards/margins": 5.1369218826293945, |
| "rewards/rejected": -5.328940391540527, |
| "step": 1180 |
| }, |
| { |
| "epoch": 2.5269333333333335, |
| "grad_norm": 2.9113125801086426, |
| "learning_rate": 3.2256875352158435e-07, |
| "logits/chosen": -1.5767369270324707, |
| "logits/rejected": -1.5519667863845825, |
| "logps/chosen": -227.7290496826172, |
| "logps/rejected": -302.18341064453125, |
| "loss": 0.1685, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.14696352183818817, |
| "rewards/margins": 5.197046756744385, |
| "rewards/rejected": -5.344009876251221, |
| "step": 1185 |
| }, |
| { |
| "epoch": 2.5376, |
| "grad_norm": 1.3454444408416748, |
| "learning_rate": 3.085678836873399e-07, |
| "logits/chosen": -1.6368871927261353, |
| "logits/rejected": -1.610581398010254, |
| "logps/chosen": -218.0592803955078, |
| "logps/rejected": -311.0408630371094, |
| "loss": 0.1269, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.03764452040195465, |
| "rewards/margins": 5.279090881347656, |
| "rewards/rejected": -5.241446495056152, |
| "step": 1190 |
| }, |
| { |
| "epoch": 2.5482666666666667, |
| "grad_norm": 7.484287261962891, |
| "learning_rate": 2.9485763970002106e-07, |
| "logits/chosen": -1.591990351676941, |
| "logits/rejected": -1.5665239095687866, |
| "logps/chosen": -212.2829132080078, |
| "logps/rejected": -309.5106506347656, |
| "loss": 0.1343, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": 0.053197573870420456, |
| "rewards/margins": 5.905511856079102, |
| "rewards/rejected": -5.8523149490356445, |
| "step": 1195 |
| }, |
| { |
| "epoch": 2.558933333333333, |
| "grad_norm": 3.2489264011383057, |
| "learning_rate": 2.8143983980042174e-07, |
| "logits/chosen": -1.541517734527588, |
| "logits/rejected": -1.4897912740707397, |
| "logps/chosen": -216.9764862060547, |
| "logps/rejected": -293.786865234375, |
| "loss": 0.1538, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.21131077408790588, |
| "rewards/margins": 5.59000301361084, |
| "rewards/rejected": -5.801313877105713, |
| "step": 1200 |
| }, |
| { |
| "epoch": 2.5696, |
| "grad_norm": 4.643258571624756, |
| "learning_rate": 2.683162634456515e-07, |
| "logits/chosen": -1.5824702978134155, |
| "logits/rejected": -1.5110851526260376, |
| "logps/chosen": -246.80264282226562, |
| "logps/rejected": -279.2035217285156, |
| "loss": 0.1596, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.21497675776481628, |
| "rewards/margins": 4.839934349060059, |
| "rewards/rejected": -5.054910659790039, |
| "step": 1205 |
| }, |
| { |
| "epoch": 2.5802666666666667, |
| "grad_norm": 5.287117004394531, |
| "learning_rate": 2.5548865107314606e-07, |
| "logits/chosen": -1.4827041625976562, |
| "logits/rejected": -1.4560563564300537, |
| "logps/chosen": -254.863525390625, |
| "logps/rejected": -300.84552001953125, |
| "loss": 0.1416, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.15708236396312714, |
| "rewards/margins": 5.223883152008057, |
| "rewards/rejected": -5.380965232849121, |
| "step": 1210 |
| }, |
| { |
| "epoch": 2.590933333333333, |
| "grad_norm": 6.302121162414551, |
| "learning_rate": 2.429587038698511e-07, |
| "logits/chosen": -1.6150802373886108, |
| "logits/rejected": -1.5412870645523071, |
| "logps/chosen": -223.01675415039062, |
| "logps/rejected": -272.59735107421875, |
| "loss": 0.1892, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.3169715702533722, |
| "rewards/margins": 4.87038516998291, |
| "rewards/rejected": -5.187356948852539, |
| "step": 1215 |
| }, |
| { |
| "epoch": 2.6016, |
| "grad_norm": 4.08319091796875, |
| "learning_rate": 2.307280835466144e-07, |
| "logits/chosen": -1.5628572702407837, |
| "logits/rejected": -1.4623311758041382, |
| "logps/chosen": -210.0499267578125, |
| "logps/rejected": -287.72686767578125, |
| "loss": 0.1337, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.2616520822048187, |
| "rewards/margins": 5.518267631530762, |
| "rewards/rejected": -5.779919624328613, |
| "step": 1220 |
| }, |
| { |
| "epoch": 2.6122666666666667, |
| "grad_norm": 3.5064926147460938, |
| "learning_rate": 2.1879841211780856e-07, |
| "logits/chosen": -1.5222089290618896, |
| "logits/rejected": -1.5433714389801025, |
| "logps/chosen": -223.13699340820312, |
| "logps/rejected": -277.22027587890625, |
| "loss": 0.1881, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.1823982298374176, |
| "rewards/margins": 4.619892120361328, |
| "rewards/rejected": -4.802290916442871, |
| "step": 1225 |
| }, |
| { |
| "epoch": 2.622933333333333, |
| "grad_norm": 5.202704906463623, |
| "learning_rate": 2.0717127168622202e-07, |
| "logits/chosen": -1.495309591293335, |
| "logits/rejected": -1.4390442371368408, |
| "logps/chosen": -219.75357055664062, |
| "logps/rejected": -292.7464599609375, |
| "loss": 0.1962, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.33655446767807007, |
| "rewards/margins": 4.839663982391357, |
| "rewards/rejected": -5.1762189865112305, |
| "step": 1230 |
| }, |
| { |
| "epoch": 2.6336, |
| "grad_norm": 2.8011467456817627, |
| "learning_rate": 1.958482042332424e-07, |
| "logits/chosen": -1.6091105937957764, |
| "logits/rejected": -1.5161319971084595, |
| "logps/chosen": -237.4774627685547, |
| "logps/rejected": -300.9270324707031, |
| "loss": 0.1419, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.3717946410179138, |
| "rewards/margins": 5.390846252441406, |
| "rewards/rejected": -5.762640953063965, |
| "step": 1235 |
| }, |
| { |
| "epoch": 2.6442666666666668, |
| "grad_norm": 3.398773193359375, |
| "learning_rate": 1.8483071141435937e-07, |
| "logits/chosen": -1.5985854864120483, |
| "logits/rejected": -1.5678365230560303, |
| "logps/chosen": -239.23867797851562, |
| "logps/rejected": -289.22503662109375, |
| "loss": 0.1431, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.35332396626472473, |
| "rewards/margins": 4.7726006507873535, |
| "rewards/rejected": -5.125924587249756, |
| "step": 1240 |
| }, |
| { |
| "epoch": 2.654933333333333, |
| "grad_norm": 4.891382217407227, |
| "learning_rate": 1.741202543600176e-07, |
| "logits/chosen": -1.581493616104126, |
| "logits/rejected": -1.582097053527832, |
| "logps/chosen": -207.20639038085938, |
| "logps/rejected": -289.98748779296875, |
| "loss": 0.1611, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.2472093105316162, |
| "rewards/margins": 5.706404209136963, |
| "rewards/rejected": -5.95361328125, |
| "step": 1245 |
| }, |
| { |
| "epoch": 2.6656, |
| "grad_norm": 2.183753728866577, |
| "learning_rate": 1.6371825348184155e-07, |
| "logits/chosen": -1.5279576778411865, |
| "logits/rejected": -1.4338546991348267, |
| "logps/chosen": -278.4076232910156, |
| "logps/rejected": -308.9873046875, |
| "loss": 0.1213, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.06257566064596176, |
| "rewards/margins": 6.102546691894531, |
| "rewards/rejected": -6.165122032165527, |
| "step": 1250 |
| }, |
| { |
| "epoch": 2.676266666666667, |
| "grad_norm": 4.505012035369873, |
| "learning_rate": 1.5362608828426246e-07, |
| "logits/chosen": -1.498521089553833, |
| "logits/rejected": -1.533311128616333, |
| "logps/chosen": -235.9097900390625, |
| "logps/rejected": -305.294677734375, |
| "loss": 0.1317, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.4602181017398834, |
| "rewards/margins": 5.288244247436523, |
| "rewards/rejected": -5.748462200164795, |
| "step": 1255 |
| }, |
| { |
| "epoch": 2.686933333333333, |
| "grad_norm": 3.8435416221618652, |
| "learning_rate": 1.438450971815686e-07, |
| "logits/chosen": -1.635210633277893, |
| "logits/rejected": -1.6011021137237549, |
| "logps/chosen": -203.96682739257812, |
| "logps/rejected": -292.73248291015625, |
| "loss": 0.1212, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.29221805930137634, |
| "rewards/margins": 5.458622932434082, |
| "rewards/rejected": -5.7508416175842285, |
| "step": 1260 |
| }, |
| { |
| "epoch": 2.6976, |
| "grad_norm": 3.876904249191284, |
| "learning_rate": 1.3437657732040783e-07, |
| "logits/chosen": -1.5189704895019531, |
| "logits/rejected": -1.479864239692688, |
| "logps/chosen": -234.113037109375, |
| "logps/rejected": -301.35260009765625, |
| "loss": 0.1127, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.4382593035697937, |
| "rewards/margins": 5.527853488922119, |
| "rewards/rejected": -5.9661126136779785, |
| "step": 1265 |
| }, |
| { |
| "epoch": 2.708266666666667, |
| "grad_norm": 3.5501556396484375, |
| "learning_rate": 1.2522178440776022e-07, |
| "logits/chosen": -1.5354712009429932, |
| "logits/rejected": -1.5287415981292725, |
| "logps/chosen": -232.2896270751953, |
| "logps/rejected": -295.0993957519531, |
| "loss": 0.1816, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.33327555656433105, |
| "rewards/margins": 5.117754936218262, |
| "rewards/rejected": -5.451030731201172, |
| "step": 1270 |
| }, |
| { |
| "epoch": 2.718933333333333, |
| "grad_norm": 4.883026599884033, |
| "learning_rate": 1.1638193254440655e-07, |
| "logits/chosen": -1.546303391456604, |
| "logits/rejected": -1.469744086265564, |
| "logps/chosen": -218.6349639892578, |
| "logps/rejected": -278.43115234375, |
| "loss": 0.1493, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.04183761402964592, |
| "rewards/margins": 5.151387691497803, |
| "rewards/rejected": -5.193225383758545, |
| "step": 1275 |
| }, |
| { |
| "epoch": 2.7296, |
| "grad_norm": 3.7158358097076416, |
| "learning_rate": 1.0785819406391562e-07, |
| "logits/chosen": -1.535130262374878, |
| "logits/rejected": -1.5816833972930908, |
| "logps/chosen": -226.7908172607422, |
| "logps/rejected": -315.193603515625, |
| "loss": 0.1752, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.836959958076477, |
| "rewards/margins": 5.0551886558532715, |
| "rewards/rejected": -5.892148971557617, |
| "step": 1280 |
| }, |
| { |
| "epoch": 2.740266666666667, |
| "grad_norm": 3.4619786739349365, |
| "learning_rate": 9.965169937717095e-08, |
| "logits/chosen": -1.5055862665176392, |
| "logits/rejected": -1.485164999961853, |
| "logps/chosen": -214.0679473876953, |
| "logps/rejected": -299.93524169921875, |
| "loss": 0.1093, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.2596716284751892, |
| "rewards/margins": 5.675719261169434, |
| "rewards/rejected": -5.935390949249268, |
| "step": 1285 |
| }, |
| { |
| "epoch": 2.7509333333333332, |
| "grad_norm": 2.670748472213745, |
| "learning_rate": 9.176353682245676e-08, |
| "logits/chosen": -1.5897644758224487, |
| "logits/rejected": -1.4919850826263428, |
| "logps/chosen": -233.66592407226562, |
| "logps/rejected": -283.1734924316406, |
| "loss": 0.1356, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.1264813244342804, |
| "rewards/margins": 5.175812721252441, |
| "rewards/rejected": -5.302294731140137, |
| "step": 1290 |
| }, |
| { |
| "epoch": 2.7616, |
| "grad_norm": 3.714576482772827, |
| "learning_rate": 8.41947525211223e-08, |
| "logits/chosen": -1.508699655532837, |
| "logits/rejected": -1.4295189380645752, |
| "logps/chosen": -229.2381591796875, |
| "logps/rejected": -296.42657470703125, |
| "loss": 0.1774, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.4729483127593994, |
| "rewards/margins": 5.336666584014893, |
| "rewards/rejected": -5.809614658355713, |
| "step": 1295 |
| }, |
| { |
| "epoch": 2.772266666666667, |
| "grad_norm": 4.889886379241943, |
| "learning_rate": 7.694635023884789e-08, |
| "logits/chosen": -1.4413424730300903, |
| "logits/rejected": -1.502366304397583, |
| "logps/chosen": -234.2248992919922, |
| "logps/rejected": -292.7245788574219, |
| "loss": 0.1684, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.6228765845298767, |
| "rewards/margins": 4.261327743530273, |
| "rewards/rejected": -4.884203910827637, |
| "step": 1300 |
| }, |
| { |
| "epoch": 2.7829333333333333, |
| "grad_norm": 3.7302002906799316, |
| "learning_rate": 7.00192912525241e-08, |
| "logits/chosen": -1.5013799667358398, |
| "logits/rejected": -1.5552741289138794, |
| "logps/chosen": -219.25308227539062, |
| "logps/rejected": -306.8885803222656, |
| "loss": 0.1487, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.20830953121185303, |
| "rewards/margins": 5.794388771057129, |
| "rewards/rejected": -6.002697467803955, |
| "step": 1305 |
| }, |
| { |
| "epoch": 2.7936, |
| "grad_norm": 4.609808444976807, |
| "learning_rate": 6.341449422277047e-08, |
| "logits/chosen": -1.4839293956756592, |
| "logits/rejected": -1.5049326419830322, |
| "logps/chosen": -248.7401885986328, |
| "logps/rejected": -316.1900939941406, |
| "loss": 0.1847, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.1252104490995407, |
| "rewards/margins": 5.684288024902344, |
| "rewards/rejected": -5.809498310089111, |
| "step": 1310 |
| }, |
| { |
| "epoch": 2.804266666666667, |
| "grad_norm": 3.7130322456359863, |
| "learning_rate": 5.7132835072101486e-08, |
| "logits/chosen": -1.5436820983886719, |
| "logits/rejected": -1.5339748859405518, |
| "logps/chosen": -221.01025390625, |
| "logps/rejected": -291.3807373046875, |
| "loss": 0.1641, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.1684941202402115, |
| "rewards/margins": 5.483412742614746, |
| "rewards/rejected": -5.651907444000244, |
| "step": 1315 |
| }, |
| { |
| "epoch": 2.8149333333333333, |
| "grad_norm": 4.641164302825928, |
| "learning_rate": 5.117514686876379e-08, |
| "logits/chosen": -1.4652761220932007, |
| "logits/rejected": -1.3969151973724365, |
| "logps/chosen": -221.5576629638672, |
| "logps/rejected": -288.90411376953125, |
| "loss": 0.1661, |
| "rewards/accuracies": 0.9437500238418579, |
| "rewards/chosen": -0.37317955493927, |
| "rewards/margins": 5.2659220695495605, |
| "rewards/rejected": -5.639101505279541, |
| "step": 1320 |
| }, |
| { |
| "epoch": 2.8256, |
| "grad_norm": 1.6908352375030518, |
| "learning_rate": 4.5542219716255274e-08, |
| "logits/chosen": -1.651746392250061, |
| "logits/rejected": -1.603569746017456, |
| "logps/chosen": -239.42745971679688, |
| "logps/rejected": -300.3352966308594, |
| "loss": 0.1821, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.31848442554473877, |
| "rewards/margins": 5.208474159240723, |
| "rewards/rejected": -5.526959419250488, |
| "step": 1325 |
| }, |
| { |
| "epoch": 2.836266666666667, |
| "grad_norm": 2.5582754611968994, |
| "learning_rate": 4.023480064854174e-08, |
| "logits/chosen": -1.669661283493042, |
| "logits/rejected": -1.6277196407318115, |
| "logps/chosen": -222.32205200195312, |
| "logps/rejected": -291.809814453125, |
| "loss": 0.1485, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.6328510046005249, |
| "rewards/margins": 4.7722649574279785, |
| "rewards/rejected": -5.405115604400635, |
| "step": 1330 |
| }, |
| { |
| "epoch": 2.8469333333333333, |
| "grad_norm": 3.1268057823181152, |
| "learning_rate": 3.525359353098584e-08, |
| "logits/chosen": -1.5684897899627686, |
| "logits/rejected": -1.5883140563964844, |
| "logps/chosen": -231.85134887695312, |
| "logps/rejected": -299.2911071777344, |
| "loss": 0.1581, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.49852675199508667, |
| "rewards/margins": 5.060816764831543, |
| "rewards/rejected": -5.5593438148498535, |
| "step": 1335 |
| }, |
| { |
| "epoch": 2.8576, |
| "grad_norm": 1.8423947095870972, |
| "learning_rate": 3.059925896700094e-08, |
| "logits/chosen": -1.4214311838150024, |
| "logits/rejected": -1.356693983078003, |
| "logps/chosen": -239.08755493164062, |
| "logps/rejected": -274.2549133300781, |
| "loss": 0.1151, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.49795475602149963, |
| "rewards/margins": 4.846909046173096, |
| "rewards/rejected": -5.344862937927246, |
| "step": 1340 |
| }, |
| { |
| "epoch": 2.8682666666666665, |
| "grad_norm": 3.8253424167633057, |
| "learning_rate": 2.6272414210443964e-08, |
| "logits/chosen": -1.5384304523468018, |
| "logits/rejected": -1.509032964706421, |
| "logps/chosen": -236.12686157226562, |
| "logps/rejected": -284.77642822265625, |
| "loss": 0.2041, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.509702742099762, |
| "rewards/margins": 4.945359230041504, |
| "rewards/rejected": -5.455061435699463, |
| "step": 1345 |
| }, |
| { |
| "epoch": 2.8789333333333333, |
| "grad_norm": 3.1354925632476807, |
| "learning_rate": 2.2273633083753632e-08, |
| "logits/chosen": -1.6047881841659546, |
| "logits/rejected": -1.5616092681884766, |
| "logps/chosen": -213.57376098632812, |
| "logps/rejected": -300.4778747558594, |
| "loss": 0.1115, |
| "rewards/accuracies": 0.9624999761581421, |
| "rewards/chosen": -0.4236835539340973, |
| "rewards/margins": 5.589079856872559, |
| "rewards/rejected": -6.012763023376465, |
| "step": 1350 |
| }, |
| { |
| "epoch": 2.8895999999999997, |
| "grad_norm": 2.7940657138824463, |
| "learning_rate": 1.8603445901851647e-08, |
| "logits/chosen": -1.6039320230484009, |
| "logits/rejected": -1.4985624551773071, |
| "logps/chosen": -236.09060668945312, |
| "logps/rejected": -281.80096435546875, |
| "loss": 0.1656, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.3135233521461487, |
| "rewards/margins": 4.965387344360352, |
| "rewards/rejected": -5.2789106369018555, |
| "step": 1355 |
| }, |
| { |
| "epoch": 2.9002666666666665, |
| "grad_norm": 4.520009517669678, |
| "learning_rate": 1.5262339401813375e-08, |
| "logits/chosen": -1.6661808490753174, |
| "logits/rejected": -1.5840932130813599, |
| "logps/chosen": -219.1964111328125, |
| "logps/rejected": -289.63824462890625, |
| "loss": 0.1875, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.30475789308547974, |
| "rewards/margins": 5.524220943450928, |
| "rewards/rejected": -5.828978538513184, |
| "step": 1360 |
| }, |
| { |
| "epoch": 2.9109333333333334, |
| "grad_norm": 3.332031726837158, |
| "learning_rate": 1.2250756678315601e-08, |
| "logits/chosen": -1.5584747791290283, |
| "logits/rejected": -1.5064611434936523, |
| "logps/chosen": -241.1908721923828, |
| "logps/rejected": -306.5281677246094, |
| "loss": 0.1813, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.28189021348953247, |
| "rewards/margins": 5.6129279136657715, |
| "rewards/rejected": -5.8948187828063965, |
| "step": 1365 |
| }, |
| { |
| "epoch": 2.9215999999999998, |
| "grad_norm": 3.0526838302612305, |
| "learning_rate": 9.569097124875193e-09, |
| "logits/chosen": -1.555757999420166, |
| "logits/rejected": -1.552303671836853, |
| "logps/chosen": -229.64736938476562, |
| "logps/rejected": -298.7793273925781, |
| "loss": 0.1325, |
| "rewards/accuracies": 0.949999988079071, |
| "rewards/chosen": -0.7364352345466614, |
| "rewards/margins": 5.4410200119018555, |
| "rewards/rejected": -6.177455425262451, |
| "step": 1370 |
| }, |
| { |
| "epoch": 2.9322666666666666, |
| "grad_norm": 4.28289270401001, |
| "learning_rate": 7.217716380881479e-09, |
| "logits/chosen": -1.6077375411987305, |
| "logits/rejected": -1.6054372787475586, |
| "logps/chosen": -244.438720703125, |
| "logps/rejected": -294.0166320800781, |
| "loss": 0.1795, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.4182673394680023, |
| "rewards/margins": 5.0655107498168945, |
| "rewards/rejected": -5.483778476715088, |
| "step": 1375 |
| }, |
| { |
| "epoch": 2.9429333333333334, |
| "grad_norm": 3.2869319915771484, |
| "learning_rate": 5.196926284430359e-09, |
| "logits/chosen": -1.563001036643982, |
| "logits/rejected": -1.467656135559082, |
| "logps/chosen": -226.94686889648438, |
| "logps/rejected": -281.23114013671875, |
| "loss": 0.1337, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.4600711464881897, |
| "rewards/margins": 5.309510707855225, |
| "rewards/rejected": -5.7695817947387695, |
| "step": 1380 |
| }, |
| { |
| "epoch": 2.9536, |
| "grad_norm": 2.482046365737915, |
| "learning_rate": 3.5069948309712775e-09, |
| "logits/chosen": -1.5494192838668823, |
| "logits/rejected": -1.5291993618011475, |
| "logps/chosen": -215.4961395263672, |
| "logps/rejected": -297.94671630859375, |
| "loss": 0.1336, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3590046763420105, |
| "rewards/margins": 5.251859188079834, |
| "rewards/rejected": -5.610864162445068, |
| "step": 1385 |
| }, |
| { |
| "epoch": 2.9642666666666666, |
| "grad_norm": 2.1505300998687744, |
| "learning_rate": 2.1481461377634294e-09, |
| "logits/chosen": -1.5560667514801025, |
| "logits/rejected": -1.570675253868103, |
| "logps/chosen": -232.88662719726562, |
| "logps/rejected": -308.8785400390625, |
| "loss": 0.1682, |
| "rewards/accuracies": 0.9312499761581421, |
| "rewards/chosen": -0.34640324115753174, |
| "rewards/margins": 5.028754711151123, |
| "rewards/rejected": -5.375157356262207, |
| "step": 1390 |
| }, |
| { |
| "epoch": 2.9749333333333334, |
| "grad_norm": 2.367377758026123, |
| "learning_rate": 1.120560414154259e-09, |
| "logits/chosen": -1.5260568857192993, |
| "logits/rejected": -1.4501224756240845, |
| "logps/chosen": -213.99169921875, |
| "logps/rejected": -291.2949523925781, |
| "loss": 0.1375, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.27664098143577576, |
| "rewards/margins": 5.49239444732666, |
| "rewards/rejected": -5.769036769866943, |
| "step": 1395 |
| }, |
| { |
| "epoch": 2.9856, |
| "grad_norm": 2.2598955631256104, |
| "learning_rate": 4.2437393768079983e-10, |
| "logits/chosen": -1.4077026844024658, |
| "logits/rejected": -1.4860719442367554, |
| "logps/chosen": -228.6597137451172, |
| "logps/rejected": -294.4121398925781, |
| "loss": 0.138, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.3458075523376465, |
| "rewards/margins": 5.261725902557373, |
| "rewards/rejected": -5.6075334548950195, |
| "step": 1400 |
| }, |
| { |
| "epoch": 2.9962666666666666, |
| "grad_norm": 3.635240077972412, |
| "learning_rate": 5.967903599635171e-11, |
| "logits/chosen": -1.5582494735717773, |
| "logits/rejected": -1.4538967609405518, |
| "logps/chosen": -232.40707397460938, |
| "logps/rejected": -286.7071838378906, |
| "loss": 0.1605, |
| "rewards/accuracies": 0.956250011920929, |
| "rewards/chosen": -0.3285108506679535, |
| "rewards/margins": 5.033994674682617, |
| "rewards/rejected": -5.362505912780762, |
| "step": 1405 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 1407, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 3, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
