diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,6351 @@ +{ + "best_global_step": 300, + "best_metric": 0.68528539, + "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_absgt10_noidk__pari0.5_0602_p2prompt_r16/v0-20250604-235901/checkpoint-300", + "epoch": 0.8802263439170073, + "eval_steps": 300, + "global_step": 350, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.002514932411191449, + "grad_norm": 5.559185028076172, + "learning_rate": 3.3333333333333333e-06, + "logits/chosen": -0.5158984065055847, + "logits/rejected": -0.6433685421943665, + "logps/chosen": -8.879493713378906, + "logps/rejected": -18.298219680786133, + "loss": 1.2430726289749146, + "memory(GiB)": 30.69, + "nll_loss": 0.5499253869056702, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1, + "train_speed(iter/s)": 0.010477 + }, + { + "epoch": 0.005029864822382898, + "grad_norm": 5.400235176086426, + "learning_rate": 6.666666666666667e-06, + "logits/chosen": -0.5219721794128418, + "logits/rejected": -0.6263731718063354, + "logps/chosen": -4.77984094619751, + "logps/rejected": -15.150838851928711, + "loss": 1.0675525665283203, + "memory(GiB)": 33.9, + "nll_loss": 0.3744054436683655, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2, + "train_speed(iter/s)": 0.010866 + }, + { + "epoch": 0.007544797233574348, + "grad_norm": 7.852550983428955, + "learning_rate": 1e-05, + "logits/chosen": -0.48881009221076965, + "logits/rejected": -0.6540625691413879, + "logps/chosen": -4.714770793914795, + "logps/rejected": -17.99374008178711, + "loss": 1.0851404666900635, + "memory(GiB)": 33.9, + "nll_loss": 0.3894187808036804, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.002094469266012311, + "rewards/margins": -0.004971538204699755, + "rewards/rejected": 0.0028770684730261564, + "step": 3, + "train_speed(iter/s)": 0.011002 + }, + { + "epoch": 0.010059729644765796, + "grad_norm": 10.150762557983398, + "learning_rate": 1.3333333333333333e-05, + "logits/chosen": -0.4506370425224304, + "logits/rejected": -0.6422093510627747, + "logps/chosen": -3.6849846839904785, + "logps/rejected": -21.201997756958008, + "loss": 1.092491865158081, + "memory(GiB)": 37.36, + "nll_loss": 0.4020901322364807, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.007022961974143982, + "rewards/margins": 0.0057290042750537395, + "rewards/rejected": 0.001293957349844277, + "step": 4, + "train_speed(iter/s)": 0.011051 + }, + { + "epoch": 0.012574662055957246, + "grad_norm": 6.588510036468506, + "learning_rate": 1.6666666666666667e-05, + "logits/chosen": -0.4159216284751892, + "logits/rejected": -0.5884239673614502, + "logps/chosen": -8.611150741577148, + "logps/rejected": -19.931533813476562, + "loss": 1.1986628770828247, + "memory(GiB)": 37.36, + "nll_loss": 0.5155895948410034, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.005048489198088646, + "rewards/margins": 0.020845137536525726, + "rewards/rejected": -0.01579664833843708, + "step": 5, + "train_speed(iter/s)": 0.011086 + }, + { + "epoch": 0.015089594467148696, + "grad_norm": 3.965754508972168, + "learning_rate": 2e-05, + "logits/chosen": -0.47118157148361206, + "logits/rejected": -0.5790206789970398, + "logps/chosen": -9.572708129882812, + "logps/rejected": -19.066829681396484, + "loss": 1.1965497732162476, + "memory(GiB)": 37.36, + "nll_loss": 0.5279884338378906, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.006116065196692944, + "rewards/margins": 0.05174415558576584, + "rewards/rejected": -0.04562808945775032, + "step": 6, + "train_speed(iter/s)": 0.01111 + }, + { + "epoch": 0.017604526878340146, + "grad_norm": 2.719625234603882, + "learning_rate": 2.3333333333333336e-05, + "logits/chosen": -0.40740644931793213, + "logits/rejected": -0.5419386029243469, + "logps/chosen": -9.413152694702148, + "logps/rejected": -24.703401565551758, + "loss": 1.1549676656723022, + "memory(GiB)": 40.85, + "nll_loss": 0.5149396061897278, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.02250390499830246, + "rewards/margins": 0.12123934924602509, + "rewards/rejected": -0.14374326169490814, + "step": 7, + "train_speed(iter/s)": 0.011114 + }, + { + "epoch": 0.020119459289531592, + "grad_norm": 3.325852394104004, + "learning_rate": 2.6666666666666667e-05, + "logits/chosen": -0.4091758728027344, + "logits/rejected": -0.5692812204360962, + "logps/chosen": -5.904629230499268, + "logps/rejected": -20.752662658691406, + "loss": 1.0213468074798584, + "memory(GiB)": 40.85, + "nll_loss": 0.3945533037185669, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.041384391486644745, + "rewards/margins": 0.17802417278289795, + "rewards/rejected": -0.2194085717201233, + "step": 8, + "train_speed(iter/s)": 0.011127 + }, + { + "epoch": 0.022634391700723042, + "grad_norm": 3.5531411170959473, + "learning_rate": 3e-05, + "logits/chosen": -0.3281947374343872, + "logits/rejected": -0.5156837701797485, + "logps/chosen": -5.975313186645508, + "logps/rejected": -27.066158294677734, + "loss": 0.8695712089538574, + "memory(GiB)": 40.85, + "nll_loss": 0.3611774742603302, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0457351878285408, + "rewards/margins": 0.4895857870578766, + "rewards/rejected": -0.443850576877594, + "step": 9, + "train_speed(iter/s)": 0.011136 + }, + { + "epoch": 0.025149324111914492, + "grad_norm": 4.356088161468506, + "learning_rate": 3.3333333333333335e-05, + "logits/chosen": -0.43971675634384155, + "logits/rejected": -0.5702978372573853, + "logps/chosen": -6.376318454742432, + "logps/rejected": -19.42030143737793, + "loss": 1.0046417713165283, + "memory(GiB)": 40.85, + "nll_loss": 0.42133060097694397, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.17984721064567566, + "rewards/margins": 0.43266722559928894, + "rewards/rejected": -0.6125144362449646, + "step": 10, + "train_speed(iter/s)": 0.01115 + }, + { + "epoch": 0.027664256523105942, + "grad_norm": 3.9732213020324707, + "learning_rate": 3.6666666666666666e-05, + "logits/chosen": -0.4561832845211029, + "logits/rejected": -0.6065505146980286, + "logps/chosen": -6.498957633972168, + "logps/rejected": -27.428552627563477, + "loss": 0.9565106630325317, + "memory(GiB)": 40.85, + "nll_loss": 0.40273764729499817, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12391924858093262, + "rewards/margins": 0.5706068277359009, + "rewards/rejected": -0.6945260763168335, + "step": 11, + "train_speed(iter/s)": 0.011161 + }, + { + "epoch": 0.030179188934297392, + "grad_norm": 2.2540578842163086, + "learning_rate": 4e-05, + "logits/chosen": -0.42088037729263306, + "logits/rejected": -0.6009025573730469, + "logps/chosen": -4.243446350097656, + "logps/rejected": -26.567686080932617, + "loss": 0.7599425911903381, + "memory(GiB)": 40.85, + "nll_loss": 0.28007903695106506, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.14110325276851654, + "rewards/margins": 0.6590132713317871, + "rewards/rejected": -0.8001165390014648, + "step": 12, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.03269412134548884, + "grad_norm": 6.460833549499512, + "learning_rate": 4.3333333333333334e-05, + "logits/chosen": -0.4136754274368286, + "logits/rejected": -0.5452494621276855, + "logps/chosen": -7.585506439208984, + "logps/rejected": -21.19340705871582, + "loss": 1.223608136177063, + "memory(GiB)": 40.85, + "nll_loss": 0.6720148921012878, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.25483113527297974, + "rewards/margins": 0.49035438895225525, + "rewards/rejected": -0.7451854944229126, + "step": 13, + "train_speed(iter/s)": 0.011181 + }, + { + "epoch": 0.03520905375668029, + "grad_norm": 4.092947006225586, + "learning_rate": 4.666666666666667e-05, + "logits/chosen": -0.527854323387146, + "logits/rejected": -0.5994939804077148, + "logps/chosen": -8.185604095458984, + "logps/rejected": -17.946279525756836, + "loss": 1.2557107210159302, + "memory(GiB)": 40.85, + "nll_loss": 0.66472989320755, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.145107239484787, + "rewards/margins": 0.41058170795440674, + "rewards/rejected": -0.5556889772415161, + "step": 14, + "train_speed(iter/s)": 0.011194 + }, + { + "epoch": 0.03772398616787174, + "grad_norm": 3.926342248916626, + "learning_rate": 5e-05, + "logits/chosen": -0.407508909702301, + "logits/rejected": -0.5949459075927734, + "logps/chosen": -3.4148104190826416, + "logps/rejected": -22.69104766845703, + "loss": 0.8548452258110046, + "memory(GiB)": 40.85, + "nll_loss": 0.40506720542907715, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.025914989411830902, + "rewards/margins": 0.6952927708625793, + "rewards/rejected": -0.7212077379226685, + "step": 15, + "train_speed(iter/s)": 0.0112 + }, + { + "epoch": 0.040238918579063185, + "grad_norm": 2.4273853302001953, + "learning_rate": 5.333333333333333e-05, + "logits/chosen": -0.3462049961090088, + "logits/rejected": -0.58342045545578, + "logps/chosen": -3.8651039600372314, + "logps/rejected": -28.470134735107422, + "loss": 0.821665346622467, + "memory(GiB)": 40.85, + "nll_loss": 0.3309793472290039, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.04178892821073532, + "rewards/margins": 0.5164157152175903, + "rewards/rejected": -0.4746267795562744, + "step": 16, + "train_speed(iter/s)": 0.011202 + }, + { + "epoch": 0.04275385099025464, + "grad_norm": 1.8921568393707275, + "learning_rate": 5.666666666666667e-05, + "logits/chosen": -0.5054872632026672, + "logits/rejected": -0.6153637170791626, + "logps/chosen": -1.8910857439041138, + "logps/rejected": -16.781475067138672, + "loss": 0.7665270566940308, + "memory(GiB)": 40.85, + "nll_loss": 0.2412014603614807, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04062865674495697, + "rewards/margins": 0.41468262672424316, + "rewards/rejected": -0.3740540146827698, + "step": 17, + "train_speed(iter/s)": 0.011212 + }, + { + "epoch": 0.045268783401446085, + "grad_norm": 3.2724339962005615, + "learning_rate": 6e-05, + "logits/chosen": -0.45994284749031067, + "logits/rejected": -0.5327339768409729, + "logps/chosen": -8.705971717834473, + "logps/rejected": -17.33376121520996, + "loss": 1.2050995826721191, + "memory(GiB)": 40.85, + "nll_loss": 0.5643416047096252, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.048418305814266205, + "rewards/margins": 0.16582812368869781, + "rewards/rejected": -0.21424642205238342, + "step": 18, + "train_speed(iter/s)": 0.011215 + }, + { + "epoch": 0.04778371581263754, + "grad_norm": 2.6650567054748535, + "learning_rate": 6.333333333333333e-05, + "logits/chosen": -0.48406994342803955, + "logits/rejected": -0.5535129308700562, + "logps/chosen": -10.087296485900879, + "logps/rejected": -17.584102630615234, + "loss": 1.1602699756622314, + "memory(GiB)": 40.85, + "nll_loss": 0.5509804487228394, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.005405411124229431, + "rewards/margins": 0.23162491619586945, + "rewards/rejected": -0.23703034222126007, + "step": 19, + "train_speed(iter/s)": 0.01122 + }, + { + "epoch": 0.050298648223828984, + "grad_norm": 2.393120527267456, + "learning_rate": 6.666666666666667e-05, + "logits/chosen": -0.4699333906173706, + "logits/rejected": -0.5745525360107422, + "logps/chosen": -5.482749938964844, + "logps/rejected": -15.294036865234375, + "loss": 1.0463979244232178, + "memory(GiB)": 40.85, + "nll_loss": 0.40886539220809937, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.030414806678891182, + "rewards/margins": 0.18470264971256256, + "rewards/rejected": -0.15428784489631653, + "step": 20, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.05281358063502043, + "grad_norm": 1.7328065633773804, + "learning_rate": 7e-05, + "logits/chosen": -0.5130109190940857, + "logits/rejected": -0.6171243190765381, + "logps/chosen": -8.619268417358398, + "logps/rejected": -18.43971824645996, + "loss": 1.1801689863204956, + "memory(GiB)": 40.85, + "nll_loss": 0.6120539903640747, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.07324957102537155, + "rewards/margins": 0.3033149242401123, + "rewards/rejected": -0.23006536066532135, + "step": 21, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.055328513046211884, + "grad_norm": 2.848578691482544, + "learning_rate": 7.333333333333333e-05, + "logits/chosen": -0.46683233976364136, + "logits/rejected": -0.5329924821853638, + "logps/chosen": -4.357663154602051, + "logps/rejected": -15.35399055480957, + "loss": 0.9406945705413818, + "memory(GiB)": 40.85, + "nll_loss": 0.3216635584831238, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.010179778560996056, + "rewards/margins": 0.2008945494890213, + "rewards/rejected": -0.21107429265975952, + "step": 22, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.05784344545740333, + "grad_norm": 1.7426271438598633, + "learning_rate": 7.666666666666667e-05, + "logits/chosen": -0.5002316236495972, + "logits/rejected": -0.5488481521606445, + "logps/chosen": -5.690494537353516, + "logps/rejected": -15.590461730957031, + "loss": 0.9775924682617188, + "memory(GiB)": 40.85, + "nll_loss": 0.355554461479187, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.009325886145234108, + "rewards/margins": 0.18430516123771667, + "rewards/rejected": -0.1749793142080307, + "step": 23, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.060358377868594784, + "grad_norm": 1.8518409729003906, + "learning_rate": 8e-05, + "logits/chosen": -0.5258264541625977, + "logits/rejected": -0.5805689096450806, + "logps/chosen": -9.133979797363281, + "logps/rejected": -17.886051177978516, + "loss": 1.1810051202774048, + "memory(GiB)": 40.85, + "nll_loss": 0.6029603481292725, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.04269062727689743, + "rewards/margins": 0.2787190079689026, + "rewards/rejected": -0.23602835834026337, + "step": 24, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.06287331027978624, + "grad_norm": 2.4842066764831543, + "learning_rate": 8.333333333333334e-05, + "logits/chosen": -0.44220152497291565, + "logits/rejected": -0.5788168907165527, + "logps/chosen": -4.989194869995117, + "logps/rejected": -18.047056198120117, + "loss": 1.0011777877807617, + "memory(GiB)": 40.85, + "nll_loss": 0.4066605269908905, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.021916719153523445, + "rewards/margins": 0.2587287425994873, + "rewards/rejected": -0.236812025308609, + "step": 25, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.06538824269097768, + "grad_norm": 1.766322135925293, + "learning_rate": 8.666666666666667e-05, + "logits/chosen": -0.3389984369277954, + "logits/rejected": -0.543292224407196, + "logps/chosen": -11.735610961914062, + "logps/rejected": -26.25033950805664, + "loss": 1.0063844919204712, + "memory(GiB)": 40.85, + "nll_loss": 0.5172990560531616, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.07315510511398315, + "rewards/margins": 0.5734083652496338, + "rewards/rejected": -0.5002533197402954, + "step": 26, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.06790317510216913, + "grad_norm": 3.3314130306243896, + "learning_rate": 9e-05, + "logits/chosen": -0.37741950154304504, + "logits/rejected": -0.5884826183319092, + "logps/chosen": -5.442512035369873, + "logps/rejected": -27.738323211669922, + "loss": 0.8821365237236023, + "memory(GiB)": 40.85, + "nll_loss": 0.4055330753326416, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.03083166666328907, + "rewards/margins": 0.6056570410728455, + "rewards/rejected": -0.6364887952804565, + "step": 27, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.07041810751336058, + "grad_norm": 2.849647283554077, + "learning_rate": 9.333333333333334e-05, + "logits/chosen": -0.4310716688632965, + "logits/rejected": -0.5529810190200806, + "logps/chosen": -8.224971771240234, + "logps/rejected": -26.79846954345703, + "loss": 0.8747767806053162, + "memory(GiB)": 40.85, + "nll_loss": 0.45313140749931335, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.07478700578212738, + "rewards/margins": 0.8052245378494263, + "rewards/rejected": -0.8800115585327148, + "step": 28, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.07293303992455202, + "grad_norm": 2.247368097305298, + "learning_rate": 9.666666666666667e-05, + "logits/chosen": -0.37296879291534424, + "logits/rejected": -0.5259015560150146, + "logps/chosen": -6.546780109405518, + "logps/rejected": -25.603477478027344, + "loss": 0.9804578423500061, + "memory(GiB)": 40.85, + "nll_loss": 0.4873877465724945, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.046319298446178436, + "rewards/margins": 0.5485984086990356, + "rewards/rejected": -0.5949177145957947, + "step": 29, + "train_speed(iter/s)": 0.011239 + }, + { + "epoch": 0.07544797233574348, + "grad_norm": 1.7451585531234741, + "learning_rate": 0.0001, + "logits/chosen": -0.36992424726486206, + "logits/rejected": -0.45540890097618103, + "logps/chosen": -6.483048439025879, + "logps/rejected": -18.744525909423828, + "loss": 0.9453181624412537, + "memory(GiB)": 40.85, + "nll_loss": 0.37866073846817017, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0520947091281414, + "rewards/margins": 0.33587026596069336, + "rewards/rejected": -0.28377556800842285, + "step": 30, + "train_speed(iter/s)": 0.01124 + }, + { + "epoch": 0.07796290474693493, + "grad_norm": 2.8843863010406494, + "learning_rate": 0.00010333333333333334, + "logits/chosen": -0.3973531723022461, + "logits/rejected": -0.4880366325378418, + "logps/chosen": -6.5046491622924805, + "logps/rejected": -18.793209075927734, + "loss": 1.0352380275726318, + "memory(GiB)": 40.85, + "nll_loss": 0.514582097530365, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.06333249807357788, + "rewards/margins": 0.5268359780311584, + "rewards/rejected": -0.4635034203529358, + "step": 31, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.08047783715812637, + "grad_norm": 1.9934616088867188, + "learning_rate": 0.00010666666666666667, + "logits/chosen": -0.38435813784599304, + "logits/rejected": -0.4866703450679779, + "logps/chosen": -5.984093189239502, + "logps/rejected": -28.37721061706543, + "loss": 0.9228850603103638, + "memory(GiB)": 40.85, + "nll_loss": 0.3919341564178467, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08762034773826599, + "rewards/margins": 0.4990919232368469, + "rewards/rejected": -0.41147157549858093, + "step": 32, + "train_speed(iter/s)": 0.011242 + }, + { + "epoch": 0.08299276956931782, + "grad_norm": 2.605156421661377, + "learning_rate": 0.00011000000000000002, + "logits/chosen": -0.3683161437511444, + "logits/rejected": -0.4800977110862732, + "logps/chosen": -6.79887056350708, + "logps/rejected": -18.521530151367188, + "loss": 0.9287618398666382, + "memory(GiB)": 40.85, + "nll_loss": 0.4155515730381012, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.062238909304142, + "rewards/margins": 0.4814634323120117, + "rewards/rejected": -0.41922450065612793, + "step": 33, + "train_speed(iter/s)": 0.011243 + }, + { + "epoch": 0.08550770198050928, + "grad_norm": 2.1593198776245117, + "learning_rate": 0.00011333333333333334, + "logits/chosen": -0.3462904989719391, + "logits/rejected": -0.4482634663581848, + "logps/chosen": -4.484028339385986, + "logps/rejected": -22.155223846435547, + "loss": 0.8287199139595032, + "memory(GiB)": 40.85, + "nll_loss": 0.3564324378967285, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.011370105668902397, + "rewards/margins": 0.8015316724777222, + "rewards/rejected": -0.8129018545150757, + "step": 34, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.08802263439170073, + "grad_norm": 2.1223862171173096, + "learning_rate": 0.00011666666666666668, + "logits/chosen": -0.23746490478515625, + "logits/rejected": -0.4123544991016388, + "logps/chosen": -4.062009334564209, + "logps/rejected": -30.214771270751953, + "loss": 0.8381334543228149, + "memory(GiB)": 40.85, + "nll_loss": 0.45768678188323975, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.06588438153266907, + "rewards/margins": 1.1650195121765137, + "rewards/rejected": -1.0991352796554565, + "step": 35, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.09053756680289217, + "grad_norm": 1.7519793510437012, + "learning_rate": 0.00012, + "logits/chosen": -0.274143248796463, + "logits/rejected": -0.4072067439556122, + "logps/chosen": -7.12678861618042, + "logps/rejected": -29.586929321289062, + "loss": 0.7818687558174133, + "memory(GiB)": 40.85, + "nll_loss": 0.38336753845214844, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.030900314450263977, + "rewards/margins": 1.0601840019226074, + "rewards/rejected": -1.0292836427688599, + "step": 36, + "train_speed(iter/s)": 0.011244 + }, + { + "epoch": 0.09305249921408362, + "grad_norm": 2.2621781826019287, + "learning_rate": 0.00012333333333333334, + "logits/chosen": -0.30612003803253174, + "logits/rejected": -0.44930776953697205, + "logps/chosen": -5.637537956237793, + "logps/rejected": -30.86564064025879, + "loss": 0.7518173456192017, + "memory(GiB)": 40.85, + "nll_loss": 0.4073091149330139, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1094760149717331, + "rewards/margins": 1.2509604692459106, + "rewards/rejected": -1.3604364395141602, + "step": 37, + "train_speed(iter/s)": 0.011245 + }, + { + "epoch": 0.09556743162527508, + "grad_norm": 2.6005046367645264, + "learning_rate": 0.00012666666666666666, + "logits/chosen": -0.36841830611228943, + "logits/rejected": -0.44501256942749023, + "logps/chosen": -8.786764144897461, + "logps/rejected": -21.693777084350586, + "loss": 0.967311441898346, + "memory(GiB)": 40.85, + "nll_loss": 0.5350124835968018, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.05166015028953552, + "rewards/margins": 0.9184513092041016, + "rewards/rejected": -0.9701113700866699, + "step": 38, + "train_speed(iter/s)": 0.011247 + }, + { + "epoch": 0.09808236403646652, + "grad_norm": 7.679723262786865, + "learning_rate": 0.00013000000000000002, + "logits/chosen": -0.3933795094490051, + "logits/rejected": -0.4303474426269531, + "logps/chosen": -9.630545616149902, + "logps/rejected": -25.0108642578125, + "loss": 1.1940371990203857, + "memory(GiB)": 40.85, + "nll_loss": 0.6422770023345947, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.19723013043403625, + "rewards/margins": 0.8150386214256287, + "rewards/rejected": -1.0122687816619873, + "step": 39, + "train_speed(iter/s)": 0.011247 + }, + { + "epoch": 0.10059729644765797, + "grad_norm": 2.497746229171753, + "learning_rate": 0.00013333333333333334, + "logits/chosen": -0.3561173677444458, + "logits/rejected": -0.4964173436164856, + "logps/chosen": -6.288295269012451, + "logps/rejected": -28.722339630126953, + "loss": 0.7734813690185547, + "memory(GiB)": 40.85, + "nll_loss": 0.35979557037353516, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.020053118467330933, + "rewards/margins": 1.0713183879852295, + "rewards/rejected": -1.0512654781341553, + "step": 40, + "train_speed(iter/s)": 0.011248 + }, + { + "epoch": 0.10311222885884942, + "grad_norm": 3.900786876678467, + "learning_rate": 0.00013666666666666666, + "logits/chosen": -0.30438148975372314, + "logits/rejected": -0.4660162031650543, + "logps/chosen": -9.80126953125, + "logps/rejected": -22.89883804321289, + "loss": 1.0617499351501465, + "memory(GiB)": 40.85, + "nll_loss": 0.5348615646362305, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.037312038242816925, + "rewards/margins": 0.48836010694503784, + "rewards/rejected": -0.4510480761528015, + "step": 41, + "train_speed(iter/s)": 0.011248 + }, + { + "epoch": 0.10562716127004086, + "grad_norm": 4.212996006011963, + "learning_rate": 0.00014, + "logits/chosen": -0.37392380833625793, + "logits/rejected": -0.4811669588088989, + "logps/chosen": -8.54138469696045, + "logps/rejected": -24.88803482055664, + "loss": 0.9449828267097473, + "memory(GiB)": 40.85, + "nll_loss": 0.40984830260276794, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.078499935567379, + "rewards/margins": 0.6149317026138306, + "rewards/rejected": -0.6934316754341125, + "step": 42, + "train_speed(iter/s)": 0.011249 + }, + { + "epoch": 0.10814209368123232, + "grad_norm": 2.7297940254211426, + "learning_rate": 0.00014333333333333334, + "logits/chosen": -0.22235050797462463, + "logits/rejected": -0.44656193256378174, + "logps/chosen": -3.8628368377685547, + "logps/rejected": -29.070890426635742, + "loss": 0.6864454746246338, + "memory(GiB)": 40.85, + "nll_loss": 0.3404179513454437, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10101112723350525, + "rewards/margins": 1.1784940958023071, + "rewards/rejected": -1.0774829387664795, + "step": 43, + "train_speed(iter/s)": 0.011247 + }, + { + "epoch": 0.11065702609242377, + "grad_norm": 4.357929229736328, + "learning_rate": 0.00014666666666666666, + "logits/chosen": -0.3152369558811188, + "logits/rejected": -0.4675106406211853, + "logps/chosen": -8.47473430633545, + "logps/rejected": -35.49666976928711, + "loss": 0.8841890096664429, + "memory(GiB)": 40.85, + "nll_loss": 0.543793261051178, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.2599925100803375, + "rewards/margins": 1.5472975969314575, + "rewards/rejected": -1.8072898387908936, + "step": 44, + "train_speed(iter/s)": 0.011248 + }, + { + "epoch": 0.11317195850361522, + "grad_norm": 4.053813934326172, + "learning_rate": 0.00015000000000000001, + "logits/chosen": -0.47932106256484985, + "logits/rejected": -0.5357992053031921, + "logps/chosen": -10.139131546020508, + "logps/rejected": -29.30261993408203, + "loss": 1.0090879201889038, + "memory(GiB)": 40.85, + "nll_loss": 0.6002873182296753, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.2646394371986389, + "rewards/margins": 1.3857942819595337, + "rewards/rejected": -1.6504336595535278, + "step": 45, + "train_speed(iter/s)": 0.01125 + }, + { + "epoch": 0.11568689091480666, + "grad_norm": 3.4071342945098877, + "learning_rate": 0.00015333333333333334, + "logits/chosen": -0.35965389013290405, + "logits/rejected": -0.4886631965637207, + "logps/chosen": -6.492125988006592, + "logps/rejected": -33.90357971191406, + "loss": 0.9477238655090332, + "memory(GiB)": 40.85, + "nll_loss": 0.5519906878471375, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.13393068313598633, + "rewards/margins": 1.3498430252075195, + "rewards/rejected": -1.4837737083435059, + "step": 46, + "train_speed(iter/s)": 0.011251 + }, + { + "epoch": 0.11820182332599811, + "grad_norm": 1.5676898956298828, + "learning_rate": 0.00015666666666666666, + "logits/chosen": -0.3339039087295532, + "logits/rejected": -0.5405474305152893, + "logps/chosen": -5.18754768371582, + "logps/rejected": -34.61940383911133, + "loss": 0.5877978205680847, + "memory(GiB)": 40.85, + "nll_loss": 0.26233971118927, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.04858784377574921, + "rewards/margins": 1.4838428497314453, + "rewards/rejected": -1.4352549314498901, + "step": 47, + "train_speed(iter/s)": 0.011252 + }, + { + "epoch": 0.12071675573718957, + "grad_norm": 1.7600055932998657, + "learning_rate": 0.00016, + "logits/chosen": -0.38554954528808594, + "logits/rejected": -0.5349220633506775, + "logps/chosen": -5.35024356842041, + "logps/rejected": -30.936561584472656, + "loss": 0.7445188760757446, + "memory(GiB)": 40.85, + "nll_loss": 0.3484676480293274, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.04428477957844734, + "rewards/margins": 1.158116340637207, + "rewards/rejected": -1.2024011611938477, + "step": 48, + "train_speed(iter/s)": 0.011253 + }, + { + "epoch": 0.12323168814838101, + "grad_norm": 1.8286348581314087, + "learning_rate": 0.00016333333333333334, + "logits/chosen": -0.3060781955718994, + "logits/rejected": -0.5333737134933472, + "logps/chosen": -4.790343761444092, + "logps/rejected": -32.540409088134766, + "loss": 0.6686277985572815, + "memory(GiB)": 40.85, + "nll_loss": 0.3360018730163574, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0967080146074295, + "rewards/margins": 1.285821795463562, + "rewards/rejected": -1.1891138553619385, + "step": 49, + "train_speed(iter/s)": 0.011254 + }, + { + "epoch": 0.12574662055957248, + "grad_norm": 1.8983184099197388, + "learning_rate": 0.0001666666666666667, + "logits/chosen": -0.22582519054412842, + "logits/rejected": -0.47035256028175354, + "logps/chosen": -6.48522424697876, + "logps/rejected": -32.73487091064453, + "loss": 0.8076327443122864, + "memory(GiB)": 40.85, + "nll_loss": 0.3926582336425781, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.01478707417845726, + "rewards/margins": 1.167083501815796, + "rewards/rejected": -1.1522964239120483, + "step": 50, + "train_speed(iter/s)": 0.011253 + }, + { + "epoch": 0.12826155297076391, + "grad_norm": 2.4192955493927, + "learning_rate": 0.00017, + "logits/chosen": -0.30057138204574585, + "logits/rejected": -0.4678650498390198, + "logps/chosen": -7.537961483001709, + "logps/rejected": -34.26948547363281, + "loss": 0.8188683390617371, + "memory(GiB)": 40.85, + "nll_loss": 0.4251425266265869, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.051345158368349075, + "rewards/margins": 1.1266952753067017, + "rewards/rejected": -1.178040623664856, + "step": 51, + "train_speed(iter/s)": 0.01121 + }, + { + "epoch": 0.13077648538195535, + "grad_norm": 2.0905075073242188, + "learning_rate": 0.00017333333333333334, + "logits/chosen": -0.2889178395271301, + "logits/rejected": -0.49443909525871277, + "logps/chosen": -6.827200412750244, + "logps/rejected": -44.23398971557617, + "loss": 0.7206944227218628, + "memory(GiB)": 40.85, + "nll_loss": 0.39006102085113525, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.06844954192638397, + "rewards/margins": 2.328122854232788, + "rewards/rejected": -2.3965721130371094, + "step": 52, + "train_speed(iter/s)": 0.01121 + }, + { + "epoch": 0.13329141779314682, + "grad_norm": 1.9464434385299683, + "learning_rate": 0.00017666666666666666, + "logits/chosen": -0.30524271726608276, + "logits/rejected": -0.5259577631950378, + "logps/chosen": -8.820595741271973, + "logps/rejected": -34.23869705200195, + "loss": 0.8924205899238586, + "memory(GiB)": 40.85, + "nll_loss": 0.5174475312232971, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.007829657755792141, + "rewards/margins": 1.6594951152801514, + "rewards/rejected": -1.6673249006271362, + "step": 53, + "train_speed(iter/s)": 0.011211 + }, + { + "epoch": 0.13580635020433826, + "grad_norm": 2.15484356880188, + "learning_rate": 0.00018, + "logits/chosen": -0.36469364166259766, + "logits/rejected": -0.5003997087478638, + "logps/chosen": -9.828875541687012, + "logps/rejected": -35.949886322021484, + "loss": 0.8116447925567627, + "memory(GiB)": 40.85, + "nll_loss": 0.4780670404434204, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04445749148726463, + "rewards/margins": 1.7651525735855103, + "rewards/rejected": -1.7206950187683105, + "step": 54, + "train_speed(iter/s)": 0.011212 + }, + { + "epoch": 0.1383212826155297, + "grad_norm": 5.288140773773193, + "learning_rate": 0.00018333333333333334, + "logits/chosen": -0.357359915971756, + "logits/rejected": -0.5485042333602905, + "logps/chosen": -4.562381744384766, + "logps/rejected": -40.544979095458984, + "loss": 0.8048219680786133, + "memory(GiB)": 40.85, + "nll_loss": 0.4945175051689148, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.026466388255357742, + "rewards/margins": 2.5180537700653076, + "rewards/rejected": -2.544520378112793, + "step": 55, + "train_speed(iter/s)": 0.011214 + }, + { + "epoch": 0.14083621502672117, + "grad_norm": 4.063258647918701, + "learning_rate": 0.0001866666666666667, + "logits/chosen": -0.39062973856925964, + "logits/rejected": -0.5040320158004761, + "logps/chosen": -10.983360290527344, + "logps/rejected": -33.32127380371094, + "loss": 1.1587333679199219, + "memory(GiB)": 40.85, + "nll_loss": 0.7283708453178406, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.13727927207946777, + "rewards/margins": 1.9422543048858643, + "rewards/rejected": -2.079533576965332, + "step": 56, + "train_speed(iter/s)": 0.011215 + }, + { + "epoch": 0.1433511474379126, + "grad_norm": 3.7057104110717773, + "learning_rate": 0.00019, + "logits/chosen": -0.34116944670677185, + "logits/rejected": -0.5513606667518616, + "logps/chosen": -5.59161376953125, + "logps/rejected": -47.29106903076172, + "loss": 0.8299980759620667, + "memory(GiB)": 40.85, + "nll_loss": 0.5494524836540222, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.13214518129825592, + "rewards/margins": 2.937675952911377, + "rewards/rejected": -3.069821357727051, + "step": 57, + "train_speed(iter/s)": 0.011217 + }, + { + "epoch": 0.14586607984910405, + "grad_norm": 4.7582502365112305, + "learning_rate": 0.00019333333333333333, + "logits/chosen": -0.4154261350631714, + "logits/rejected": -0.5971646904945374, + "logps/chosen": -3.059924364089966, + "logps/rejected": -43.29597091674805, + "loss": 0.5003064274787903, + "memory(GiB)": 40.85, + "nll_loss": 0.25123852491378784, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.07654222846031189, + "rewards/margins": 2.7813808917999268, + "rewards/rejected": -2.704838752746582, + "step": 58, + "train_speed(iter/s)": 0.011219 + }, + { + "epoch": 0.14838101226029551, + "grad_norm": 2.1735236644744873, + "learning_rate": 0.00019666666666666666, + "logits/chosen": -0.4227810800075531, + "logits/rejected": -0.47660815715789795, + "logps/chosen": -8.593082427978516, + "logps/rejected": -26.549867630004883, + "loss": 1.0610322952270508, + "memory(GiB)": 40.85, + "nll_loss": 0.5426512360572815, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.10025864094495773, + "rewards/margins": 1.2511481046676636, + "rewards/rejected": -1.3514068126678467, + "step": 59, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.15089594467148695, + "grad_norm": 2.805598258972168, + "learning_rate": 0.0002, + "logits/chosen": -0.300743043422699, + "logits/rejected": -0.47928643226623535, + "logps/chosen": -6.593828201293945, + "logps/rejected": -33.23126983642578, + "loss": 0.7988956570625305, + "memory(GiB)": 40.85, + "nll_loss": 0.40749770402908325, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.0006714202463626862, + "rewards/margins": 1.6283259391784668, + "rewards/rejected": -1.6276545524597168, + "step": 60, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.1534108770826784, + "grad_norm": 2.290149211883545, + "learning_rate": 0.0001999996142159566, + "logits/chosen": -0.3385659158229828, + "logits/rejected": -0.4102441966533661, + "logps/chosen": -8.33851146697998, + "logps/rejected": -29.508546829223633, + "loss": 0.7559148073196411, + "memory(GiB)": 40.85, + "nll_loss": 0.35449326038360596, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0228387713432312, + "rewards/margins": 1.5235671997070312, + "rewards/rejected": -1.5464060306549072, + "step": 61, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.15592580949386986, + "grad_norm": 1.7177425622940063, + "learning_rate": 0.0001999984568668029, + "logits/chosen": -0.32937324047088623, + "logits/rejected": -0.4484848380088806, + "logps/chosen": -3.493271827697754, + "logps/rejected": -34.78036880493164, + "loss": 0.6279686093330383, + "memory(GiB)": 40.85, + "nll_loss": 0.2725304365158081, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.11392105370759964, + "rewards/margins": 1.8176403045654297, + "rewards/rejected": -1.703719139099121, + "step": 62, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.1584407419050613, + "grad_norm": 1.8914873600006104, + "learning_rate": 0.00019999652796146876, + "logits/chosen": -0.29077184200286865, + "logits/rejected": -0.48235204815864563, + "logps/chosen": -7.319096565246582, + "logps/rejected": -39.54444885253906, + "loss": 0.7298395037651062, + "memory(GiB)": 40.85, + "nll_loss": 0.40350908041000366, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0820951834321022, + "rewards/margins": 1.882514476776123, + "rewards/rejected": -1.8004192113876343, + "step": 63, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.16095567431625274, + "grad_norm": 1.429303526878357, + "learning_rate": 0.0001999938275148369, + "logits/chosen": -0.3300917446613312, + "logits/rejected": -0.45771050453186035, + "logps/chosen": -3.8289737701416016, + "logps/rejected": -39.278968811035156, + "loss": 0.39868107438087463, + "memory(GiB)": 40.85, + "nll_loss": 0.18221822381019592, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.07021164149045944, + "rewards/margins": 2.249926805496216, + "rewards/rejected": -2.320138454437256, + "step": 64, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.1634706067274442, + "grad_norm": 2.639005661010742, + "learning_rate": 0.00019999035554774314, + "logits/chosen": -0.36244961619377136, + "logits/rejected": -0.5052404403686523, + "logps/chosen": -5.230596542358398, + "logps/rejected": -39.43718338012695, + "loss": 0.7721049189567566, + "memory(GiB)": 40.85, + "nll_loss": 0.4854365885257721, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.1088234931230545, + "rewards/margins": 2.1535189151763916, + "rewards/rejected": -2.2623424530029297, + "step": 65, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.16598553913863565, + "grad_norm": 2.576199769973755, + "learning_rate": 0.00019998611208697607, + "logits/chosen": -0.3720204830169678, + "logits/rejected": -0.507696270942688, + "logps/chosen": -4.436941146850586, + "logps/rejected": -34.991371154785156, + "loss": 0.7472538352012634, + "memory(GiB)": 40.85, + "nll_loss": 0.39483439922332764, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0759558230638504, + "rewards/margins": 1.7400535345077515, + "rewards/rejected": -1.816009521484375, + "step": 66, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.16850047154982709, + "grad_norm": 1.724285364151001, + "learning_rate": 0.00019998109716527686, + "logits/chosen": -0.25216248631477356, + "logits/rejected": -0.4494931101799011, + "logps/chosen": -3.201148748397827, + "logps/rejected": -39.95793533325195, + "loss": 0.4833192229270935, + "memory(GiB)": 40.85, + "nll_loss": 0.20408841967582703, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06092430651187897, + "rewards/margins": 2.2583165168762207, + "rewards/rejected": -2.197392225265503, + "step": 67, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.17101540396101855, + "grad_norm": 2.91644287109375, + "learning_rate": 0.00019997531082133904, + "logits/chosen": -0.30811840295791626, + "logits/rejected": -0.413838654756546, + "logps/chosen": -7.5586838722229, + "logps/rejected": -29.464954376220703, + "loss": 1.0136064291000366, + "memory(GiB)": 40.85, + "nll_loss": 0.5009399652481079, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.0595913827419281, + "rewards/margins": 1.2450876235961914, + "rewards/rejected": -1.304679036140442, + "step": 68, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.17353033637221, + "grad_norm": 1.6453865766525269, + "learning_rate": 0.00019996875309980826, + "logits/chosen": -0.32626980543136597, + "logits/rejected": -0.47710081934928894, + "logps/chosen": -4.140695095062256, + "logps/rejected": -33.830894470214844, + "loss": 0.6451548933982849, + "memory(GiB)": 40.85, + "nll_loss": 0.3717566728591919, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.094700388610363, + "rewards/margins": 1.8348017930984497, + "rewards/rejected": -1.7401013374328613, + "step": 69, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.17604526878340146, + "grad_norm": 2.180673360824585, + "learning_rate": 0.0001999614240512817, + "logits/chosen": -0.2670721411705017, + "logits/rejected": -0.46321332454681396, + "logps/chosen": -7.102023601531982, + "logps/rejected": -27.861366271972656, + "loss": 0.7481832504272461, + "memory(GiB)": 40.85, + "nll_loss": 0.37576019763946533, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.16741110384464264, + "rewards/margins": 1.3596775531768799, + "rewards/rejected": -1.1922664642333984, + "step": 70, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.1785602011945929, + "grad_norm": 2.049133539199829, + "learning_rate": 0.00019995332373230808, + "logits/chosen": -0.241715669631958, + "logits/rejected": -0.407115638256073, + "logps/chosen": -4.861550331115723, + "logps/rejected": -34.9086799621582, + "loss": 0.6162200570106506, + "memory(GiB)": 40.85, + "nll_loss": 0.25812438130378723, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.037743180990219116, + "rewards/margins": 1.569511890411377, + "rewards/rejected": -1.5317686796188354, + "step": 71, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.18107513360578434, + "grad_norm": 4.052990913391113, + "learning_rate": 0.00019994445220538677, + "logits/chosen": -0.2541053295135498, + "logits/rejected": -0.36276954412460327, + "logps/chosen": -7.95726203918457, + "logps/rejected": -36.47919845581055, + "loss": 0.9209573268890381, + "memory(GiB)": 40.85, + "nll_loss": 0.5536596179008484, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.17974892258644104, + "rewards/margins": 1.929343581199646, + "rewards/rejected": -2.1090924739837646, + "step": 72, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.1835900660169758, + "grad_norm": 3.210548162460327, + "learning_rate": 0.0001999348095389677, + "logits/chosen": -0.24167482554912567, + "logits/rejected": -0.45636847615242004, + "logps/chosen": -7.025866508483887, + "logps/rejected": -48.629905700683594, + "loss": 0.7358823418617249, + "memory(GiB)": 40.85, + "nll_loss": 0.4621264636516571, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.05025668814778328, + "rewards/margins": 2.2894766330718994, + "rewards/rejected": -2.339733123779297, + "step": 73, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.18610499842816725, + "grad_norm": 4.565255165100098, + "learning_rate": 0.0001999243958074506, + "logits/chosen": -0.21860051155090332, + "logits/rejected": -0.42088112235069275, + "logps/chosen": -13.931788444519043, + "logps/rejected": -42.80027770996094, + "loss": 1.2019851207733154, + "memory(GiB)": 40.85, + "nll_loss": 0.7477802038192749, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.36434897780418396, + "rewards/margins": 1.4156924486160278, + "rewards/rejected": -1.7800413370132446, + "step": 74, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.18861993083935868, + "grad_norm": 2.291332244873047, + "learning_rate": 0.00019991321109118448, + "logits/chosen": -0.39398425817489624, + "logits/rejected": -0.44725537300109863, + "logps/chosen": -6.393754482269287, + "logps/rejected": -27.47494888305664, + "loss": 0.7488479018211365, + "memory(GiB)": 40.85, + "nll_loss": 0.3586839437484741, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.012238548137247562, + "rewards/margins": 1.2588202953338623, + "rewards/rejected": -1.246581792831421, + "step": 75, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.19113486325055015, + "grad_norm": 1.830254077911377, + "learning_rate": 0.00019990125547646704, + "logits/chosen": -0.31489449739456177, + "logits/rejected": -0.47091639041900635, + "logps/chosen": -2.980022668838501, + "logps/rejected": -29.753929138183594, + "loss": 0.6747189164161682, + "memory(GiB)": 40.85, + "nll_loss": 0.2754742503166199, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.03740588575601578, + "rewards/margins": 1.4201977252960205, + "rewards/rejected": -1.3827917575836182, + "step": 76, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.1936497956617416, + "grad_norm": 1.903449296951294, + "learning_rate": 0.00019988852905554404, + "logits/chosen": -0.2765476703643799, + "logits/rejected": -0.4408729076385498, + "logps/chosen": -5.630273342132568, + "logps/rejected": -31.533220291137695, + "loss": 0.7030317783355713, + "memory(GiB)": 40.85, + "nll_loss": 0.3307255208492279, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1330985128879547, + "rewards/margins": 1.2641156911849976, + "rewards/rejected": -1.1310172080993652, + "step": 77, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.19616472807293303, + "grad_norm": 1.736433744430542, + "learning_rate": 0.0001998750319266084, + "logits/chosen": -0.3374086618423462, + "logits/rejected": -0.48207730054855347, + "logps/chosen": -5.3590593338012695, + "logps/rejected": -29.23048973083496, + "loss": 0.8495813608169556, + "memory(GiB)": 40.85, + "nll_loss": 0.37136876583099365, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.09036409854888916, + "rewards/margins": 1.2102724313735962, + "rewards/rejected": -1.119908332824707, + "step": 78, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.1986796604841245, + "grad_norm": 2.0839672088623047, + "learning_rate": 0.00019986076419379974, + "logits/chosen": -0.26226145029067993, + "logits/rejected": -0.4474099576473236, + "logps/chosen": -7.3566060066223145, + "logps/rejected": -40.514286041259766, + "loss": 0.9187750220298767, + "memory(GiB)": 40.85, + "nll_loss": 0.60602867603302, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09458937495946884, + "rewards/margins": 1.9139156341552734, + "rewards/rejected": -1.8193262815475464, + "step": 79, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.20119459289531594, + "grad_norm": 2.656200885772705, + "learning_rate": 0.00019984572596720324, + "logits/chosen": -0.4218776226043701, + "logits/rejected": -0.548494279384613, + "logps/chosen": -5.5665717124938965, + "logps/rejected": -34.339046478271484, + "loss": 0.7904801964759827, + "memory(GiB)": 40.85, + "nll_loss": 0.4805757403373718, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.07708686590194702, + "rewards/margins": 1.846845269203186, + "rewards/rejected": -1.9239320755004883, + "step": 80, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.20370952530650738, + "grad_norm": 1.582141399383545, + "learning_rate": 0.00019982991736284915, + "logits/chosen": -0.384469211101532, + "logits/rejected": -0.48424291610717773, + "logps/chosen": -6.730848789215088, + "logps/rejected": -35.332828521728516, + "loss": 0.6736842393875122, + "memory(GiB)": 40.85, + "nll_loss": 0.38718143105506897, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.02753191813826561, + "rewards/margins": 2.01827073097229, + "rewards/rejected": -1.9907389879226685, + "step": 81, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.20622445771769884, + "grad_norm": 10.319942474365234, + "learning_rate": 0.00019981333850271158, + "logits/chosen": -0.3957505226135254, + "logits/rejected": -0.5100387334823608, + "logps/chosen": -7.3371148109436035, + "logps/rejected": -40.714134216308594, + "loss": 0.7473828196525574, + "memory(GiB)": 40.85, + "nll_loss": 0.4370650351047516, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.05588718503713608, + "rewards/margins": 2.654069423675537, + "rewards/rejected": -2.709956645965576, + "step": 82, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.20873939012889028, + "grad_norm": 2.3877530097961426, + "learning_rate": 0.0001997959895147077, + "logits/chosen": -0.36577096581459045, + "logits/rejected": -0.5359358787536621, + "logps/chosen": -4.45527458190918, + "logps/rejected": -46.861942291259766, + "loss": 0.5462505221366882, + "memory(GiB)": 40.85, + "nll_loss": 0.30273228883743286, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.15123440325260162, + "rewards/margins": 2.9262986183166504, + "rewards/rejected": -2.77506422996521, + "step": 83, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.21125432254008172, + "grad_norm": 2.1806390285491943, + "learning_rate": 0.0001997778705326968, + "logits/chosen": -0.3485352396965027, + "logits/rejected": -0.5878627300262451, + "logps/chosen": -2.89451265335083, + "logps/rejected": -44.27880859375, + "loss": 0.6081856489181519, + "memory(GiB)": 40.85, + "nll_loss": 0.3298896849155426, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.05770353972911835, + "rewards/margins": 2.5825228691101074, + "rewards/rejected": -2.5248193740844727, + "step": 84, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.2137692549512732, + "grad_norm": 2.80009388923645, + "learning_rate": 0.00019975898169647915, + "logits/chosen": -0.39257609844207764, + "logits/rejected": -0.5535434484481812, + "logps/chosen": -7.839548587799072, + "logps/rejected": -42.132286071777344, + "loss": 0.7175056338310242, + "memory(GiB)": 40.85, + "nll_loss": 0.405160129070282, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.025028400123119354, + "rewards/margins": 2.475644111633301, + "rewards/rejected": -2.450615644454956, + "step": 85, + "train_speed(iter/s)": 0.011235 + }, + { + "epoch": 0.21628418736246463, + "grad_norm": 3.3921735286712646, + "learning_rate": 0.000199739323151795, + "logits/chosen": -0.35620519518852234, + "logits/rejected": -0.6006304025650024, + "logps/chosen": -5.945646286010742, + "logps/rejected": -45.728271484375, + "loss": 0.6699403524398804, + "memory(GiB)": 40.85, + "nll_loss": 0.33112284541130066, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.012618173845112324, + "rewards/margins": 2.700666666030884, + "rewards/rejected": -2.713285207748413, + "step": 86, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.21879911977365607, + "grad_norm": 3.2411773204803467, + "learning_rate": 0.00019971889505032334, + "logits/chosen": -0.42666560411453247, + "logits/rejected": -0.6167599558830261, + "logps/chosen": -5.318458080291748, + "logps/rejected": -49.244285583496094, + "loss": 0.6356828808784485, + "memory(GiB)": 40.85, + "nll_loss": 0.39307594299316406, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1333954930305481, + "rewards/margins": 3.1458327770233154, + "rewards/rejected": -3.2792282104492188, + "step": 87, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.22131405218484754, + "grad_norm": 1.924742341041565, + "learning_rate": 0.00019969769754968098, + "logits/chosen": -0.3413890600204468, + "logits/rejected": -0.5464233160018921, + "logps/chosen": -5.729033946990967, + "logps/rejected": -41.974422454833984, + "loss": 0.5986770987510681, + "memory(GiB)": 40.85, + "nll_loss": 0.3535672724246979, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.14028939604759216, + "rewards/margins": 2.51139235496521, + "rewards/rejected": -2.371102809906006, + "step": 88, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.22382898459603898, + "grad_norm": 2.161083459854126, + "learning_rate": 0.00019967573081342103, + "logits/chosen": -0.38623473048210144, + "logits/rejected": -0.5679455399513245, + "logps/chosen": -6.484611511230469, + "logps/rejected": -38.930511474609375, + "loss": 0.7250155806541443, + "memory(GiB)": 40.85, + "nll_loss": 0.4377710521221161, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.024773895740509033, + "rewards/margins": 2.3375205993652344, + "rewards/rejected": -2.3622944355010986, + "step": 89, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.22634391700723044, + "grad_norm": 6.164645195007324, + "learning_rate": 0.00019965299501103177, + "logits/chosen": -0.33336132764816284, + "logits/rejected": -0.5168649554252625, + "logps/chosen": -11.845653533935547, + "logps/rejected": -55.690311431884766, + "loss": 1.0204803943634033, + "memory(GiB)": 40.85, + "nll_loss": 0.7011927962303162, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.22887784242630005, + "rewards/margins": 3.111534357070923, + "rewards/rejected": -3.3404123783111572, + "step": 90, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.22885884941842188, + "grad_norm": 2.4981861114501953, + "learning_rate": 0.00019962949031793542, + "logits/chosen": -0.4224125146865845, + "logits/rejected": -0.5427595376968384, + "logps/chosen": -5.719327926635742, + "logps/rejected": -33.303741455078125, + "loss": 0.7469587326049805, + "memory(GiB)": 40.85, + "nll_loss": 0.4206176996231079, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.008410206064581871, + "rewards/margins": 1.994816541671753, + "rewards/rejected": -2.0032269954681396, + "step": 91, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.23137378182961332, + "grad_norm": 2.5139830112457275, + "learning_rate": 0.00019960521691548674, + "logits/chosen": -0.34622836112976074, + "logits/rejected": -0.5427547097206116, + "logps/chosen": -5.069763660430908, + "logps/rejected": -41.81755828857422, + "loss": 0.6454588770866394, + "memory(GiB)": 40.85, + "nll_loss": 0.35349082946777344, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.046083107590675354, + "rewards/margins": 2.2620620727539062, + "rewards/rejected": -2.30814528465271, + "step": 92, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.2338887142408048, + "grad_norm": 2.1890244483947754, + "learning_rate": 0.0001995801749909715, + "logits/chosen": -0.3081744611263275, + "logits/rejected": -0.47990015149116516, + "logps/chosen": -6.5690107345581055, + "logps/rejected": -34.549991607666016, + "loss": 0.6765730381011963, + "memory(GiB)": 42.61, + "nll_loss": 0.3245474696159363, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.060123831033706665, + "rewards/margins": 1.8496551513671875, + "rewards/rejected": -1.7895313501358032, + "step": 93, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.23640364665199623, + "grad_norm": 1.8172200918197632, + "learning_rate": 0.00019955436473760525, + "logits/chosen": -0.24766620993614197, + "logits/rejected": -0.49844813346862793, + "logps/chosen": -5.177490711212158, + "logps/rejected": -48.42265701293945, + "loss": 0.7104548811912537, + "memory(GiB)": 42.61, + "nll_loss": 0.4418777823448181, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10037174820899963, + "rewards/margins": 2.629190683364868, + "rewards/rejected": -2.5288188457489014, + "step": 94, + "train_speed(iter/s)": 0.011236 + }, + { + "epoch": 0.23891857906318767, + "grad_norm": 2.0303428173065186, + "learning_rate": 0.0001995277863545316, + "logits/chosen": -0.3552500307559967, + "logits/rejected": -0.5088093876838684, + "logps/chosen": -3.972975730895996, + "logps/rejected": -29.582290649414062, + "loss": 0.5607516169548035, + "memory(GiB)": 42.61, + "nll_loss": 0.27718478441238403, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.09949840605258942, + "rewards/margins": 1.6923424005508423, + "rewards/rejected": -1.592844009399414, + "step": 95, + "train_speed(iter/s)": 0.011237 + }, + { + "epoch": 0.24143351147437914, + "grad_norm": 2.3172338008880615, + "learning_rate": 0.00019950044004682092, + "logits/chosen": -0.30597299337387085, + "logits/rejected": -0.4608762860298157, + "logps/chosen": -10.467157363891602, + "logps/rejected": -44.41074752807617, + "loss": 0.9089665412902832, + "memory(GiB)": 42.61, + "nll_loss": 0.5540332794189453, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0023995572701096535, + "rewards/margins": 2.6161789894104004, + "rewards/rejected": -2.6185781955718994, + "step": 96, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.24394844388557058, + "grad_norm": 1.6338391304016113, + "learning_rate": 0.0001994723260254686, + "logits/chosen": -0.20743098855018616, + "logits/rejected": -0.4207964837551117, + "logps/chosen": -4.458022117614746, + "logps/rejected": -48.164268493652344, + "loss": 0.5020785927772522, + "memory(GiB)": 42.61, + "nll_loss": 0.2824496626853943, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15383610129356384, + "rewards/margins": 3.0794906616210938, + "rewards/rejected": -2.925654172897339, + "step": 97, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.24646337629676202, + "grad_norm": 2.5537126064300537, + "learning_rate": 0.0001994434445073934, + "logits/chosen": -0.1935003101825714, + "logits/rejected": -0.4715961217880249, + "logps/chosen": -3.131989002227783, + "logps/rejected": -61.07079315185547, + "loss": 0.3960198163986206, + "memory(GiB)": 42.61, + "nll_loss": 0.21717773377895355, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.07503610849380493, + "rewards/margins": 4.007692337036133, + "rewards/rejected": -3.9326562881469727, + "step": 98, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.24897830870795348, + "grad_norm": 7.057893753051758, + "learning_rate": 0.00019941379571543596, + "logits/chosen": -0.21255502104759216, + "logits/rejected": -0.4117867946624756, + "logps/chosen": -5.764989852905273, + "logps/rejected": -59.11027908325195, + "loss": 0.7163435816764832, + "memory(GiB)": 42.61, + "nll_loss": 0.4923526346683502, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.1150842159986496, + "rewards/margins": 3.945159912109375, + "rewards/rejected": -4.060243606567383, + "step": 99, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.25149324111914495, + "grad_norm": 6.774058818817139, + "learning_rate": 0.00019938337987835688, + "logits/chosen": -0.2647398114204407, + "logits/rejected": -0.34906458854675293, + "logps/chosen": -10.037410736083984, + "logps/rejected": -37.84819030761719, + "loss": 1.0176275968551636, + "memory(GiB)": 42.61, + "nll_loss": 0.5385105609893799, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.28268206119537354, + "rewards/margins": 2.0650789737701416, + "rewards/rejected": -2.3477611541748047, + "step": 100, + "train_speed(iter/s)": 0.011238 + }, + { + "epoch": 0.2540081735303364, + "grad_norm": 1.4746135473251343, + "learning_rate": 0.00019935219723083502, + "logits/chosen": -0.28873857855796814, + "logits/rejected": -0.4278784990310669, + "logps/chosen": -4.487892150878906, + "logps/rejected": -43.24613952636719, + "loss": 0.5632098317146301, + "memory(GiB)": 42.61, + "nll_loss": 0.31394267082214355, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.09316689521074295, + "rewards/margins": 3.0039639472961426, + "rewards/rejected": -2.910796880722046, + "step": 101, + "train_speed(iter/s)": 0.011219 + }, + { + "epoch": 0.25652310594152783, + "grad_norm": 2.2414140701293945, + "learning_rate": 0.0001993202480134658, + "logits/chosen": -0.286653995513916, + "logits/rejected": -0.4876902103424072, + "logps/chosen": -2.5823984146118164, + "logps/rejected": -50.10431671142578, + "loss": 0.3667687475681305, + "memory(GiB)": 42.61, + "nll_loss": 0.19701245427131653, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11824626475572586, + "rewards/margins": 3.1774046421051025, + "rewards/rejected": -3.0591585636138916, + "step": 102, + "train_speed(iter/s)": 0.011219 + }, + { + "epoch": 0.25903803835271927, + "grad_norm": 2.5298242568969727, + "learning_rate": 0.0001992875324727591, + "logits/chosen": -0.2699410319328308, + "logits/rejected": -0.421194463968277, + "logps/chosen": -7.033985137939453, + "logps/rejected": -51.89061737060547, + "loss": 0.6059828400611877, + "memory(GiB)": 42.61, + "nll_loss": 0.37160247564315796, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.07941240072250366, + "rewards/margins": 3.3084330558776855, + "rewards/rejected": -3.387845039367676, + "step": 103, + "train_speed(iter/s)": 0.01122 + }, + { + "epoch": 0.2615529707639107, + "grad_norm": 5.99639368057251, + "learning_rate": 0.00019925405086113768, + "logits/chosen": -0.23520317673683167, + "logits/rejected": -0.3863908052444458, + "logps/chosen": -6.1102681159973145, + "logps/rejected": -41.600799560546875, + "loss": 0.8184252381324768, + "memory(GiB)": 42.61, + "nll_loss": 0.5032728910446167, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.09978590905666351, + "rewards/margins": 2.611102819442749, + "rewards/rejected": -2.7108888626098633, + "step": 104, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.26406790317510215, + "grad_norm": 4.219518184661865, + "learning_rate": 0.0001992198034369349, + "logits/chosen": -0.2237715721130371, + "logits/rejected": -0.38729795813560486, + "logps/chosen": -6.913872718811035, + "logps/rejected": -42.98509979248047, + "loss": 0.769071638584137, + "memory(GiB)": 42.61, + "nll_loss": 0.5157548189163208, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.060354869812726974, + "rewards/margins": 2.590625762939453, + "rewards/rejected": -2.530271053314209, + "step": 105, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.26658283558629364, + "grad_norm": 4.175528049468994, + "learning_rate": 0.000199184790464393, + "logits/chosen": -0.2919754683971405, + "logits/rejected": -0.4413781762123108, + "logps/chosen": -4.1272172927856445, + "logps/rejected": -32.008277893066406, + "loss": 0.7491403222084045, + "memory(GiB)": 42.61, + "nll_loss": 0.41593173146247864, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.08565365523099899, + "rewards/margins": 2.0251622200012207, + "rewards/rejected": -1.9395084381103516, + "step": 106, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.2690977679974851, + "grad_norm": 2.857898473739624, + "learning_rate": 0.00019914901221366086, + "logits/chosen": -0.2756834924221039, + "logits/rejected": -0.4335779845714569, + "logps/chosen": -5.15769624710083, + "logps/rejected": -41.91234588623047, + "loss": 0.63164883852005, + "memory(GiB)": 42.61, + "nll_loss": 0.3513546586036682, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06739385426044464, + "rewards/margins": 2.5824873447418213, + "rewards/rejected": -2.5150933265686035, + "step": 107, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.2716127004086765, + "grad_norm": 1.9767286777496338, + "learning_rate": 0.0001991124689607921, + "logits/chosen": -0.23816093802452087, + "logits/rejected": -0.4088671803474426, + "logps/chosen": -6.058350563049316, + "logps/rejected": -36.48659133911133, + "loss": 0.795885443687439, + "memory(GiB)": 42.61, + "nll_loss": 0.4665300250053406, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.02518155239522457, + "rewards/margins": 2.0897223949432373, + "rewards/rejected": -2.0645408630371094, + "step": 108, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.27412763281986796, + "grad_norm": 4.606684684753418, + "learning_rate": 0.00019907516098774275, + "logits/chosen": -0.3413383960723877, + "logits/rejected": -0.4257667064666748, + "logps/chosen": -5.4365692138671875, + "logps/rejected": -36.4229621887207, + "loss": 0.8568770885467529, + "memory(GiB)": 42.61, + "nll_loss": 0.528153121471405, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.014822255820035934, + "rewards/margins": 2.660560369491577, + "rewards/rejected": -2.645737886428833, + "step": 109, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.2766425652310594, + "grad_norm": 2.080709457397461, + "learning_rate": 0.00019903708858236925, + "logits/chosen": -0.22644048929214478, + "logits/rejected": -0.3494427800178528, + "logps/chosen": -6.887048721313477, + "logps/rejected": -39.19084167480469, + "loss": 0.7752804756164551, + "memory(GiB)": 42.61, + "nll_loss": 0.4188353717327118, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.046474143862724304, + "rewards/margins": 2.0502114295959473, + "rewards/rejected": -2.0966856479644775, + "step": 110, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.27915749764225084, + "grad_norm": 2.133538007736206, + "learning_rate": 0.00019899825203842613, + "logits/chosen": -0.24960362911224365, + "logits/rejected": -0.42648327350616455, + "logps/chosen": -4.220504283905029, + "logps/rejected": -38.22932815551758, + "loss": 0.7358196377754211, + "memory(GiB)": 42.61, + "nll_loss": 0.4154231548309326, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.12325219810009003, + "rewards/margins": 2.0720622539520264, + "rewards/rejected": -1.9488099813461304, + "step": 111, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.28167243005344234, + "grad_norm": 1.9261109828948975, + "learning_rate": 0.00019895865165556377, + "logits/chosen": -0.2359025776386261, + "logits/rejected": -0.4521249830722809, + "logps/chosen": -5.312178611755371, + "logps/rejected": -42.527828216552734, + "loss": 0.6443281769752502, + "memory(GiB)": 42.61, + "nll_loss": 0.33275920152664185, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.00696786493062973, + "rewards/margins": 2.2711665630340576, + "rewards/rejected": -2.2781341075897217, + "step": 112, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.2841873624646338, + "grad_norm": 1.973440170288086, + "learning_rate": 0.00019891828773932604, + "logits/chosen": -0.31173762679100037, + "logits/rejected": -0.4782518148422241, + "logps/chosen": -5.571796417236328, + "logps/rejected": -36.36091995239258, + "loss": 0.6021603345870972, + "memory(GiB)": 42.61, + "nll_loss": 0.3728388547897339, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.25308576226234436, + "rewards/margins": 2.2811777591705322, + "rewards/rejected": -2.0280919075012207, + "step": 113, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.2867022948758252, + "grad_norm": 2.1112029552459717, + "learning_rate": 0.0001988771606011481, + "logits/chosen": -0.29321083426475525, + "logits/rejected": -0.4459826946258545, + "logps/chosen": -7.568819046020508, + "logps/rejected": -36.86845016479492, + "loss": 0.8353914022445679, + "memory(GiB)": 42.61, + "nll_loss": 0.557527482509613, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08000842481851578, + "rewards/margins": 2.1105971336364746, + "rewards/rejected": -2.0305888652801514, + "step": 114, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.28921722728701665, + "grad_norm": 1.5973310470581055, + "learning_rate": 0.0001988352705583538, + "logits/chosen": -0.35277947783470154, + "logits/rejected": -0.4771711826324463, + "logps/chosen": -6.40337610244751, + "logps/rejected": -36.56932830810547, + "loss": 0.7110580801963806, + "memory(GiB)": 42.61, + "nll_loss": 0.380901962518692, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.0295229684561491, + "rewards/margins": 2.1401078701019287, + "rewards/rejected": -2.1105847358703613, + "step": 115, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.2917321596982081, + "grad_norm": 2.2071542739868164, + "learning_rate": 0.0001987926179341533, + "logits/chosen": -0.3233400583267212, + "logits/rejected": -0.49479973316192627, + "logps/chosen": -7.279633522033691, + "logps/rejected": -42.67523956298828, + "loss": 0.6601336002349854, + "memory(GiB)": 42.61, + "nll_loss": 0.42307049036026, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.14696629345417023, + "rewards/margins": 2.9207940101623535, + "rewards/rejected": -2.77382755279541, + "step": 116, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.29424709210939953, + "grad_norm": 7.165996551513672, + "learning_rate": 0.00019874920305764068, + "logits/chosen": -0.34290945529937744, + "logits/rejected": -0.5152956247329712, + "logps/chosen": -7.311550140380859, + "logps/rejected": -41.441612243652344, + "loss": 0.9759319424629211, + "memory(GiB)": 42.61, + "nll_loss": 0.6613105535507202, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.17784443497657776, + "rewards/margins": 2.404524803161621, + "rewards/rejected": -2.582369565963745, + "step": 117, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.29676202452059103, + "grad_norm": 2.9480412006378174, + "learning_rate": 0.00019870502626379127, + "logits/chosen": -0.3888930678367615, + "logits/rejected": -0.5601359009742737, + "logps/chosen": -6.752213954925537, + "logps/rejected": -42.02001190185547, + "loss": 0.6632174849510193, + "memory(GiB)": 42.61, + "nll_loss": 0.41683804988861084, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.10449926555156708, + "rewards/margins": 2.6166954040527344, + "rewards/rejected": -2.721194267272949, + "step": 118, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.29927695693178247, + "grad_norm": 8.507670402526855, + "learning_rate": 0.00019866008789345917, + "logits/chosen": -0.40375155210494995, + "logits/rejected": -0.5714547634124756, + "logps/chosen": -6.916952610015869, + "logps/rejected": -41.485107421875, + "loss": 0.8475279211997986, + "memory(GiB)": 42.61, + "nll_loss": 0.5039829015731812, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.07569311559200287, + "rewards/margins": 2.307457685470581, + "rewards/rejected": -2.383150577545166, + "step": 119, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.3017918893429739, + "grad_norm": 1.6022049188613892, + "learning_rate": 0.00019861438829337438, + "logits/chosen": -0.3536994159221649, + "logits/rejected": -0.5457285046577454, + "logps/chosen": -7.962162971496582, + "logps/rejected": -50.184635162353516, + "loss": 0.7923818826675415, + "memory(GiB)": 42.61, + "nll_loss": 0.5494890809059143, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.098821260035038, + "rewards/margins": 2.853030204772949, + "rewards/rejected": -2.75420880317688, + "step": 120, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.30430682175416535, + "grad_norm": 4.437849521636963, + "learning_rate": 0.00019856792781614054, + "logits/chosen": -0.4381689429283142, + "logits/rejected": -0.6431064605712891, + "logps/chosen": -3.382079601287842, + "logps/rejected": -36.976806640625, + "loss": 0.7761183977127075, + "memory(GiB)": 42.61, + "nll_loss": 0.43555858731269836, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.0065191928297281265, + "rewards/margins": 1.8534878492355347, + "rewards/rejected": -1.846968650817871, + "step": 121, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.3068217541653568, + "grad_norm": 2.79468035697937, + "learning_rate": 0.00019852070682023176, + "logits/chosen": -0.5022692680358887, + "logits/rejected": -0.6749197244644165, + "logps/chosen": -6.771264553070068, + "logps/rejected": -38.43341827392578, + "loss": 0.6576388478279114, + "memory(GiB)": 42.61, + "nll_loss": 0.40165674686431885, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21301744878292084, + "rewards/margins": 2.2969624996185303, + "rewards/rejected": -2.0839452743530273, + "step": 122, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.3093366865765483, + "grad_norm": 1.964633822441101, + "learning_rate": 0.00019847272566999026, + "logits/chosen": -0.5233901739120483, + "logits/rejected": -0.6417133212089539, + "logps/chosen": -9.739703178405762, + "logps/rejected": -28.968997955322266, + "loss": 0.8914016485214233, + "memory(GiB)": 42.61, + "nll_loss": 0.5449220538139343, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.16006287932395935, + "rewards/margins": 1.5245118141174316, + "rewards/rejected": -1.3644486665725708, + "step": 123, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.3118516189877397, + "grad_norm": 1.871849775314331, + "learning_rate": 0.0001984239847356233, + "logits/chosen": -0.5555144548416138, + "logits/rejected": -0.6761674284934998, + "logps/chosen": -4.62461519241333, + "logps/rejected": -41.918800354003906, + "loss": 0.5709429383277893, + "memory(GiB)": 42.61, + "nll_loss": 0.3194217383861542, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.10139736533164978, + "rewards/margins": 3.0129401683807373, + "rewards/rejected": -2.9115428924560547, + "step": 124, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.31436655139893116, + "grad_norm": 2.3828136920928955, + "learning_rate": 0.00019837448439320027, + "logits/chosen": -0.5682411789894104, + "logits/rejected": -0.7420387268066406, + "logps/chosen": -3.964232921600342, + "logps/rejected": -61.1876335144043, + "loss": 0.5395267605781555, + "memory(GiB)": 42.61, + "nll_loss": 0.3251190483570099, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.031132016330957413, + "rewards/margins": 4.458404541015625, + "rewards/rejected": -4.427271842956543, + "step": 125, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.3168814838101226, + "grad_norm": 2.152522325515747, + "learning_rate": 0.0001983242250246501, + "logits/chosen": -0.5287045240402222, + "logits/rejected": -0.7190724611282349, + "logps/chosen": -6.970052719116211, + "logps/rejected": -73.82839965820312, + "loss": 0.5324504971504211, + "memory(GiB)": 42.61, + "nll_loss": 0.40117147564888, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.012534398585557938, + "rewards/margins": 5.6087493896484375, + "rewards/rejected": -5.621283054351807, + "step": 126, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.31939641622131404, + "grad_norm": 5.082659721374512, + "learning_rate": 0.00019827320701775806, + "logits/chosen": -0.4820547103881836, + "logits/rejected": -0.6797658205032349, + "logps/chosen": -6.213442802429199, + "logps/rejected": -70.91389465332031, + "loss": 0.6097095012664795, + "memory(GiB)": 42.61, + "nll_loss": 0.4043075144290924, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.03578203544020653, + "rewards/margins": 4.8603901863098145, + "rewards/rejected": -4.896172046661377, + "step": 127, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.3219113486325055, + "grad_norm": 1.7538974285125732, + "learning_rate": 0.0001982214307661627, + "logits/chosen": -0.45252111554145813, + "logits/rejected": -0.6814027428627014, + "logps/chosen": -4.733944416046143, + "logps/rejected": -62.52580261230469, + "loss": 0.5949385166168213, + "memory(GiB)": 42.61, + "nll_loss": 0.3638124167919159, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.1602221429347992, + "rewards/margins": 4.626731872558594, + "rewards/rejected": -4.466509819030762, + "step": 128, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.324426281043697, + "grad_norm": 1.4243698120117188, + "learning_rate": 0.00019816889666935317, + "logits/chosen": -0.3787720203399658, + "logits/rejected": -0.6261472702026367, + "logps/chosen": -6.471246719360352, + "logps/rejected": -71.88873291015625, + "loss": 0.49682942032814026, + "memory(GiB)": 42.61, + "nll_loss": 0.33228519558906555, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13370710611343384, + "rewards/margins": 4.678413391113281, + "rewards/rejected": -4.544705867767334, + "step": 129, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.3269412134548884, + "grad_norm": 2.5607693195343018, + "learning_rate": 0.00019811560513266572, + "logits/chosen": -0.4134800434112549, + "logits/rejected": -0.611393928527832, + "logps/chosen": -5.9260759353637695, + "logps/rejected": -60.01020431518555, + "loss": 0.601435124874115, + "memory(GiB)": 42.61, + "nll_loss": 0.4060709476470947, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08814961463212967, + "rewards/margins": 4.3574066162109375, + "rewards/rejected": -4.269256591796875, + "step": 130, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.32945614586607985, + "grad_norm": 1.9927021265029907, + "learning_rate": 0.00019806155656728084, + "logits/chosen": -0.33065900206565857, + "logits/rejected": -0.6154841184616089, + "logps/chosen": -7.140951156616211, + "logps/rejected": -73.75382232666016, + "loss": 0.7812880277633667, + "memory(GiB)": 42.61, + "nll_loss": 0.5943235158920288, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.07201401889324188, + "rewards/margins": 4.931735992431641, + "rewards/rejected": -5.003749370574951, + "step": 131, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.3319710782772713, + "grad_norm": 2.2726259231567383, + "learning_rate": 0.00019800675139022006, + "logits/chosen": -0.48140621185302734, + "logits/rejected": -0.5989112257957458, + "logps/chosen": -8.895767211914062, + "logps/rejected": -55.02824783325195, + "loss": 0.6833999156951904, + "memory(GiB)": 42.61, + "nll_loss": 0.3953901529312134, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.2496323436498642, + "rewards/margins": 3.696012258529663, + "rewards/rejected": -3.9456448554992676, + "step": 132, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.33448601068846273, + "grad_norm": 2.0731728076934814, + "learning_rate": 0.00019795119002434262, + "logits/chosen": -0.4898430407047272, + "logits/rejected": -0.5940038561820984, + "logps/chosen": -6.4923415184021, + "logps/rejected": -53.08872985839844, + "loss": 0.6633037328720093, + "memory(GiB)": 42.61, + "nll_loss": 0.38765111565589905, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.014707805588841438, + "rewards/margins": 3.6382081508636475, + "rewards/rejected": -3.623500108718872, + "step": 133, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.33700094309965417, + "grad_norm": 2.124537467956543, + "learning_rate": 0.00019789487289834228, + "logits/chosen": -0.38429343700408936, + "logits/rejected": -0.587513267993927, + "logps/chosen": -3.70666241645813, + "logps/rejected": -56.821434020996094, + "loss": 0.5152694582939148, + "memory(GiB)": 42.61, + "nll_loss": 0.30610963702201843, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09033096581697464, + "rewards/margins": 4.2761945724487305, + "rewards/rejected": -4.185863494873047, + "step": 134, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.33951587551084567, + "grad_norm": 1.165412425994873, + "learning_rate": 0.000197837800446744, + "logits/chosen": -0.2946363687515259, + "logits/rejected": -0.6158965229988098, + "logps/chosen": -5.17818021774292, + "logps/rejected": -85.47969818115234, + "loss": 0.34627044200897217, + "memory(GiB)": 42.61, + "nll_loss": 0.2048141062259674, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.022251788526773453, + "rewards/margins": 6.3592729568481445, + "rewards/rejected": -6.337022304534912, + "step": 135, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.3420308079220371, + "grad_norm": 1.6704083681106567, + "learning_rate": 0.00019777997310990063, + "logits/chosen": -0.34255802631378174, + "logits/rejected": -0.6556228995323181, + "logps/chosen": -2.7305595874786377, + "logps/rejected": -83.5248031616211, + "loss": 0.355721652507782, + "memory(GiB)": 42.61, + "nll_loss": 0.25845229625701904, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13533437252044678, + "rewards/margins": 6.411257266998291, + "rewards/rejected": -6.275922775268555, + "step": 136, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.34454574033322855, + "grad_norm": 1.8042700290679932, + "learning_rate": 0.00019772139133398942, + "logits/chosen": -0.3927345275878906, + "logits/rejected": -0.6234123706817627, + "logps/chosen": -5.4586615562438965, + "logps/rejected": -81.85598754882812, + "loss": 0.4663994014263153, + "memory(GiB)": 42.61, + "nll_loss": 0.2979077696800232, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.012417782098054886, + "rewards/margins": 6.194499492645264, + "rewards/rejected": -6.2069172859191895, + "step": 137, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.34706067274442, + "grad_norm": 2.92842960357666, + "learning_rate": 0.00019766205557100868, + "logits/chosen": -0.559550940990448, + "logits/rejected": -0.7328889966011047, + "logps/chosen": -9.122381210327148, + "logps/rejected": -70.6622314453125, + "loss": 0.7152190208435059, + "memory(GiB)": 42.61, + "nll_loss": 0.5275388956069946, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.06365346908569336, + "rewards/margins": 5.660825729370117, + "rewards/rejected": -5.724478721618652, + "step": 138, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.3495756051556114, + "grad_norm": 1.7731250524520874, + "learning_rate": 0.00019760196627877422, + "logits/chosen": -0.3356359004974365, + "logits/rejected": -0.7464644908905029, + "logps/chosen": -3.0185112953186035, + "logps/rejected": -116.795654296875, + "loss": 0.4166179299354553, + "memory(GiB)": 42.61, + "nll_loss": 0.28380244970321655, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.07928900420665741, + "rewards/margins": 9.400187492370605, + "rewards/rejected": -9.32089900970459, + "step": 139, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.3520905375668029, + "grad_norm": 6.910381317138672, + "learning_rate": 0.0001975411239209158, + "logits/chosen": -0.6025805473327637, + "logits/rejected": -0.7230421900749207, + "logps/chosen": -13.585058212280273, + "logps/rejected": -62.04022979736328, + "loss": 1.3271528482437134, + "memory(GiB)": 42.61, + "nll_loss": 0.8942356109619141, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.4771670699119568, + "rewards/margins": 4.614741802215576, + "rewards/rejected": -5.0919084548950195, + "step": 140, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.35460546997799436, + "grad_norm": 5.268382549285889, + "learning_rate": 0.0001974795289668737, + "logits/chosen": -0.47753405570983887, + "logits/rejected": -0.8787558674812317, + "logps/chosen": -8.216950416564941, + "logps/rejected": -102.49224090576172, + "loss": 0.7080188393592834, + "memory(GiB)": 42.61, + "nll_loss": 0.6081770658493042, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.37120816111564636, + "rewards/margins": 7.621390342712402, + "rewards/rejected": -7.992597579956055, + "step": 141, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.3571204023891858, + "grad_norm": 6.239411354064941, + "learning_rate": 0.00019741718189189485, + "logits/chosen": -0.5375499725341797, + "logits/rejected": -0.7797288298606873, + "logps/chosen": -7.561271667480469, + "logps/rejected": -72.9504623413086, + "loss": 0.9173229932785034, + "memory(GiB)": 42.61, + "nll_loss": 0.6234984397888184, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.2173769325017929, + "rewards/margins": 5.198176383972168, + "rewards/rejected": -5.415553092956543, + "step": 142, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.35963533480037724, + "grad_norm": 4.152512073516846, + "learning_rate": 0.00019735408317702948, + "logits/chosen": -0.5347225069999695, + "logits/rejected": -0.7430693507194519, + "logps/chosen": -4.227977752685547, + "logps/rejected": -57.00012969970703, + "loss": 0.5255740284919739, + "memory(GiB)": 42.61, + "nll_loss": 0.34119245409965515, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0003469698131084442, + "rewards/margins": 4.147022247314453, + "rewards/rejected": -4.1466755867004395, + "step": 143, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.3621502672115687, + "grad_norm": 1.935512900352478, + "learning_rate": 0.0001972902333091271, + "logits/chosen": -0.4890541732311249, + "logits/rejected": -0.6482999920845032, + "logps/chosen": -8.091506958007812, + "logps/rejected": -47.94240951538086, + "loss": 0.6421104669570923, + "memory(GiB)": 42.61, + "nll_loss": 0.4184344708919525, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.008477868512272835, + "rewards/margins": 3.3832898139953613, + "rewards/rejected": -3.3748116493225098, + "step": 144, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.3646651996227601, + "grad_norm": 3.257821559906006, + "learning_rate": 0.00019722563278083287, + "logits/chosen": -0.46462729573249817, + "logits/rejected": -0.71808260679245, + "logps/chosen": -3.831825017929077, + "logps/rejected": -43.67421340942383, + "loss": 0.6770502924919128, + "memory(GiB)": 42.61, + "nll_loss": 0.40698307752609253, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.03249932453036308, + "rewards/margins": 2.939378023147583, + "rewards/rejected": -2.9068784713745117, + "step": 145, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.3671801320339516, + "grad_norm": 3.0926783084869385, + "learning_rate": 0.00019716028209058387, + "logits/chosen": -0.5261669158935547, + "logits/rejected": -0.6899480223655701, + "logps/chosen": -6.620856285095215, + "logps/rejected": -44.43523406982422, + "loss": 0.9013221263885498, + "memory(GiB)": 42.61, + "nll_loss": 0.658555269241333, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.10268372297286987, + "rewards/margins": 2.719848155975342, + "rewards/rejected": -2.8225317001342773, + "step": 146, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.36969506444514305, + "grad_norm": 1.952876329421997, + "learning_rate": 0.0001970941817426052, + "logits/chosen": -0.3797496557235718, + "logits/rejected": -0.6095885038375854, + "logps/chosen": -3.3423259258270264, + "logps/rejected": -51.2366943359375, + "loss": 0.3124345541000366, + "memory(GiB)": 42.61, + "nll_loss": 0.19474804401397705, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15071216225624084, + "rewards/margins": 3.3171157836914062, + "rewards/rejected": -3.166403293609619, + "step": 147, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.3722099968563345, + "grad_norm": 5.522589683532715, + "learning_rate": 0.00019702733224690605, + "logits/chosen": -0.4721255898475647, + "logits/rejected": -0.6467065215110779, + "logps/chosen": -8.671660423278809, + "logps/rejected": -41.544979095458984, + "loss": 0.7880657911300659, + "memory(GiB)": 42.61, + "nll_loss": 0.553983211517334, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.06357112526893616, + "rewards/margins": 2.7004964351654053, + "rewards/rejected": -2.7640676498413086, + "step": 148, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.37472492926752593, + "grad_norm": 1.7939379215240479, + "learning_rate": 0.00019695973411927578, + "logits/chosen": -0.518312394618988, + "logits/rejected": -0.689118504524231, + "logps/chosen": -2.949949026107788, + "logps/rejected": -47.91648864746094, + "loss": 0.3869367241859436, + "memory(GiB)": 42.61, + "nll_loss": 0.22588227689266205, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06105111539363861, + "rewards/margins": 3.0687060356140137, + "rewards/rejected": -3.0076546669006348, + "step": 149, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.37723986167871737, + "grad_norm": 1.6997706890106201, + "learning_rate": 0.00019689138788127995, + "logits/chosen": -0.49549129605293274, + "logits/rejected": -0.6151180267333984, + "logps/chosen": -6.0540666580200195, + "logps/rejected": -35.12141036987305, + "loss": 0.6853822469711304, + "memory(GiB)": 42.61, + "nll_loss": 0.4531955420970917, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.015824541449546814, + "rewards/margins": 2.214078903198242, + "rewards/rejected": -2.198254346847534, + "step": 150, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.3797547940899088, + "grad_norm": 1.94599449634552, + "learning_rate": 0.00019682229406025635, + "logits/chosen": -0.44517412781715393, + "logits/rejected": -0.583071231842041, + "logps/chosen": -8.055398941040039, + "logps/rejected": -40.7237548828125, + "loss": 0.6454893350601196, + "memory(GiB)": 42.61, + "nll_loss": 0.38740435242652893, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.09544569253921509, + "rewards/margins": 2.4482340812683105, + "rewards/rejected": -2.3527884483337402, + "step": 151, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.3822697265011003, + "grad_norm": 1.4162402153015137, + "learning_rate": 0.00019675245318931083, + "logits/chosen": -0.5134773254394531, + "logits/rejected": -0.6419417858123779, + "logps/chosen": -9.19031047821045, + "logps/rejected": -41.0048828125, + "loss": 0.6307085752487183, + "memory(GiB)": 42.61, + "nll_loss": 0.42102259397506714, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.08155234903097153, + "rewards/margins": 2.483502149581909, + "rewards/rejected": -2.565054416656494, + "step": 152, + "train_speed(iter/s)": 0.011221 + }, + { + "epoch": 0.38478465891229174, + "grad_norm": 1.932735562324524, + "learning_rate": 0.0001966818658073133, + "logits/chosen": -0.5457805395126343, + "logits/rejected": -0.680858850479126, + "logps/chosen": -6.187590599060059, + "logps/rejected": -37.30434036254883, + "loss": 0.6069836020469666, + "memory(GiB)": 42.61, + "nll_loss": 0.3702396750450134, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.003183305263519287, + "rewards/margins": 2.3362035751342773, + "rewards/rejected": -2.3330204486846924, + "step": 153, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.3872995913234832, + "grad_norm": 2.0515635013580322, + "learning_rate": 0.0001966105324588934, + "logits/chosen": -0.5253856182098389, + "logits/rejected": -0.6266660094261169, + "logps/chosen": -11.048845291137695, + "logps/rejected": -40.014915466308594, + "loss": 0.8203007578849792, + "memory(GiB)": 42.61, + "nll_loss": 0.56341153383255, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.09865356236696243, + "rewards/margins": 2.6950490474700928, + "rewards/rejected": -2.596395492553711, + "step": 154, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.3898145237346746, + "grad_norm": 2.258105516433716, + "learning_rate": 0.00019653845369443657, + "logits/chosen": -0.49116846919059753, + "logits/rejected": -0.6502944231033325, + "logps/chosen": -6.3410539627075195, + "logps/rejected": -45.01954650878906, + "loss": 0.6601016521453857, + "memory(GiB)": 42.61, + "nll_loss": 0.4770495295524597, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08940570056438446, + "rewards/margins": 2.79660701751709, + "rewards/rejected": -2.7072014808654785, + "step": 155, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.39232945614586606, + "grad_norm": 1.9334651231765747, + "learning_rate": 0.00019646563007007952, + "logits/chosen": -0.5132689476013184, + "logits/rejected": -0.6826451420783997, + "logps/chosen": -4.647260665893555, + "logps/rejected": -35.987144470214844, + "loss": 0.5781624913215637, + "memory(GiB)": 42.61, + "nll_loss": 0.333342581987381, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.0551791712641716, + "rewards/margins": 1.9761953353881836, + "rewards/rejected": -1.9210160970687866, + "step": 156, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.3948443885570575, + "grad_norm": 2.0030107498168945, + "learning_rate": 0.00019639206214770608, + "logits/chosen": -0.5250845551490784, + "logits/rejected": -0.6993058919906616, + "logps/chosen": -3.4496142864227295, + "logps/rejected": -45.091514587402344, + "loss": 0.4796870946884155, + "memory(GiB)": 42.61, + "nll_loss": 0.27256864309310913, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.039092980325222015, + "rewards/margins": 2.82478666305542, + "rewards/rejected": -2.785693645477295, + "step": 157, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.397359320968249, + "grad_norm": 1.6849418878555298, + "learning_rate": 0.00019631775049494285, + "logits/chosen": -0.43024155497550964, + "logits/rejected": -0.6700482964515686, + "logps/chosen": -3.823197364807129, + "logps/rejected": -56.609588623046875, + "loss": 0.42021045088768005, + "memory(GiB)": 42.61, + "nll_loss": 0.24996069073677063, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.004662778228521347, + "rewards/margins": 3.1713931560516357, + "rewards/rejected": -3.1667304039001465, + "step": 158, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.39987425337944044, + "grad_norm": 3.1281967163085938, + "learning_rate": 0.00019624269568515486, + "logits/chosen": -0.43441203236579895, + "logits/rejected": -0.6460458040237427, + "logps/chosen": -6.727141857147217, + "logps/rejected": -45.23570251464844, + "loss": 0.7260392904281616, + "memory(GiB)": 42.61, + "nll_loss": 0.5018711090087891, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.011816263198852539, + "rewards/margins": 2.4935953617095947, + "rewards/rejected": -2.4817793369293213, + "step": 159, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.4023891857906319, + "grad_norm": 3.3100078105926514, + "learning_rate": 0.00019616689829744105, + "logits/chosen": -0.6007479429244995, + "logits/rejected": -0.7636129856109619, + "logps/chosen": -6.275008678436279, + "logps/rejected": -35.27611541748047, + "loss": 0.6686663627624512, + "memory(GiB)": 42.61, + "nll_loss": 0.4275904893875122, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1287657767534256, + "rewards/margins": 2.2340850830078125, + "rewards/rejected": -2.1053192615509033, + "step": 160, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.4049041182018233, + "grad_norm": 3.4858033657073975, + "learning_rate": 0.0001960903589166299, + "logits/chosen": -0.6405993700027466, + "logits/rejected": -0.697019100189209, + "logps/chosen": -9.289031982421875, + "logps/rejected": -38.57951736450195, + "loss": 0.7603179216384888, + "memory(GiB)": 42.61, + "nll_loss": 0.4232831597328186, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.159571573138237, + "rewards/margins": 2.720222234725952, + "rewards/rejected": -2.56065034866333, + "step": 161, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.40741905061301475, + "grad_norm": 1.844115972518921, + "learning_rate": 0.00019601307813327482, + "logits/chosen": -0.5259416103363037, + "logits/rejected": -0.6975188255310059, + "logps/chosen": -2.951578140258789, + "logps/rejected": -41.11060333251953, + "loss": 0.4498739242553711, + "memory(GiB)": 42.61, + "nll_loss": 0.23409336805343628, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14196570217609406, + "rewards/margins": 2.5788159370422363, + "rewards/rejected": -2.436850070953369, + "step": 162, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.40993398302420625, + "grad_norm": 5.637467861175537, + "learning_rate": 0.00019593505654364965, + "logits/chosen": -0.45152366161346436, + "logits/rejected": -0.6798401474952698, + "logps/chosen": -4.170862674713135, + "logps/rejected": -53.169776916503906, + "loss": 0.6846017241477966, + "memory(GiB)": 42.61, + "nll_loss": 0.5021944642066956, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.010230405256152153, + "rewards/margins": 3.3515424728393555, + "rewards/rejected": -3.3413121700286865, + "step": 163, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.4124489154353977, + "grad_norm": 1.4745384454727173, + "learning_rate": 0.00019585629474974415, + "logits/chosen": -0.4008867144584656, + "logits/rejected": -0.7353089451789856, + "logps/chosen": -0.4037323594093323, + "logps/rejected": -61.83756637573242, + "loss": 0.18311206996440887, + "memory(GiB)": 42.61, + "nll_loss": 0.05691269040107727, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.09548275917768478, + "rewards/margins": 4.11773157119751, + "rewards/rejected": -4.0222487449646, + "step": 164, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.41496384784658913, + "grad_norm": 1.9870681762695312, + "learning_rate": 0.0001957767933592591, + "logits/chosen": -0.5013211965560913, + "logits/rejected": -0.6647384762763977, + "logps/chosen": -6.108734130859375, + "logps/rejected": -45.8098030090332, + "loss": 0.6409457921981812, + "memory(GiB)": 42.61, + "nll_loss": 0.3451662063598633, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.025398313999176025, + "rewards/margins": 3.0071825981140137, + "rewards/rejected": -3.032581090927124, + "step": 165, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.41747878025778057, + "grad_norm": 3.7264280319213867, + "learning_rate": 0.00019569655298560195, + "logits/chosen": -0.5106596350669861, + "logits/rejected": -0.7329829931259155, + "logps/chosen": -4.24332857131958, + "logps/rejected": -48.86112594604492, + "loss": 0.5813378691673279, + "memory(GiB)": 42.61, + "nll_loss": 0.3752741515636444, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06557303667068481, + "rewards/margins": 3.4898738861083984, + "rewards/rejected": -3.4243011474609375, + "step": 166, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.419993712668972, + "grad_norm": 11.64145278930664, + "learning_rate": 0.0001956155742478817, + "logits/chosen": -0.4935857653617859, + "logits/rejected": -0.7044666409492493, + "logps/chosen": -2.654214382171631, + "logps/rejected": -57.629425048828125, + "loss": 0.3905934691429138, + "memory(GiB)": 42.61, + "nll_loss": 0.22700335085391998, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1508655548095703, + "rewards/margins": 4.179229736328125, + "rewards/rejected": -4.028364181518555, + "step": 167, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.42250864508016345, + "grad_norm": 1.6378587484359741, + "learning_rate": 0.00019553385777090458, + "logits/chosen": -0.4575374722480774, + "logits/rejected": -0.6927694082260132, + "logps/chosen": -9.244451522827148, + "logps/rejected": -63.81715774536133, + "loss": 0.6180571913719177, + "memory(GiB)": 44.38, + "nll_loss": 0.47105246782302856, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12050072848796844, + "rewards/margins": 4.137886047363281, + "rewards/rejected": -4.017385005950928, + "step": 168, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.42502357749135494, + "grad_norm": 6.146910667419434, + "learning_rate": 0.00019545140418516873, + "logits/chosen": -0.6766417622566223, + "logits/rejected": -0.7711372971534729, + "logps/chosen": -6.37576961517334, + "logps/rejected": -45.8558349609375, + "loss": 0.6669228672981262, + "memory(GiB)": 44.38, + "nll_loss": 0.5145984888076782, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18157291412353516, + "rewards/margins": 3.766644239425659, + "rewards/rejected": -3.585071563720703, + "step": 169, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.4275385099025464, + "grad_norm": 1.6984812021255493, + "learning_rate": 0.0001953682141268598, + "logits/chosen": -0.5011512637138367, + "logits/rejected": -0.688590943813324, + "logps/chosen": -3.602705478668213, + "logps/rejected": -55.99202346801758, + "loss": 0.3990403413772583, + "memory(GiB)": 44.38, + "nll_loss": 0.2418036162853241, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.027019452303647995, + "rewards/margins": 3.8597612380981445, + "rewards/rejected": -3.8327417373657227, + "step": 170, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.4300534423137378, + "grad_norm": 5.023574352264404, + "learning_rate": 0.00019528428823784567, + "logits/chosen": -0.4159168004989624, + "logits/rejected": -0.7052143216133118, + "logps/chosen": -2.2552971839904785, + "logps/rejected": -71.4410400390625, + "loss": 0.29148706793785095, + "memory(GiB)": 44.38, + "nll_loss": 0.19378283619880676, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13933303952217102, + "rewards/margins": 5.2820634841918945, + "rewards/rejected": -5.142730236053467, + "step": 171, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.43256837472492926, + "grad_norm": 1.4866183996200562, + "learning_rate": 0.0001951996271656717, + "logits/chosen": -0.5606605410575867, + "logits/rejected": -0.7212972044944763, + "logps/chosen": -3.804295539855957, + "logps/rejected": -51.83647537231445, + "loss": 0.3623957335948944, + "memory(GiB)": 44.38, + "nll_loss": 0.21611489355564117, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12722836434841156, + "rewards/margins": 3.714401960372925, + "rewards/rejected": -3.5871734619140625, + "step": 172, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.4350833071361207, + "grad_norm": 3.7671830654144287, + "learning_rate": 0.00019511423156355577, + "logits/chosen": -0.4943828582763672, + "logits/rejected": -0.68037348985672, + "logps/chosen": -5.188858509063721, + "logps/rejected": -62.85913848876953, + "loss": 0.41962915658950806, + "memory(GiB)": 44.38, + "nll_loss": 0.2764420509338379, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1342807114124298, + "rewards/margins": 4.525426387786865, + "rewards/rejected": -4.391145706176758, + "step": 173, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.43759823954731214, + "grad_norm": 9.907013893127441, + "learning_rate": 0.00019502810209038303, + "logits/chosen": -0.5515968799591064, + "logits/rejected": -0.6605091691017151, + "logps/chosen": -9.89529800415039, + "logps/rejected": -54.34276580810547, + "loss": 0.9658512473106384, + "memory(GiB)": 44.38, + "nll_loss": 0.7056573033332825, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.028916141018271446, + "rewards/margins": 3.5635647773742676, + "rewards/rejected": -3.5924811363220215, + "step": 174, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.44011317195850364, + "grad_norm": 2.1124749183654785, + "learning_rate": 0.00019494123941070108, + "logits/chosen": -0.689885675907135, + "logits/rejected": -0.7663485407829285, + "logps/chosen": -5.329808712005615, + "logps/rejected": -39.43062210083008, + "loss": 0.6925463676452637, + "memory(GiB)": 44.38, + "nll_loss": 0.473371297121048, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10141712427139282, + "rewards/margins": 3.0223309993743896, + "rewards/rejected": -2.9209136962890625, + "step": 175, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.4426281043696951, + "grad_norm": 36.679805755615234, + "learning_rate": 0.00019485364419471454, + "logits/chosen": -0.6628389954566956, + "logits/rejected": -0.7602969408035278, + "logps/chosen": -7.741974353790283, + "logps/rejected": -50.05625915527344, + "loss": 1.0915241241455078, + "memory(GiB)": 44.38, + "nll_loss": 0.7456662654876709, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.35289716720581055, + "rewards/margins": 3.5574560165405273, + "rewards/rejected": -3.910353422164917, + "step": 176, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.4451430367808865, + "grad_norm": 2.4835803508758545, + "learning_rate": 0.00019476531711828027, + "logits/chosen": -0.5590236186981201, + "logits/rejected": -0.6718421578407288, + "logps/chosen": -5.849733352661133, + "logps/rejected": -51.53736114501953, + "loss": 0.40270960330963135, + "memory(GiB)": 44.38, + "nll_loss": 0.2558823227882385, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11730007827281952, + "rewards/margins": 3.6312365531921387, + "rewards/rejected": -3.5139362812042236, + "step": 177, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.44765796919207795, + "grad_norm": 2.382329225540161, + "learning_rate": 0.00019467625886290167, + "logits/chosen": -0.5211893916130066, + "logits/rejected": -0.6878003478050232, + "logps/chosen": -11.300158500671387, + "logps/rejected": -52.15276336669922, + "loss": 0.7076267004013062, + "memory(GiB)": 44.38, + "nll_loss": 0.5078485012054443, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.07471176981925964, + "rewards/margins": 3.560542583465576, + "rewards/rejected": -3.485830783843994, + "step": 178, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.4501729016032694, + "grad_norm": 4.822922706604004, + "learning_rate": 0.0001945864701157239, + "logits/chosen": -0.5776659250259399, + "logits/rejected": -0.6799846291542053, + "logps/chosen": -4.035768032073975, + "logps/rejected": -54.64043426513672, + "loss": 0.48212817311286926, + "memory(GiB)": 44.38, + "nll_loss": 0.2608688175678253, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.17263993620872498, + "rewards/margins": 4.305383682250977, + "rewards/rejected": -4.132743835449219, + "step": 179, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.4526878340144609, + "grad_norm": 1.466308832168579, + "learning_rate": 0.00019449595156952827, + "logits/chosen": -0.6198251247406006, + "logits/rejected": -0.7188805341720581, + "logps/chosen": -7.5114359855651855, + "logps/rejected": -50.14148712158203, + "loss": 0.5570878982543945, + "memory(GiB)": 44.38, + "nll_loss": 0.3523883819580078, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18179449439048767, + "rewards/margins": 3.7667787075042725, + "rewards/rejected": -3.584984302520752, + "step": 180, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.4552027664256523, + "grad_norm": 1.5548723936080933, + "learning_rate": 0.00019440470392272696, + "logits/chosen": -0.5984869003295898, + "logits/rejected": -0.7039155960083008, + "logps/chosen": -4.1996378898620605, + "logps/rejected": -45.629638671875, + "loss": 0.47419819235801697, + "memory(GiB)": 44.38, + "nll_loss": 0.31078362464904785, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1542869210243225, + "rewards/margins": 3.5583298206329346, + "rewards/rejected": -3.404043197631836, + "step": 181, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.45771769883684377, + "grad_norm": 2.250375747680664, + "learning_rate": 0.00019431272787935773, + "logits/chosen": -0.5019629597663879, + "logits/rejected": -0.7356317639350891, + "logps/chosen": -10.908112525939941, + "logps/rejected": -52.47066116333008, + "loss": 1.0049289464950562, + "memory(GiB)": 44.38, + "nll_loss": 0.7764756083488464, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.024810096248984337, + "rewards/margins": 3.2386226654052734, + "rewards/rejected": -3.213812828063965, + "step": 182, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.4602326312480352, + "grad_norm": 22.676647186279297, + "learning_rate": 0.00019422002414907837, + "logits/chosen": -0.5647466778755188, + "logits/rejected": -0.7478159070014954, + "logps/chosen": -5.912107944488525, + "logps/rejected": -47.091400146484375, + "loss": 0.699102520942688, + "memory(GiB)": 44.38, + "nll_loss": 0.4706324338912964, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.08286911249160767, + "rewards/margins": 3.223126173019409, + "rewards/rejected": -3.140256881713867, + "step": 183, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.46274756365922665, + "grad_norm": 3.388159990310669, + "learning_rate": 0.00019412659344716124, + "logits/chosen": -0.5706430673599243, + "logits/rejected": -0.6877136826515198, + "logps/chosen": -11.683382987976074, + "logps/rejected": -46.81083297729492, + "loss": 1.3097541332244873, + "memory(GiB)": 44.38, + "nll_loss": 0.8712559342384338, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.2759455144405365, + "rewards/margins": 3.09993839263916, + "rewards/rejected": -3.3758838176727295, + "step": 184, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.4652624960704181, + "grad_norm": 2.6059823036193848, + "learning_rate": 0.00019403243649448787, + "logits/chosen": -0.47367703914642334, + "logits/rejected": -0.7172597050666809, + "logps/chosen": -3.135293483734131, + "logps/rejected": -64.97146606445312, + "loss": 0.4278358221054077, + "memory(GiB)": 44.38, + "nll_loss": 0.2768169641494751, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1706126630306244, + "rewards/margins": 4.8074727058410645, + "rewards/rejected": -4.636859893798828, + "step": 185, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.4677774284816096, + "grad_norm": 2.2509593963623047, + "learning_rate": 0.00019393755401754322, + "logits/chosen": -0.48067784309387207, + "logits/rejected": -0.6601268649101257, + "logps/chosen": -3.7737672328948975, + "logps/rejected": -61.828006744384766, + "loss": 0.44278520345687866, + "memory(GiB)": 44.38, + "nll_loss": 0.3135760426521301, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05582159385085106, + "rewards/margins": 4.652613162994385, + "rewards/rejected": -4.5967912673950195, + "step": 186, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.470292360892801, + "grad_norm": 2.220067262649536, + "learning_rate": 0.00019384194674841026, + "logits/chosen": -0.5368369221687317, + "logits/rejected": -0.7164134383201599, + "logps/chosen": -4.382404804229736, + "logps/rejected": -63.785423278808594, + "loss": 0.5859192609786987, + "memory(GiB)": 44.38, + "nll_loss": 0.4215310215950012, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.042068447917699814, + "rewards/margins": 5.157468795776367, + "rewards/rejected": -5.115400314331055, + "step": 187, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.47280729330399246, + "grad_norm": 1.3244057893753052, + "learning_rate": 0.0001937456154247641, + "logits/chosen": -0.5555019974708557, + "logits/rejected": -0.682890772819519, + "logps/chosen": -4.023111343383789, + "logps/rejected": -66.17463684082031, + "loss": 0.3469849228858948, + "memory(GiB)": 44.38, + "nll_loss": 0.22850053012371063, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08244925737380981, + "rewards/margins": 5.2128143310546875, + "rewards/rejected": -5.130365371704102, + "step": 188, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.4753222257151839, + "grad_norm": 5.5003886222839355, + "learning_rate": 0.00019364856078986652, + "logits/chosen": -0.504755437374115, + "logits/rejected": -0.6091613173484802, + "logps/chosen": -13.118067741394043, + "logps/rejected": -52.780235290527344, + "loss": 0.970127284526825, + "memory(GiB)": 44.38, + "nll_loss": 0.7232432961463928, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.04733971506357193, + "rewards/margins": 3.863572120666504, + "rewards/rejected": -3.910911798477173, + "step": 189, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.47783715812637534, + "grad_norm": 2.35336971282959, + "learning_rate": 0.0001935507835925601, + "logits/chosen": -0.4966239631175995, + "logits/rejected": -0.6422093510627747, + "logps/chosen": -5.089540004730225, + "logps/rejected": -60.03197479248047, + "loss": 0.48357081413269043, + "memory(GiB)": 44.38, + "nll_loss": 0.29401686787605286, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.06470837444067001, + "rewards/margins": 4.633599758148193, + "rewards/rejected": -4.568891525268555, + "step": 190, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.4803520905375668, + "grad_norm": 2.2711422443389893, + "learning_rate": 0.00019345228458726252, + "logits/chosen": -0.41524171829223633, + "logits/rejected": -0.5779014229774475, + "logps/chosen": -6.11699914932251, + "logps/rejected": -65.24181365966797, + "loss": 0.5390986800193787, + "memory(GiB)": 44.38, + "nll_loss": 0.2827398180961609, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.04118676483631134, + "rewards/margins": 4.437079906463623, + "rewards/rejected": -4.395893096923828, + "step": 191, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.4828670229487583, + "grad_norm": 3.4174418449401855, + "learning_rate": 0.00019335306453396064, + "logits/chosen": -0.3581904470920563, + "logits/rejected": -0.5604187250137329, + "logps/chosen": -4.7871503829956055, + "logps/rejected": -56.127437591552734, + "loss": 0.6041537523269653, + "memory(GiB)": 44.38, + "nll_loss": 0.35478317737579346, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0018234718590974808, + "rewards/margins": 3.8149027824401855, + "rewards/rejected": -3.8130791187286377, + "step": 192, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.4853819553599497, + "grad_norm": 1.6018799543380737, + "learning_rate": 0.00019325312419820473, + "logits/chosen": -0.40660223364830017, + "logits/rejected": -0.5828196406364441, + "logps/chosen": -4.125640869140625, + "logps/rejected": -52.407806396484375, + "loss": 0.4834650158882141, + "memory(GiB)": 44.38, + "nll_loss": 0.2773113250732422, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.08193957805633545, + "rewards/margins": 3.58589506149292, + "rewards/rejected": -3.503955602645874, + "step": 193, + "train_speed(iter/s)": 0.011232 + }, + { + "epoch": 0.48789688777114115, + "grad_norm": 2.956265926361084, + "learning_rate": 0.00019315246435110256, + "logits/chosen": -0.41891956329345703, + "logits/rejected": -0.5709394812583923, + "logps/chosen": -6.136990547180176, + "logps/rejected": -54.53316116333008, + "loss": 0.5783406496047974, + "memory(GiB)": 44.38, + "nll_loss": 0.4396575689315796, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.22025173902511597, + "rewards/margins": 3.940415143966675, + "rewards/rejected": -3.7201638221740723, + "step": 194, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.4904118201823326, + "grad_norm": 2.4830644130706787, + "learning_rate": 0.00019305108576931336, + "logits/chosen": -0.4251477122306824, + "logits/rejected": -0.5646131038665771, + "logps/chosen": -6.740220069885254, + "logps/rejected": -59.58277893066406, + "loss": 0.6179866194725037, + "memory(GiB)": 44.38, + "nll_loss": 0.39784371852874756, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.03337670490145683, + "rewards/margins": 4.295322418212891, + "rewards/rejected": -4.261946201324463, + "step": 195, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.49292675259352403, + "grad_norm": 2.0805859565734863, + "learning_rate": 0.0001929489892350419, + "logits/chosen": -0.4012482464313507, + "logits/rejected": -0.5744894742965698, + "logps/chosen": -3.9474775791168213, + "logps/rejected": -64.57502746582031, + "loss": 0.4640367329120636, + "memory(GiB)": 44.38, + "nll_loss": 0.32923948764801025, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11397141218185425, + "rewards/margins": 4.956214904785156, + "rewards/rejected": -4.842243194580078, + "step": 196, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.49544168500471547, + "grad_norm": 1.0079342126846313, + "learning_rate": 0.00019284617553603242, + "logits/chosen": -0.40687650442123413, + "logits/rejected": -0.5095179677009583, + "logps/chosen": -5.0278239250183105, + "logps/rejected": -57.115501403808594, + "loss": 0.5296419262886047, + "memory(GiB)": 44.38, + "nll_loss": 0.3394710123538971, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.223480224609375, + "rewards/margins": 4.145529270172119, + "rewards/rejected": -3.9220495223999023, + "step": 197, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.49795661741590697, + "grad_norm": 4.113004207611084, + "learning_rate": 0.0001927426454655627, + "logits/chosen": -0.35911667346954346, + "logits/rejected": -0.5175841450691223, + "logps/chosen": -7.666455268859863, + "logps/rejected": -67.4910659790039, + "loss": 0.5694496035575867, + "memory(GiB)": 44.38, + "nll_loss": 0.4178715944290161, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.04037248343229294, + "rewards/margins": 4.807923793792725, + "rewards/rejected": -4.848296165466309, + "step": 198, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5004715498270984, + "grad_norm": 0.9368858933448792, + "learning_rate": 0.0001926383998224377, + "logits/chosen": -0.3456733524799347, + "logits/rejected": -0.5820738077163696, + "logps/chosen": -1.7521214485168457, + "logps/rejected": -69.5865249633789, + "loss": 0.225211039185524, + "memory(GiB)": 46.14, + "nll_loss": 0.13497470319271088, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19091293215751648, + "rewards/margins": 5.2458882331848145, + "rewards/rejected": -5.0549750328063965, + "step": 199, + "train_speed(iter/s)": 0.011233 + }, + { + "epoch": 0.5029864822382899, + "grad_norm": 2.017502784729004, + "learning_rate": 0.0001925334394109835, + "logits/chosen": -0.34290656447410583, + "logits/rejected": -0.5616070032119751, + "logps/chosen": -5.75649881362915, + "logps/rejected": -66.36112213134766, + "loss": 0.5342864990234375, + "memory(GiB)": 46.14, + "nll_loss": 0.3393324017524719, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.017528314143419266, + "rewards/margins": 4.61277437210083, + "rewards/rejected": -4.630302906036377, + "step": 200, + "train_speed(iter/s)": 0.011234 + }, + { + "epoch": 0.5055014146494813, + "grad_norm": 11.226442337036133, + "learning_rate": 0.00019242776504104118, + "logits/chosen": -0.35404524207115173, + "logits/rejected": -0.5164979696273804, + "logps/chosen": -6.239109992980957, + "logps/rejected": -57.19633102416992, + "loss": 0.751146674156189, + "memory(GiB)": 46.14, + "nll_loss": 0.5300025939941406, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.03473217412829399, + "rewards/margins": 4.055646896362305, + "rewards/rejected": -4.020914554595947, + "step": 201, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.5080163470606728, + "grad_norm": 2.073298454284668, + "learning_rate": 0.00019232137752796044, + "logits/chosen": -0.4284801483154297, + "logits/rejected": -0.5613614916801453, + "logps/chosen": -4.676477909088135, + "logps/rejected": -53.5533447265625, + "loss": 0.6112902164459229, + "memory(GiB)": 46.14, + "nll_loss": 0.47125840187072754, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11576966941356659, + "rewards/margins": 3.918217420578003, + "rewards/rejected": -3.80244779586792, + "step": 202, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.5105312794718642, + "grad_norm": 1.5557101964950562, + "learning_rate": 0.00019221427769259333, + "logits/chosen": -0.34159645438194275, + "logits/rejected": -0.539400577545166, + "logps/chosen": -3.861686944961548, + "logps/rejected": -61.914703369140625, + "loss": 0.40771356225013733, + "memory(GiB)": 46.14, + "nll_loss": 0.29954829812049866, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2123047411441803, + "rewards/margins": 4.397878646850586, + "rewards/rejected": -4.185573577880859, + "step": 203, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.5130462118830557, + "grad_norm": 9.60893726348877, + "learning_rate": 0.00019210646636128807, + "logits/chosen": -0.37730276584625244, + "logits/rejected": -0.4876512885093689, + "logps/chosen": -4.578537464141846, + "logps/rejected": -47.36624526977539, + "loss": 0.45369648933410645, + "memory(GiB)": 46.14, + "nll_loss": 0.274221271276474, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08797872066497803, + "rewards/margins": 3.566829204559326, + "rewards/rejected": -3.4788498878479004, + "step": 204, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.515561144294247, + "grad_norm": 1.5941282510757446, + "learning_rate": 0.00019199794436588243, + "logits/chosen": -0.27956530451774597, + "logits/rejected": -0.4699462652206421, + "logps/chosen": -7.073207378387451, + "logps/rejected": -68.29771423339844, + "loss": 0.3854275047779083, + "memory(GiB)": 46.14, + "nll_loss": 0.2786342203617096, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12901465594768524, + "rewards/margins": 4.810124397277832, + "rewards/rejected": -4.68110990524292, + "step": 205, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.5180760767054385, + "grad_norm": 5.089592456817627, + "learning_rate": 0.00019188871254369752, + "logits/chosen": -0.31783169507980347, + "logits/rejected": -0.427703857421875, + "logps/chosen": -5.960029602050781, + "logps/rejected": -43.00883102416992, + "loss": 0.8745622634887695, + "memory(GiB)": 46.14, + "nll_loss": 0.5852820873260498, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.11267540603876114, + "rewards/margins": 2.461634635925293, + "rewards/rejected": -2.574310064315796, + "step": 206, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.52059100911663, + "grad_norm": 2.929906129837036, + "learning_rate": 0.00019177877173753128, + "logits/chosen": -0.35540899634361267, + "logits/rejected": -0.4668577313423157, + "logps/chosen": -5.586899757385254, + "logps/rejected": -46.6971435546875, + "loss": 0.5985509753227234, + "memory(GiB)": 46.14, + "nll_loss": 0.3464246988296509, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.003932179883122444, + "rewards/margins": 3.3567466735839844, + "rewards/rejected": -3.3606786727905273, + "step": 207, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.5231059415278214, + "grad_norm": 1.8040342330932617, + "learning_rate": 0.00019166812279565178, + "logits/chosen": -0.3284756541252136, + "logits/rejected": -0.43804702162742615, + "logps/chosen": -5.879787445068359, + "logps/rejected": -52.28423309326172, + "loss": 0.6572303175926208, + "memory(GiB)": 46.14, + "nll_loss": 0.46364596486091614, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10764454305171967, + "rewards/margins": 3.84014892578125, + "rewards/rejected": -3.732503890991211, + "step": 208, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.5256208739390129, + "grad_norm": 2.197678565979004, + "learning_rate": 0.000191556766571791, + "logits/chosen": -0.2658138871192932, + "logits/rejected": -0.4723971486091614, + "logps/chosen": -6.082084655761719, + "logps/rejected": -70.72124481201172, + "loss": 0.563379168510437, + "memory(GiB)": 46.14, + "nll_loss": 0.4301711916923523, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.03504669666290283, + "rewards/margins": 4.98071813583374, + "rewards/rejected": -5.0157647132873535, + "step": 209, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.5281358063502043, + "grad_norm": 1.8066035509109497, + "learning_rate": 0.000191444703925138, + "logits/chosen": -0.266013503074646, + "logits/rejected": -0.4869195520877838, + "logps/chosen": -3.5636188983917236, + "logps/rejected": -65.75146484375, + "loss": 0.41913098096847534, + "memory(GiB)": 46.14, + "nll_loss": 0.29366493225097656, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.20083719491958618, + "rewards/margins": 4.579287052154541, + "rewards/rejected": -4.3784499168396, + "step": 210, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.5306507387613958, + "grad_norm": 2.058502197265625, + "learning_rate": 0.00019133193572033242, + "logits/chosen": -0.2633608877658844, + "logits/rejected": -0.4534919857978821, + "logps/chosen": -5.140444278717041, + "logps/rejected": -65.33726501464844, + "loss": 0.4311324954032898, + "memory(GiB)": 46.14, + "nll_loss": 0.25075870752334595, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.013513857498764992, + "rewards/margins": 4.880782127380371, + "rewards/rejected": -4.894296169281006, + "step": 211, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5331656711725873, + "grad_norm": 4.458026885986328, + "learning_rate": 0.00019121846282745778, + "logits/chosen": -0.31052568554878235, + "logits/rejected": -0.40828073024749756, + "logps/chosen": -6.845982074737549, + "logps/rejected": -55.935951232910156, + "loss": 0.6287424564361572, + "memory(GiB)": 46.14, + "nll_loss": 0.47837764024734497, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.0341886430978775, + "rewards/margins": 4.157693862915039, + "rewards/rejected": -4.191883087158203, + "step": 212, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5356806035837787, + "grad_norm": 1.700737476348877, + "learning_rate": 0.00019110428612203464, + "logits/chosen": -0.3173929452896118, + "logits/rejected": -0.49177926778793335, + "logps/chosen": -6.933238506317139, + "logps/rejected": -63.277774810791016, + "loss": 0.48178917169570923, + "memory(GiB)": 46.14, + "nll_loss": 0.32791826128959656, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.009786809794604778, + "rewards/margins": 4.001126289367676, + "rewards/rejected": -3.991339683532715, + "step": 213, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5381955359949702, + "grad_norm": 2.141183614730835, + "learning_rate": 0.00019098940648501405, + "logits/chosen": -0.34126630425453186, + "logits/rejected": -0.5091477036476135, + "logps/chosen": -2.5000686645507812, + "logps/rejected": -56.68462371826172, + "loss": 0.26290345191955566, + "memory(GiB)": 46.14, + "nll_loss": 0.13039958477020264, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0922369658946991, + "rewards/margins": 4.214053153991699, + "rewards/rejected": -4.1218156814575195, + "step": 214, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5407104684061615, + "grad_norm": 1.0281932353973389, + "learning_rate": 0.0001908738248027706, + "logits/chosen": -0.31923872232437134, + "logits/rejected": -0.5267022252082825, + "logps/chosen": -3.376938819885254, + "logps/rejected": -68.53611755371094, + "loss": 0.3598852753639221, + "memory(GiB)": 46.14, + "nll_loss": 0.2368268519639969, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.13397309184074402, + "rewards/margins": 5.190869331359863, + "rewards/rejected": -5.056896209716797, + "step": 215, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.543225400817353, + "grad_norm": 1.979490876197815, + "learning_rate": 0.00019075754196709572, + "logits/chosen": -0.3671177327632904, + "logits/rejected": -0.5938350558280945, + "logps/chosen": -2.393630266189575, + "logps/rejected": -76.2723388671875, + "loss": 0.3168295919895172, + "memory(GiB)": 46.14, + "nll_loss": 0.20362113416194916, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1339578777551651, + "rewards/margins": 5.975349426269531, + "rewards/rejected": -5.8413920402526855, + "step": 216, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5457403332285444, + "grad_norm": 7.617611408233643, + "learning_rate": 0.00019064055887519062, + "logits/chosen": -0.4243953227996826, + "logits/rejected": -0.5465304851531982, + "logps/chosen": -8.081028938293457, + "logps/rejected": -70.03741455078125, + "loss": 0.7748145461082458, + "memory(GiB)": 46.14, + "nll_loss": 0.5802915096282959, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.02728523313999176, + "rewards/margins": 5.3238701820373535, + "rewards/rejected": -5.3511552810668945, + "step": 217, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.5482552656397359, + "grad_norm": 2.162104845046997, + "learning_rate": 0.00019052287642965952, + "logits/chosen": -0.40015673637390137, + "logits/rejected": -0.5828264355659485, + "logps/chosen": -5.614627838134766, + "logps/rejected": -72.79359436035156, + "loss": 0.4646216928958893, + "memory(GiB)": 46.14, + "nll_loss": 0.3376343548297882, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.10579778999090195, + "rewards/margins": 5.431089878082275, + "rewards/rejected": -5.325291633605957, + "step": 218, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5507701980509274, + "grad_norm": 7.270425319671631, + "learning_rate": 0.00019040449553850257, + "logits/chosen": -0.4019389748573303, + "logits/rejected": -0.6102054715156555, + "logps/chosen": -8.833084106445312, + "logps/rejected": -68.63448333740234, + "loss": 0.931887686252594, + "memory(GiB)": 46.14, + "nll_loss": 0.6732166409492493, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.29061418771743774, + "rewards/margins": 4.946336269378662, + "rewards/rejected": -5.236949920654297, + "step": 219, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.5532851304621188, + "grad_norm": 1.4706538915634155, + "learning_rate": 0.000190285417115109, + "logits/chosen": -0.44927898049354553, + "logits/rejected": -0.6523910164833069, + "logps/chosen": -6.016222953796387, + "logps/rejected": -71.89757537841797, + "loss": 0.5478033423423767, + "memory(GiB)": 46.14, + "nll_loss": 0.45914456248283386, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.15853077173233032, + "rewards/margins": 5.350086212158203, + "rewards/rejected": -5.191555500030518, + "step": 220, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.5558000628733103, + "grad_norm": 4.954250812530518, + "learning_rate": 0.00019016564207824992, + "logits/chosen": -0.49154138565063477, + "logits/rejected": -0.5963756442070007, + "logps/chosen": -6.144622325897217, + "logps/rejected": -47.65474319458008, + "loss": 0.753817081451416, + "memory(GiB)": 46.14, + "nll_loss": 0.5517919659614563, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.045755282044410706, + "rewards/margins": 3.4072749614715576, + "rewards/rejected": -3.4530303478240967, + "step": 221, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.5583149952845017, + "grad_norm": 20.300432205200195, + "learning_rate": 0.00019004517135207127, + "logits/chosen": -0.4070438742637634, + "logits/rejected": -0.6205348372459412, + "logps/chosen": -4.287306308746338, + "logps/rejected": -71.27021789550781, + "loss": 0.3889833390712738, + "memory(GiB)": 46.14, + "nll_loss": 0.2950345575809479, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24340416491031647, + "rewards/margins": 5.386775493621826, + "rewards/rejected": -5.143371105194092, + "step": 222, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.5608299276956932, + "grad_norm": 4.125121116638184, + "learning_rate": 0.00018992400586608676, + "logits/chosen": -0.39273932576179504, + "logits/rejected": -0.6418203115463257, + "logps/chosen": -2.4810829162597656, + "logps/rejected": -68.2148208618164, + "loss": 0.44946739077568054, + "memory(GiB)": 46.14, + "nll_loss": 0.2778445780277252, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.022916745394468307, + "rewards/margins": 4.892718315124512, + "rewards/rejected": -4.915634632110596, + "step": 223, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.5633448601068847, + "grad_norm": 1.3127343654632568, + "learning_rate": 0.00018980214655517057, + "logits/chosen": -0.3633681833744049, + "logits/rejected": -0.6210101246833801, + "logps/chosen": -2.5004661083221436, + "logps/rejected": -63.47230911254883, + "loss": 0.2738821506500244, + "memory(GiB)": 46.14, + "nll_loss": 0.1929439902305603, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16036228835582733, + "rewards/margins": 4.588406085968018, + "rewards/rejected": -4.428044319152832, + "step": 224, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.565859792518076, + "grad_norm": 1.8078713417053223, + "learning_rate": 0.00018967959435955026, + "logits/chosen": -0.401949405670166, + "logits/rejected": -0.654238224029541, + "logps/chosen": -2.999825954437256, + "logps/rejected": -78.48786926269531, + "loss": 0.30304455757141113, + "memory(GiB)": 46.14, + "nll_loss": 0.22698336839675903, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11030103266239166, + "rewards/margins": 6.056008338928223, + "rewards/rejected": -5.945706844329834, + "step": 225, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.5683747249292675, + "grad_norm": 1.1858668327331543, + "learning_rate": 0.0001895563502247995, + "logits/chosen": -0.42564502358436584, + "logits/rejected": -0.617318868637085, + "logps/chosen": -4.143329620361328, + "logps/rejected": -63.9769287109375, + "loss": 0.4845999479293823, + "memory(GiB)": 46.14, + "nll_loss": 0.3030940592288971, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05418945848941803, + "rewards/margins": 4.7669267654418945, + "rewards/rejected": -4.712737083435059, + "step": 226, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5708896573404589, + "grad_norm": 10.818222045898438, + "learning_rate": 0.00018943241510183063, + "logits/chosen": -0.4244062006473541, + "logits/rejected": -0.5733806490898132, + "logps/chosen": -6.869604587554932, + "logps/rejected": -59.63373565673828, + "loss": 0.5700305700302124, + "memory(GiB)": 46.14, + "nll_loss": 0.42680811882019043, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.11073963344097137, + "rewards/margins": 4.200588226318359, + "rewards/rejected": -4.089848518371582, + "step": 227, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5734045897516504, + "grad_norm": 1.6835912466049194, + "learning_rate": 0.00018930778994688757, + "logits/chosen": -0.43102747201919556, + "logits/rejected": -0.5742402076721191, + "logps/chosen": -5.771162986755371, + "logps/rejected": -63.63447952270508, + "loss": 0.5969828963279724, + "memory(GiB)": 46.14, + "nll_loss": 0.43208059668540955, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.05310658738017082, + "rewards/margins": 4.736733436584473, + "rewards/rejected": -4.683626651763916, + "step": 228, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5759195221628419, + "grad_norm": 11.73387336730957, + "learning_rate": 0.00018918247572153823, + "logits/chosen": -0.40985164046287537, + "logits/rejected": -0.5588063597679138, + "logps/chosen": -4.359460830688477, + "logps/rejected": -66.97330474853516, + "loss": 0.4700680375099182, + "memory(GiB)": 46.14, + "nll_loss": 0.352486252784729, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08524496108293533, + "rewards/margins": 5.1315016746521, + "rewards/rejected": -5.046257019042969, + "step": 229, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5784344545740333, + "grad_norm": 3.2223124504089355, + "learning_rate": 0.0001890564733926672, + "logits/chosen": -0.28104835748672485, + "logits/rejected": -0.4876455068588257, + "logps/chosen": -10.071537017822266, + "logps/rejected": -77.56631469726562, + "loss": 0.5610992908477783, + "memory(GiB)": 46.14, + "nll_loss": 0.38141241669654846, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.04872121661901474, + "rewards/margins": 5.269710063934326, + "rewards/rejected": -5.3184309005737305, + "step": 230, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5809493869852248, + "grad_norm": 2.0458524227142334, + "learning_rate": 0.00018892978393246818, + "logits/chosen": -0.43532639741897583, + "logits/rejected": -0.4986593723297119, + "logps/chosen": -9.826902389526367, + "logps/rejected": -54.27751541137695, + "loss": 0.5817813873291016, + "memory(GiB)": 46.14, + "nll_loss": 0.3644365966320038, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.17297013103961945, + "rewards/margins": 4.145596981048584, + "rewards/rejected": -3.972626209259033, + "step": 231, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5834643193964162, + "grad_norm": 1.3154174089431763, + "learning_rate": 0.00018880240831843666, + "logits/chosen": -0.3312157690525055, + "logits/rejected": -0.538334310054779, + "logps/chosen": -7.209171295166016, + "logps/rejected": -73.7807846069336, + "loss": 0.4854442775249481, + "memory(GiB)": 46.14, + "nll_loss": 0.35912996530532837, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1784784197807312, + "rewards/margins": 5.071676254272461, + "rewards/rejected": -4.893198013305664, + "step": 232, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5859792518076077, + "grad_norm": 2.2449073791503906, + "learning_rate": 0.00018867434753336223, + "logits/chosen": -0.4506746828556061, + "logits/rejected": -0.5418546795845032, + "logps/chosen": -6.671032428741455, + "logps/rejected": -43.13252639770508, + "loss": 0.6598604917526245, + "memory(GiB)": 46.14, + "nll_loss": 0.4920580983161926, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.15326479077339172, + "rewards/margins": 3.1810433864593506, + "rewards/rejected": -3.0277788639068604, + "step": 233, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5884941842187991, + "grad_norm": 2.0205371379852295, + "learning_rate": 0.000188545602565321, + "logits/chosen": -0.44379571080207825, + "logits/rejected": -0.5399007797241211, + "logps/chosen": -6.668996810913086, + "logps/rejected": -44.872074127197266, + "loss": 0.47623366117477417, + "memory(GiB)": 46.14, + "nll_loss": 0.3232463002204895, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.21571332216262817, + "rewards/margins": 3.2807154655456543, + "rewards/rejected": -3.065002202987671, + "step": 234, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5910091166299906, + "grad_norm": 2.841298818588257, + "learning_rate": 0.0001884161744076681, + "logits/chosen": -0.33942893147468567, + "logits/rejected": -0.5364092588424683, + "logps/chosen": -5.104942321777344, + "logps/rejected": -50.68601989746094, + "loss": 0.5965795516967773, + "memory(GiB)": 46.14, + "nll_loss": 0.4035502076148987, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.029007025063037872, + "rewards/margins": 3.369378089904785, + "rewards/rejected": -3.3403711318969727, + "step": 235, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.5935240490411821, + "grad_norm": 1.2584537267684937, + "learning_rate": 0.00018828606405902986, + "logits/chosen": -0.36284250020980835, + "logits/rejected": -0.5811104774475098, + "logps/chosen": -2.7690603733062744, + "logps/rejected": -57.4227294921875, + "loss": 0.287708044052124, + "memory(GiB)": 46.14, + "nll_loss": 0.19668889045715332, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1330767720937729, + "rewards/margins": 4.162973880767822, + "rewards/rejected": -4.029897212982178, + "step": 236, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.5960389814523734, + "grad_norm": 1.2746140956878662, + "learning_rate": 0.00018815527252329624, + "logits/chosen": -0.43088293075561523, + "logits/rejected": -0.5826693177223206, + "logps/chosen": -6.570635795593262, + "logps/rejected": -57.39180374145508, + "loss": 0.6008946895599365, + "memory(GiB)": 46.14, + "nll_loss": 0.4163006544113159, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.07822921127080917, + "rewards/margins": 4.272268772125244, + "rewards/rejected": -4.194039821624756, + "step": 237, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.5985539138635649, + "grad_norm": 2.0402894020080566, + "learning_rate": 0.00018802380080961296, + "logits/chosen": -0.3829416036605835, + "logits/rejected": -0.6285839676856995, + "logps/chosen": -2.684177875518799, + "logps/rejected": -72.15119934082031, + "loss": 0.38798508048057556, + "memory(GiB)": 46.14, + "nll_loss": 0.28602004051208496, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.056711360812187195, + "rewards/margins": 5.393987655639648, + "rewards/rejected": -5.337275981903076, + "step": 238, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.6010688462747563, + "grad_norm": 1.6787519454956055, + "learning_rate": 0.00018789164993237382, + "logits/chosen": -0.4566512107849121, + "logits/rejected": -0.6156482100486755, + "logps/chosen": -4.155914783477783, + "logps/rejected": -73.04944610595703, + "loss": 0.37050700187683105, + "memory(GiB)": 46.14, + "nll_loss": 0.2093135416507721, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.13006967306137085, + "rewards/margins": 5.779219150543213, + "rewards/rejected": -5.649148941040039, + "step": 239, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.6035837786859478, + "grad_norm": 2.6302237510681152, + "learning_rate": 0.0001877588209112128, + "logits/chosen": -0.37822332978248596, + "logits/rejected": -0.6501820087432861, + "logps/chosen": -3.1933681964874268, + "logps/rejected": -80.41326904296875, + "loss": 0.43696874380111694, + "memory(GiB)": 46.14, + "nll_loss": 0.2772809863090515, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.16505186259746552, + "rewards/margins": 6.543651103973389, + "rewards/rejected": -6.378600120544434, + "step": 240, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.6060987110971393, + "grad_norm": 1.6916310787200928, + "learning_rate": 0.00018762531477099626, + "logits/chosen": -0.29892975091934204, + "logits/rejected": -0.6198312640190125, + "logps/chosen": -5.560736656188965, + "logps/rejected": -102.83201599121094, + "loss": 0.4650719165802002, + "memory(GiB)": 46.14, + "nll_loss": 0.33948570489883423, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.010243753902614117, + "rewards/margins": 7.854136943817139, + "rewards/rejected": -7.843893051147461, + "step": 241, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.6086136435083307, + "grad_norm": 5.208255290985107, + "learning_rate": 0.00018749113254181498, + "logits/chosen": -0.5256325006484985, + "logits/rejected": -0.7164933681488037, + "logps/chosen": -2.9781463146209717, + "logps/rejected": -80.5984878540039, + "loss": 0.5060344338417053, + "memory(GiB)": 46.14, + "nll_loss": 0.3545001149177551, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0441756509244442, + "rewards/margins": 7.027956485748291, + "rewards/rejected": -6.983780860900879, + "step": 242, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6111285759195222, + "grad_norm": 2.999555826187134, + "learning_rate": 0.0001873562752589762, + "logits/chosen": -0.4497723877429962, + "logits/rejected": -0.6424503326416016, + "logps/chosen": -9.136008262634277, + "logps/rejected": -83.97636413574219, + "loss": 0.5317915678024292, + "memory(GiB)": 46.14, + "nll_loss": 0.45959895849227905, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0020806342363357544, + "rewards/margins": 6.646152496337891, + "rewards/rejected": -6.648233890533447, + "step": 243, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6136435083307136, + "grad_norm": 1.6152504682540894, + "learning_rate": 0.00018722074396299566, + "logits/chosen": -0.4504960775375366, + "logits/rejected": -0.6728327870368958, + "logps/chosen": -8.307985305786133, + "logps/rejected": -92.50747680664062, + "loss": 0.3881627023220062, + "memory(GiB)": 46.14, + "nll_loss": 0.27829721570014954, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1872478723526001, + "rewards/margins": 7.5645575523376465, + "rewards/rejected": -7.377309799194336, + "step": 244, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6161584407419051, + "grad_norm": 1.9773950576782227, + "learning_rate": 0.00018708453969958958, + "logits/chosen": -0.48630112409591675, + "logits/rejected": -0.6566104888916016, + "logps/chosen": -6.018984317779541, + "logps/rejected": -62.95851516723633, + "loss": 0.6716858744621277, + "memory(GiB)": 46.14, + "nll_loss": 0.48807603120803833, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0030519720166921616, + "rewards/margins": 4.69732141494751, + "rewards/rejected": -4.694269180297852, + "step": 245, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6186733731530966, + "grad_norm": 2.007908821105957, + "learning_rate": 0.00018694766351966667, + "logits/chosen": -0.4003494679927826, + "logits/rejected": -0.6231962442398071, + "logps/chosen": -6.337024688720703, + "logps/rejected": -75.96829223632812, + "loss": 0.49646520614624023, + "memory(GiB)": 46.14, + "nll_loss": 0.39788398146629333, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2538176476955414, + "rewards/margins": 5.768869876861572, + "rewards/rejected": -5.515052318572998, + "step": 246, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.621188305564288, + "grad_norm": 1.3329335451126099, + "learning_rate": 0.00018681011647931974, + "logits/chosen": -0.35181480646133423, + "logits/rejected": -0.6176152229309082, + "logps/chosen": -5.239382266998291, + "logps/rejected": -74.54182434082031, + "loss": 0.5128018856048584, + "memory(GiB)": 46.14, + "nll_loss": 0.3926093578338623, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.14708997309207916, + "rewards/margins": 5.654610633850098, + "rewards/rejected": -5.5075201988220215, + "step": 247, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6237032379754794, + "grad_norm": 2.318981170654297, + "learning_rate": 0.00018667189963981793, + "logits/chosen": -0.4104291498661041, + "logits/rejected": -0.6699872016906738, + "logps/chosen": -4.179399013519287, + "logps/rejected": -62.75163269042969, + "loss": 0.595816969871521, + "memory(GiB)": 46.14, + "nll_loss": 0.4384332001209259, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.17818482220172882, + "rewards/margins": 4.639157772064209, + "rewards/rejected": -4.460972309112549, + "step": 248, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6262181703866708, + "grad_norm": 1.3054991960525513, + "learning_rate": 0.00018653301406759825, + "logits/chosen": -0.45266249775886536, + "logits/rejected": -0.7092356085777283, + "logps/chosen": -4.101914882659912, + "logps/rejected": -68.14440155029297, + "loss": 0.38595661520957947, + "memory(GiB)": 46.14, + "nll_loss": 0.2959481179714203, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18340018391609192, + "rewards/margins": 5.416258335113525, + "rewards/rejected": -5.232857704162598, + "step": 249, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.6287331027978623, + "grad_norm": 2.184175968170166, + "learning_rate": 0.00018639346083425748, + "logits/chosen": -0.5093356370925903, + "logits/rejected": -0.6794995665550232, + "logps/chosen": -8.688965797424316, + "logps/rejected": -66.35594940185547, + "loss": 0.8350189328193665, + "memory(GiB)": 46.14, + "nll_loss": 0.6493321657180786, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0820498839020729, + "rewards/margins": 5.020396709442139, + "rewards/rejected": -4.9383463859558105, + "step": 250, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.6312480352090537, + "grad_norm": 1.0683221817016602, + "learning_rate": 0.00018625324101654377, + "logits/chosen": -0.37649020552635193, + "logits/rejected": -0.6572071313858032, + "logps/chosen": -4.348738193511963, + "logps/rejected": -87.71903228759766, + "loss": 0.40204933285713196, + "memory(GiB)": 46.14, + "nll_loss": 0.3149075210094452, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15514017641544342, + "rewards/margins": 7.00824499130249, + "rewards/rejected": -6.853104591369629, + "step": 251, + "train_speed(iter/s)": 0.011222 + }, + { + "epoch": 0.6337629676202452, + "grad_norm": 2.92044997215271, + "learning_rate": 0.00018611235569634854, + "logits/chosen": -0.43412554264068604, + "logits/rejected": -0.6498897075653076, + "logps/chosen": -5.39566707611084, + "logps/rejected": -92.27975463867188, + "loss": 0.3511677384376526, + "memory(GiB)": 46.14, + "nll_loss": 0.2868626117706299, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19005966186523438, + "rewards/margins": 7.602224349975586, + "rewards/rejected": -7.412164688110352, + "step": 252, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.6362779000314367, + "grad_norm": 6.411540508270264, + "learning_rate": 0.00018597080596069793, + "logits/chosen": -0.5317556262016296, + "logits/rejected": -0.7095096111297607, + "logps/chosen": -9.501699447631836, + "logps/rejected": -75.72296142578125, + "loss": 0.7277939319610596, + "memory(GiB)": 46.14, + "nll_loss": 0.5560886859893799, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10351304709911346, + "rewards/margins": 6.114856719970703, + "rewards/rejected": -6.011343479156494, + "step": 253, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.6387928324426281, + "grad_norm": 1.3415000438690186, + "learning_rate": 0.00018582859290174452, + "logits/chosen": -0.5152278542518616, + "logits/rejected": -0.6576297283172607, + "logps/chosen": -4.799291133880615, + "logps/rejected": -65.77213287353516, + "loss": 0.5212017893791199, + "memory(GiB)": 46.14, + "nll_loss": 0.37971436977386475, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.1788114607334137, + "rewards/margins": 5.149197578430176, + "rewards/rejected": -4.970386028289795, + "step": 254, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.6413077648538196, + "grad_norm": 2.0600571632385254, + "learning_rate": 0.00018568571761675893, + "logits/chosen": -0.5194295644760132, + "logits/rejected": -0.6636701822280884, + "logps/chosen": -6.026893615722656, + "logps/rejected": -72.3927993774414, + "loss": 0.6823182106018066, + "memory(GiB)": 46.14, + "nll_loss": 0.4406730830669403, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.013813409954309464, + "rewards/margins": 5.530622482299805, + "rewards/rejected": -5.544435501098633, + "step": 255, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.643822697265011, + "grad_norm": 1.9565048217773438, + "learning_rate": 0.00018554218120812123, + "logits/chosen": -0.481246680021286, + "logits/rejected": -0.6363787651062012, + "logps/chosen": -9.490486145019531, + "logps/rejected": -75.51252746582031, + "loss": 0.6278063058853149, + "memory(GiB)": 46.14, + "nll_loss": 0.4966178238391876, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2284890115261078, + "rewards/margins": 5.836487293243408, + "rewards/rejected": -5.607998847961426, + "step": 256, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.6463376296762025, + "grad_norm": 1.4044089317321777, + "learning_rate": 0.00018539798478331253, + "logits/chosen": -0.4915826916694641, + "logits/rejected": -0.6845091581344604, + "logps/chosen": -5.873336315155029, + "logps/rejected": -88.4930419921875, + "loss": 0.44303250312805176, + "memory(GiB)": 46.14, + "nll_loss": 0.3297483026981354, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1546323597431183, + "rewards/margins": 7.295997142791748, + "rewards/rejected": -7.141365051269531, + "step": 257, + "train_speed(iter/s)": 0.011223 + }, + { + "epoch": 0.648852562087394, + "grad_norm": 3.9914138317108154, + "learning_rate": 0.00018525312945490648, + "logits/chosen": -0.5438416004180908, + "logits/rejected": -0.6368093490600586, + "logps/chosen": -8.48979377746582, + "logps/rejected": -51.59290313720703, + "loss": 0.8118419647216797, + "memory(GiB)": 46.14, + "nll_loss": 0.5555844306945801, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.0014183670282363892, + "rewards/margins": 3.946876049041748, + "rewards/rejected": -3.9454574584960938, + "step": 258, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.6513674944985853, + "grad_norm": 1.5120611190795898, + "learning_rate": 0.0001851076163405605, + "logits/chosen": -0.4779820442199707, + "logits/rejected": -0.6549505591392517, + "logps/chosen": -5.458694934844971, + "logps/rejected": -67.10589599609375, + "loss": 0.5383408069610596, + "memory(GiB)": 46.14, + "nll_loss": 0.34656140208244324, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06555496156215668, + "rewards/margins": 5.2232513427734375, + "rewards/rejected": -5.157696723937988, + "step": 259, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.6538824269097768, + "grad_norm": 1.8427090644836426, + "learning_rate": 0.0001849614465630074, + "logits/chosen": -0.5466744303703308, + "logits/rejected": -0.7123967409133911, + "logps/chosen": -4.491105079650879, + "logps/rejected": -75.23402404785156, + "loss": 0.43553516268730164, + "memory(GiB)": 46.14, + "nll_loss": 0.29759523272514343, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1540081650018692, + "rewards/margins": 6.407083988189697, + "rewards/rejected": -6.25307559967041, + "step": 260, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.6563973593209682, + "grad_norm": 1.5477027893066406, + "learning_rate": 0.00018481462125004647, + "logits/chosen": -0.493845134973526, + "logits/rejected": -0.687497615814209, + "logps/chosen": -6.834686279296875, + "logps/rejected": -85.95507049560547, + "loss": 0.5149559378623962, + "memory(GiB)": 46.14, + "nll_loss": 0.2903035581111908, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.049691248685121536, + "rewards/margins": 6.8722639083862305, + "rewards/rejected": -6.9219560623168945, + "step": 261, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.6589122917321597, + "grad_norm": 2.615581512451172, + "learning_rate": 0.00018466714153453503, + "logits/chosen": -0.402892142534256, + "logits/rejected": -0.6777759194374084, + "logps/chosen": -3.014004945755005, + "logps/rejected": -93.15167236328125, + "loss": 0.34462472796440125, + "memory(GiB)": 46.14, + "nll_loss": 0.2536306083202362, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15657053887844086, + "rewards/margins": 7.013685703277588, + "rewards/rejected": -6.857114791870117, + "step": 262, + "train_speed(iter/s)": 0.011224 + }, + { + "epoch": 0.6614272241433512, + "grad_norm": 3.427328586578369, + "learning_rate": 0.0001845190085543795, + "logits/chosen": -0.5253512859344482, + "logits/rejected": -0.7107342481613159, + "logps/chosen": -11.115226745605469, + "logps/rejected": -88.85804748535156, + "loss": 0.6678985953330994, + "memory(GiB)": 46.14, + "nll_loss": 0.48055583238601685, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.28667983412742615, + "rewards/margins": 6.657105445861816, + "rewards/rejected": -6.943784713745117, + "step": 263, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.6639421565545426, + "grad_norm": 1.1444027423858643, + "learning_rate": 0.00018437022345252664, + "logits/chosen": -0.5300915837287903, + "logits/rejected": -0.7409610748291016, + "logps/chosen": -3.0393667221069336, + "logps/rejected": -97.51324462890625, + "loss": 0.33918118476867676, + "memory(GiB)": 46.14, + "nll_loss": 0.24477338790893555, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08205480128526688, + "rewards/margins": 8.255426406860352, + "rewards/rejected": -8.173370361328125, + "step": 264, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.6664570889657341, + "grad_norm": 4.637787342071533, + "learning_rate": 0.0001842207873769548, + "logits/chosen": -0.5017226934432983, + "logits/rejected": -0.6932743787765503, + "logps/chosen": -5.375912189483643, + "logps/rejected": -72.31993865966797, + "loss": 0.6298449635505676, + "memory(GiB)": 46.14, + "nll_loss": 0.44052040576934814, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.06435458362102509, + "rewards/margins": 5.378176689147949, + "rewards/rejected": -5.313821792602539, + "step": 265, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.6689720213769255, + "grad_norm": 4.79957389831543, + "learning_rate": 0.00018407070148066513, + "logits/chosen": -0.426089882850647, + "logits/rejected": -0.639220118522644, + "logps/chosen": -5.063274383544922, + "logps/rejected": -87.9122085571289, + "loss": 0.4773625135421753, + "memory(GiB)": 46.14, + "nll_loss": 0.3494463562965393, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08732649683952332, + "rewards/margins": 7.158117294311523, + "rewards/rejected": -7.0707902908325195, + "step": 266, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.671486953788117, + "grad_norm": 2.3448801040649414, + "learning_rate": 0.00018391996692167242, + "logits/chosen": -0.5341528058052063, + "logits/rejected": -0.741336464881897, + "logps/chosen": -4.58453369140625, + "logps/rejected": -77.93905639648438, + "loss": 0.39634108543395996, + "memory(GiB)": 46.14, + "nll_loss": 0.26042404770851135, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2408846765756607, + "rewards/margins": 6.696462154388428, + "rewards/rejected": -6.455577373504639, + "step": 267, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.6740018861993083, + "grad_norm": 3.913471221923828, + "learning_rate": 0.00018376858486299647, + "logits/chosen": -0.44522032141685486, + "logits/rejected": -0.7187605500221252, + "logps/chosen": -5.051987648010254, + "logps/rejected": -110.36135864257812, + "loss": 0.45356252789497375, + "memory(GiB)": 46.14, + "nll_loss": 0.32329580187797546, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0719699114561081, + "rewards/margins": 9.275440216064453, + "rewards/rejected": -9.203469276428223, + "step": 268, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.6765168186104998, + "grad_norm": 5.869787216186523, + "learning_rate": 0.00018361655647265295, + "logits/chosen": -0.5306439399719238, + "logits/rejected": -0.7627276182174683, + "logps/chosen": -4.171728134155273, + "logps/rejected": -88.16104125976562, + "loss": 0.43114253878593445, + "memory(GiB)": 46.14, + "nll_loss": 0.3357262909412384, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06204725801944733, + "rewards/margins": 7.2651519775390625, + "rewards/rejected": -7.2031049728393555, + "step": 269, + "train_speed(iter/s)": 0.011225 + }, + { + "epoch": 0.6790317510216913, + "grad_norm": 4.243189334869385, + "learning_rate": 0.00018346388292364435, + "logits/chosen": -0.4333181083202362, + "logits/rejected": -0.704163670539856, + "logps/chosen": -2.6535372734069824, + "logps/rejected": -95.77774810791016, + "loss": 0.29697632789611816, + "memory(GiB)": 46.14, + "nll_loss": 0.17106658220291138, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.18806029856204987, + "rewards/margins": 7.7735371589660645, + "rewards/rejected": -7.585476875305176, + "step": 270, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.6815466834328827, + "grad_norm": 1.7140799760818481, + "learning_rate": 0.00018331056539395112, + "logits/chosen": -0.556128740310669, + "logits/rejected": -0.7013101577758789, + "logps/chosen": -4.00136137008667, + "logps/rejected": -67.00804901123047, + "loss": 0.5140962600708008, + "memory(GiB)": 46.14, + "nll_loss": 0.3367845118045807, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.10617542266845703, + "rewards/margins": 5.6274027824401855, + "rewards/rejected": -5.5212273597717285, + "step": 271, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.6840616158440742, + "grad_norm": 1.9975121021270752, + "learning_rate": 0.00018315660506652232, + "logits/chosen": -0.5180048942565918, + "logits/rejected": -0.7096535563468933, + "logps/chosen": -4.446587562561035, + "logps/rejected": -89.62254333496094, + "loss": 0.3145523965358734, + "memory(GiB)": 46.14, + "nll_loss": 0.23890048265457153, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19064153730869293, + "rewards/margins": 7.726916313171387, + "rewards/rejected": -7.5362749099731445, + "step": 272, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.6865765482552656, + "grad_norm": 2.6164450645446777, + "learning_rate": 0.00018300200312926674, + "logits/chosen": -0.4427982568740845, + "logits/rejected": -0.6760357618331909, + "logps/chosen": -4.68469762802124, + "logps/rejected": -69.42048645019531, + "loss": 0.6788820028305054, + "memory(GiB)": 46.14, + "nll_loss": 0.44468939304351807, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06053057312965393, + "rewards/margins": 5.069634437561035, + "rewards/rejected": -5.0091047286987305, + "step": 273, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.6890914806664571, + "grad_norm": 3.0418243408203125, + "learning_rate": 0.00018284676077504362, + "logits/chosen": -0.443393349647522, + "logits/rejected": -0.6204345226287842, + "logps/chosen": -7.827211380004883, + "logps/rejected": -61.0114631652832, + "loss": 0.6672328114509583, + "memory(GiB)": 46.14, + "nll_loss": 0.5125250816345215, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.12237226963043213, + "rewards/margins": 4.810492515563965, + "rewards/rejected": -4.688120365142822, + "step": 274, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.6916064130776486, + "grad_norm": 4.901159763336182, + "learning_rate": 0.00018269087920165332, + "logits/chosen": -0.41514497995376587, + "logits/rejected": -0.6380784511566162, + "logps/chosen": -6.606781482696533, + "logps/rejected": -82.10425567626953, + "loss": 0.49194085597991943, + "memory(GiB)": 46.14, + "nll_loss": 0.33267742395401, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.15449321269989014, + "rewards/margins": 6.2364373207092285, + "rewards/rejected": -6.081944942474365, + "step": 275, + "train_speed(iter/s)": 0.011226 + }, + { + "epoch": 0.69412134548884, + "grad_norm": 1.3309578895568848, + "learning_rate": 0.00018253435961182845, + "logits/chosen": -0.5140801668167114, + "logits/rejected": -0.6796289086341858, + "logps/chosen": -5.0471296310424805, + "logps/rejected": -85.82859802246094, + "loss": 0.47967594861984253, + "memory(GiB)": 46.14, + "nll_loss": 0.3782137930393219, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17723581194877625, + "rewards/margins": 6.987857341766357, + "rewards/rejected": -6.810622215270996, + "step": 276, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.6966362779000315, + "grad_norm": 1.0236581563949585, + "learning_rate": 0.00018237720321322409, + "logits/chosen": -0.42819637060165405, + "logits/rejected": -0.5975549221038818, + "logps/chosen": -5.334194183349609, + "logps/rejected": -66.20035552978516, + "loss": 0.4676276445388794, + "memory(GiB)": 46.14, + "nll_loss": 0.34695690870285034, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.38220563530921936, + "rewards/margins": 5.525548458099365, + "rewards/rejected": -5.143343448638916, + "step": 277, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.6991512103112228, + "grad_norm": 2.6781225204467773, + "learning_rate": 0.00018221941121840892, + "logits/chosen": -0.49049457907676697, + "logits/rejected": -0.7119749188423157, + "logps/chosen": -3.930919885635376, + "logps/rejected": -84.6829833984375, + "loss": 0.46516090631484985, + "memory(GiB)": 46.14, + "nll_loss": 0.34264886379241943, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.15465623140335083, + "rewards/margins": 7.022001266479492, + "rewards/rejected": -6.867344856262207, + "step": 278, + "train_speed(iter/s)": 0.011227 + }, + { + "epoch": 0.7016661427224143, + "grad_norm": 0.9837477207183838, + "learning_rate": 0.00018206098484485563, + "logits/chosen": -0.4659290313720703, + "logits/rejected": -0.6759628653526306, + "logps/chosen": -3.7083301544189453, + "logps/rejected": -81.18896484375, + "loss": 0.33244359493255615, + "memory(GiB)": 46.14, + "nll_loss": 0.24541471898555756, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.05203983187675476, + "rewards/margins": 6.756404399871826, + "rewards/rejected": -6.704365253448486, + "step": 279, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.7041810751336058, + "grad_norm": 1.9367793798446655, + "learning_rate": 0.00018190192531493152, + "logits/chosen": -0.4372798502445221, + "logits/rejected": -0.6702994704246521, + "logps/chosen": -3.2311387062072754, + "logps/rejected": -81.85215759277344, + "loss": 0.3242435157299042, + "memory(GiB)": 46.14, + "nll_loss": 0.18884681165218353, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.14998643100261688, + "rewards/margins": 6.440362453460693, + "rewards/rejected": -6.290376663208008, + "step": 280, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.7066960075447972, + "grad_norm": 6.8672566413879395, + "learning_rate": 0.00018174223385588917, + "logits/chosen": -0.4230336844921112, + "logits/rejected": -0.6848682761192322, + "logps/chosen": -6.529013156890869, + "logps/rejected": -90.91963195800781, + "loss": 0.501898467540741, + "memory(GiB)": 46.14, + "nll_loss": 0.3414127826690674, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.014151861891150475, + "rewards/margins": 6.871521472930908, + "rewards/rejected": -6.857369422912598, + "step": 281, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.7092109399559887, + "grad_norm": 1.8030977249145508, + "learning_rate": 0.00018158191169985696, + "logits/chosen": -0.5189124941825867, + "logits/rejected": -0.6720729470252991, + "logps/chosen": -3.5770914554595947, + "logps/rejected": -72.2397689819336, + "loss": 0.3821221888065338, + "memory(GiB)": 46.14, + "nll_loss": 0.26037099957466125, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1488955020904541, + "rewards/margins": 6.099178314208984, + "rewards/rejected": -5.950283050537109, + "step": 282, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.7117258723671801, + "grad_norm": 1.3741241693496704, + "learning_rate": 0.00018142096008382942, + "logits/chosen": -0.504133403301239, + "logits/rejected": -0.7275675535202026, + "logps/chosen": -3.2252137660980225, + "logps/rejected": -85.62193298339844, + "loss": 0.4202941358089447, + "memory(GiB)": 46.14, + "nll_loss": 0.31953656673431396, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.163682222366333, + "rewards/margins": 6.74496603012085, + "rewards/rejected": -6.5812835693359375, + "step": 283, + "train_speed(iter/s)": 0.011228 + }, + { + "epoch": 0.7142408047783716, + "grad_norm": 2.163975954055786, + "learning_rate": 0.00018125938024965793, + "logits/chosen": -0.5000174045562744, + "logits/rejected": -0.6639881134033203, + "logps/chosen": -4.828487396240234, + "logps/rejected": -76.62934875488281, + "loss": 0.640762448310852, + "memory(GiB)": 46.14, + "nll_loss": 0.42961040139198303, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.11060743778944016, + "rewards/margins": 6.323296546936035, + "rewards/rejected": -6.21268892288208, + "step": 284, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.716755737189563, + "grad_norm": 2.45220685005188, + "learning_rate": 0.0001810971734440408, + "logits/chosen": -0.5609660744667053, + "logits/rejected": -0.7155238389968872, + "logps/chosen": -5.233462333679199, + "logps/rejected": -67.92847442626953, + "loss": 0.46715691685676575, + "memory(GiB)": 46.14, + "nll_loss": 0.31150108575820923, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.1065329983830452, + "rewards/margins": 5.596135139465332, + "rewards/rejected": -5.489602088928223, + "step": 285, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.7192706696007545, + "grad_norm": 3.306640386581421, + "learning_rate": 0.0001809343409185141, + "logits/chosen": -0.48307013511657715, + "logits/rejected": -0.7068489193916321, + "logps/chosen": -4.671525955200195, + "logps/rejected": -86.49360656738281, + "loss": 0.45381927490234375, + "memory(GiB)": 46.14, + "nll_loss": 0.3168652355670929, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.32458728551864624, + "rewards/margins": 7.187753200531006, + "rewards/rejected": -6.863165378570557, + "step": 286, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.721785602011946, + "grad_norm": 8.64627742767334, + "learning_rate": 0.00018077088392944157, + "logits/chosen": -0.4658992886543274, + "logits/rejected": -0.6729714274406433, + "logps/chosen": -6.4438018798828125, + "logps/rejected": -103.12019348144531, + "loss": 0.7503845691680908, + "memory(GiB)": 46.14, + "nll_loss": 0.5230257511138916, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.06617186963558197, + "rewards/margins": 8.38872241973877, + "rewards/rejected": -8.454893112182617, + "step": 287, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.7243005344231374, + "grad_norm": 13.93918514251709, + "learning_rate": 0.00018060680373800517, + "logits/chosen": -0.550601601600647, + "logits/rejected": -0.7022511959075928, + "logps/chosen": -8.245074272155762, + "logps/rejected": -75.76158905029297, + "loss": 1.1637423038482666, + "memory(GiB)": 46.14, + "nll_loss": 0.71792072057724, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.2617635130882263, + "rewards/margins": 6.171159744262695, + "rewards/rejected": -6.432923316955566, + "step": 288, + "train_speed(iter/s)": 0.011229 + }, + { + "epoch": 0.7268154668343288, + "grad_norm": 0.9253904819488525, + "learning_rate": 0.00018044210161019536, + "logits/chosen": -0.557449996471405, + "logits/rejected": -0.7140588164329529, + "logps/chosen": -4.058953285217285, + "logps/rejected": -82.31138610839844, + "loss": 0.4642924666404724, + "memory(GiB)": 46.14, + "nll_loss": 0.3196372389793396, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24701666831970215, + "rewards/margins": 6.999866962432861, + "rewards/rejected": -6.75285005569458, + "step": 289, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.7293303992455202, + "grad_norm": 1.6528226137161255, + "learning_rate": 0.0001802767788168011, + "logits/chosen": -0.4495914578437805, + "logits/rejected": -0.6348967552185059, + "logps/chosen": -5.4721455574035645, + "logps/rejected": -72.90061950683594, + "loss": 0.5594840049743652, + "memory(GiB)": 46.14, + "nll_loss": 0.3900570571422577, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.03378419950604439, + "rewards/margins": 5.539588928222656, + "rewards/rejected": -5.505805015563965, + "step": 290, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.7318453316567117, + "grad_norm": 2.687760353088379, + "learning_rate": 0.0001801108366334004, + "logits/chosen": -0.4720868468284607, + "logits/rejected": -0.7254785299301147, + "logps/chosen": -3.6542224884033203, + "logps/rejected": -90.37942504882812, + "loss": 0.3632267117500305, + "memory(GiB)": 46.14, + "nll_loss": 0.25571519136428833, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.04553738981485367, + "rewards/margins": 7.5396728515625, + "rewards/rejected": -7.49413537979126, + "step": 291, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.7343602640679032, + "grad_norm": 2.385871410369873, + "learning_rate": 0.00017994427634035015, + "logits/chosen": -0.43004167079925537, + "logits/rejected": -0.6407785415649414, + "logps/chosen": -5.274074554443359, + "logps/rejected": -73.99935913085938, + "loss": 0.5132104754447937, + "memory(GiB)": 46.14, + "nll_loss": 0.40467336773872375, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.04765544831752777, + "rewards/margins": 5.948093891143799, + "rewards/rejected": -5.9004387855529785, + "step": 292, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.7368751964790946, + "grad_norm": 1.4301080703735352, + "learning_rate": 0.00017977709922277634, + "logits/chosen": -0.522111713886261, + "logits/rejected": -0.7116501331329346, + "logps/chosen": -3.2741823196411133, + "logps/rejected": -75.16236877441406, + "loss": 0.30991509556770325, + "memory(GiB)": 46.14, + "nll_loss": 0.18252958357334137, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1758219599723816, + "rewards/margins": 6.132888317108154, + "rewards/rejected": -5.957066059112549, + "step": 293, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.7393901288902861, + "grad_norm": 0.9421151876449585, + "learning_rate": 0.00017960930657056438, + "logits/chosen": -0.4752303957939148, + "logits/rejected": -0.763075053691864, + "logps/chosen": -2.069356918334961, + "logps/rejected": -101.09730529785156, + "loss": 0.273615300655365, + "memory(GiB)": 46.14, + "nll_loss": 0.1774016171693802, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18013092875480652, + "rewards/margins": 8.264806747436523, + "rewards/rejected": -8.084675788879395, + "step": 294, + "train_speed(iter/s)": 0.01123 + }, + { + "epoch": 0.7419050613014775, + "grad_norm": 1.586992859840393, + "learning_rate": 0.00017944089967834875, + "logits/chosen": -0.5891596674919128, + "logits/rejected": -0.7932709455490112, + "logps/chosen": -4.589641094207764, + "logps/rejected": -86.26122283935547, + "loss": 0.40633922815322876, + "memory(GiB)": 46.14, + "nll_loss": 0.2850261628627777, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.16272488236427307, + "rewards/margins": 7.014225482940674, + "rewards/rejected": -6.851500511169434, + "step": 295, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.744419993712669, + "grad_norm": 5.1989850997924805, + "learning_rate": 0.00017927187984550334, + "logits/chosen": -0.4999873638153076, + "logits/rejected": -0.7794056534767151, + "logps/chosen": -5.444850921630859, + "logps/rejected": -99.01435852050781, + "loss": 0.4726611077785492, + "memory(GiB)": 46.14, + "nll_loss": 0.3788090646266937, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.03491847217082977, + "rewards/margins": 8.243888854980469, + "rewards/rejected": -8.208970069885254, + "step": 296, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.7469349261238605, + "grad_norm": 1.32076895236969, + "learning_rate": 0.00017910224837613118, + "logits/chosen": -0.5675287246704102, + "logits/rejected": -0.7462685704231262, + "logps/chosen": -5.531588554382324, + "logps/rejected": -90.58869171142578, + "loss": 0.3903141915798187, + "memory(GiB)": 46.14, + "nll_loss": 0.2512432634830475, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1347513198852539, + "rewards/margins": 7.417156219482422, + "rewards/rejected": -7.282405853271484, + "step": 297, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.7494498585350519, + "grad_norm": 12.773048400878906, + "learning_rate": 0.0001789320065790546, + "logits/chosen": -0.5981060266494751, + "logits/rejected": -0.8135245442390442, + "logps/chosen": -6.99806547164917, + "logps/rejected": -108.45048522949219, + "loss": 0.4520961344242096, + "memory(GiB)": 46.14, + "nll_loss": 0.3480715751647949, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.1965925544500351, + "rewards/margins": 9.141500473022461, + "rewards/rejected": -8.944907188415527, + "step": 298, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.7519647909462434, + "grad_norm": 11.585864067077637, + "learning_rate": 0.00017876115576780498, + "logits/chosen": -0.6520665884017944, + "logits/rejected": -0.8114386796951294, + "logps/chosen": -11.475854873657227, + "logps/rejected": -93.8547134399414, + "loss": 1.136841893196106, + "memory(GiB)": 46.14, + "nll_loss": 0.9128180146217346, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.3643069267272949, + "rewards/margins": 7.67158317565918, + "rewards/rejected": -8.035890579223633, + "step": 299, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.7544797233574347, + "grad_norm": 10.311992645263672, + "learning_rate": 0.0001785896972606126, + "logits/chosen": -0.7280598282814026, + "logits/rejected": -0.9233004450798035, + "logps/chosen": -9.302474975585938, + "logps/rejected": -110.05669403076172, + "loss": 0.8829553723335266, + "memory(GiB)": 46.14, + "nll_loss": 0.6579189896583557, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.04713385924696922, + "rewards/margins": 9.603901863098145, + "rewards/rejected": -9.651037216186523, + "step": 300, + "train_speed(iter/s)": 0.011231 + }, + { + "epoch": 0.7544797233574347, + "eval_logits/chosen": -0.6994932889938354, + "eval_logits/rejected": -0.9091920852661133, + "eval_logps/chosen": -7.593709468841553, + "eval_logps/rejected": -103.9007797241211, + "eval_loss": 0.6852853894233704, + "eval_nll_loss": 0.48945868015289307, + "eval_rewards/accuracies": 0.9375, + "eval_rewards/chosen": -0.08310390263795853, + "eval_rewards/margins": 8.748055458068848, + "eval_rewards/rejected": -8.831159591674805, + "eval_runtime": 138.8625, + "eval_samples_per_second": 0.922, + "eval_steps_per_second": 0.461, + "step": 300 + }, + { + "epoch": 0.7569946557686262, + "grad_norm": 4.718209743499756, + "learning_rate": 0.00017841763238039663, + "logits/chosen": -0.6275457739830017, + "logits/rejected": -0.9186823964118958, + "logps/chosen": -12.871590614318848, + "logps/rejected": -112.10943603515625, + "loss": 0.719530463218689, + "memory(GiB)": 46.14, + "nll_loss": 0.5595242977142334, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.18099750578403473, + "rewards/margins": 9.218995094299316, + "rewards/rejected": -9.399992942810059, + "step": 301, + "train_speed(iter/s)": 0.011166 + }, + { + "epoch": 0.7595095881798176, + "grad_norm": 2.1542508602142334, + "learning_rate": 0.00017824496245475477, + "logits/chosen": -0.646737813949585, + "logits/rejected": -0.8969013690948486, + "logps/chosen": -5.7762556076049805, + "logps/rejected": -97.33111572265625, + "loss": 0.5689316987991333, + "memory(GiB)": 46.14, + "nll_loss": 0.438076913356781, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.00827467255294323, + "rewards/margins": 8.17311954498291, + "rewards/rejected": -8.164844512939453, + "step": 302, + "train_speed(iter/s)": 0.011167 + }, + { + "epoch": 0.7620245205910091, + "grad_norm": 3.397379159927368, + "learning_rate": 0.00017807168881595303, + "logits/chosen": -0.5890093445777893, + "logits/rejected": -0.8686433434486389, + "logps/chosen": -3.385796546936035, + "logps/rejected": -108.06370544433594, + "loss": 0.3380297124385834, + "memory(GiB)": 46.14, + "nll_loss": 0.2262319028377533, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.01376427710056305, + "rewards/margins": 8.914027214050293, + "rewards/rejected": -8.9277925491333, + "step": 303, + "train_speed(iter/s)": 0.011167 + }, + { + "epoch": 0.7645394530022006, + "grad_norm": 1.465338110923767, + "learning_rate": 0.00017789781280091557, + "logits/chosen": -0.6118184328079224, + "logits/rejected": -0.8406589031219482, + "logps/chosen": -3.1348047256469727, + "logps/rejected": -79.62229919433594, + "loss": 0.39135175943374634, + "memory(GiB)": 46.14, + "nll_loss": 0.2855513095855713, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07680860161781311, + "rewards/margins": 6.500816345214844, + "rewards/rejected": -6.424009323120117, + "step": 304, + "train_speed(iter/s)": 0.011167 + }, + { + "epoch": 0.767054385413392, + "grad_norm": 2.457659959793091, + "learning_rate": 0.0001777233357512142, + "logits/chosen": -0.5555245280265808, + "logits/rejected": -0.7983438372612, + "logps/chosen": -5.093034267425537, + "logps/rejected": -92.61869812011719, + "loss": 0.4847222566604614, + "memory(GiB)": 46.14, + "nll_loss": 0.422935426235199, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07793700695037842, + "rewards/margins": 7.504522800445557, + "rewards/rejected": -7.426586151123047, + "step": 305, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.7695693178245835, + "grad_norm": 1.6642144918441772, + "learning_rate": 0.00017754825901305813, + "logits/chosen": -0.5577365159988403, + "logits/rejected": -0.6896043419837952, + "logps/chosen": -9.644157409667969, + "logps/rejected": -58.60102462768555, + "loss": 0.6450366377830505, + "memory(GiB)": 46.14, + "nll_loss": 0.4952359199523926, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2571660280227661, + "rewards/margins": 4.676360130310059, + "rewards/rejected": -4.419194221496582, + "step": 306, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.7720842502357749, + "grad_norm": 1.6156450510025024, + "learning_rate": 0.00017737258393728364, + "logits/chosen": -0.44825202226638794, + "logits/rejected": -0.7190597057342529, + "logps/chosen": -3.3316762447357178, + "logps/rejected": -79.71031188964844, + "loss": 0.3546125590801239, + "memory(GiB)": 46.14, + "nll_loss": 0.25914254784584045, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1541021317243576, + "rewards/margins": 5.745656967163086, + "rewards/rejected": -5.591554641723633, + "step": 307, + "train_speed(iter/s)": 0.011168 + }, + { + "epoch": 0.7745991826469664, + "grad_norm": 1.6676106452941895, + "learning_rate": 0.00017719631187934352, + "logits/chosen": -0.48865610361099243, + "logits/rejected": -0.6960019469261169, + "logps/chosen": -5.700809478759766, + "logps/rejected": -73.01457214355469, + "loss": 0.581813395023346, + "memory(GiB)": 46.14, + "nll_loss": 0.4443373382091522, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.01655534654855728, + "rewards/margins": 5.344724655151367, + "rewards/rejected": -5.328168869018555, + "step": 308, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.7771141150581579, + "grad_norm": 7.093632698059082, + "learning_rate": 0.00017701944419929672, + "logits/chosen": -0.5656027793884277, + "logits/rejected": -0.744474470615387, + "logps/chosen": -5.30854606628418, + "logps/rejected": -66.55186462402344, + "loss": 0.5189223289489746, + "memory(GiB)": 46.14, + "nll_loss": 0.3126901686191559, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.012391818687319756, + "rewards/margins": 4.805446624755859, + "rewards/rejected": -4.793054580688477, + "step": 309, + "train_speed(iter/s)": 0.011169 + }, + { + "epoch": 0.7796290474693492, + "grad_norm": 5.8190016746521, + "learning_rate": 0.00017684198226179784, + "logits/chosen": -0.5459531545639038, + "logits/rejected": -0.7102005481719971, + "logps/chosen": -5.777270317077637, + "logps/rejected": -73.569091796875, + "loss": 0.3673727810382843, + "memory(GiB)": 46.14, + "nll_loss": 0.2682158648967743, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.005160713568329811, + "rewards/margins": 5.717585563659668, + "rewards/rejected": -5.722745895385742, + "step": 310, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.7821439798805407, + "grad_norm": 1.8922319412231445, + "learning_rate": 0.00017666392743608656, + "logits/chosen": -0.594789445400238, + "logits/rejected": -0.6599074602127075, + "logps/chosen": -10.497507095336914, + "logps/rejected": -49.53678894042969, + "loss": 0.6126048564910889, + "memory(GiB)": 46.14, + "nll_loss": 0.4225066006183624, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2926208972930908, + "rewards/margins": 3.908644676208496, + "rewards/rejected": -3.6160240173339844, + "step": 311, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.7846589122917321, + "grad_norm": 3.8677096366882324, + "learning_rate": 0.00017648528109597705, + "logits/chosen": -0.5457702279090881, + "logits/rejected": -0.6197797060012817, + "logps/chosen": -6.788382053375244, + "logps/rejected": -42.95294189453125, + "loss": 0.6690109372138977, + "memory(GiB)": 46.14, + "nll_loss": 0.40374916791915894, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.044411513954401016, + "rewards/margins": 3.1033527851104736, + "rewards/rejected": -3.05894136428833, + "step": 312, + "train_speed(iter/s)": 0.01117 + }, + { + "epoch": 0.7871738447029236, + "grad_norm": 1.0135570764541626, + "learning_rate": 0.00017630604461984746, + "logits/chosen": -0.5280042886734009, + "logits/rejected": -0.7244092226028442, + "logps/chosen": -4.014311790466309, + "logps/rejected": -68.12934875488281, + "loss": 0.3465278744697571, + "memory(GiB)": 46.14, + "nll_loss": 0.27361059188842773, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14259541034698486, + "rewards/margins": 5.263729095458984, + "rewards/rejected": -5.121134281158447, + "step": 313, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.789688777114115, + "grad_norm": 4.7270283699035645, + "learning_rate": 0.00017612621939062928, + "logits/chosen": -0.5396519303321838, + "logits/rejected": -0.7206518650054932, + "logps/chosen": -5.555117130279541, + "logps/rejected": -73.57891082763672, + "loss": 0.6602690815925598, + "memory(GiB)": 46.14, + "nll_loss": 0.5206518173217773, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.0014735832810401917, + "rewards/margins": 5.42769193649292, + "rewards/rejected": -5.429165840148926, + "step": 314, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.7922037095253065, + "grad_norm": 1.428287386894226, + "learning_rate": 0.00017594580679579656, + "logits/chosen": -0.4950787127017975, + "logits/rejected": -0.7107822299003601, + "logps/chosen": -3.5974302291870117, + "logps/rejected": -74.10302734375, + "loss": 0.4180299937725067, + "memory(GiB)": 46.14, + "nll_loss": 0.2984779179096222, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.08340106159448624, + "rewards/margins": 5.900142192840576, + "rewards/rejected": -5.8167405128479, + "step": 315, + "train_speed(iter/s)": 0.011171 + }, + { + "epoch": 0.794718641936498, + "grad_norm": 1.647938847541809, + "learning_rate": 0.00017576480822735532, + "logits/chosen": -0.5975894927978516, + "logits/rejected": -0.7487577199935913, + "logps/chosen": -6.398176670074463, + "logps/rejected": -77.47513580322266, + "loss": 0.494728684425354, + "memory(GiB)": 46.14, + "nll_loss": 0.4415830969810486, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16486969590187073, + "rewards/margins": 6.55712890625, + "rewards/rejected": -6.39225959777832, + "step": 316, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.7972335743476894, + "grad_norm": 3.1385247707366943, + "learning_rate": 0.00017558322508183275, + "logits/chosen": -0.5611414909362793, + "logits/rejected": -0.7583786249160767, + "logps/chosen": -10.28195571899414, + "logps/rejected": -80.05876159667969, + "loss": 0.5074371695518494, + "memory(GiB)": 46.14, + "nll_loss": 0.3455233871936798, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.034254290163517, + "rewards/margins": 6.353381156921387, + "rewards/rejected": -6.387635707855225, + "step": 317, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.7997485067588809, + "grad_norm": 4.7420196533203125, + "learning_rate": 0.00017540105876026648, + "logits/chosen": -0.5814342498779297, + "logits/rejected": -0.7620936036109924, + "logps/chosen": -5.382750034332275, + "logps/rejected": -78.29422760009766, + "loss": 0.5510153770446777, + "memory(GiB)": 46.14, + "nll_loss": 0.40605586767196655, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0036928970366716385, + "rewards/margins": 6.385010719299316, + "rewards/rejected": -6.381317615509033, + "step": 318, + "train_speed(iter/s)": 0.011172 + }, + { + "epoch": 0.8022634391700723, + "grad_norm": 8.547826766967773, + "learning_rate": 0.00017521831066819363, + "logits/chosen": -0.6748584508895874, + "logits/rejected": -0.751319944858551, + "logps/chosen": -10.142738342285156, + "logps/rejected": -62.902137756347656, + "loss": 0.7446877956390381, + "memory(GiB)": 46.14, + "nll_loss": 0.4347481429576874, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.3501091003417969, + "rewards/margins": 4.718851089477539, + "rewards/rejected": -5.068960189819336, + "step": 319, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.8047783715812638, + "grad_norm": 2.8475148677825928, + "learning_rate": 0.00017503498221564025, + "logits/chosen": -0.5278587937355042, + "logits/rejected": -0.7719522714614868, + "logps/chosen": -5.870185375213623, + "logps/rejected": -83.85944366455078, + "loss": 0.661263644695282, + "memory(GiB)": 46.74, + "nll_loss": 0.46991419792175293, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10902933776378632, + "rewards/margins": 6.237672328948975, + "rewards/rejected": -6.128643035888672, + "step": 320, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.8072933039924552, + "grad_norm": 2.305772542953491, + "learning_rate": 0.00017485107481711012, + "logits/chosen": -0.5994943976402283, + "logits/rejected": -0.7459429502487183, + "logps/chosen": -5.2525315284729, + "logps/rejected": -73.49571228027344, + "loss": 0.4045717120170593, + "memory(GiB)": 46.74, + "nll_loss": 0.28694650530815125, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.041447557508945465, + "rewards/margins": 6.024381637573242, + "rewards/rejected": -5.98293399810791, + "step": 321, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.8098082364036466, + "grad_norm": 1.735187292098999, + "learning_rate": 0.00017466658989157406, + "logits/chosen": -0.5249663591384888, + "logits/rejected": -0.7108433842658997, + "logps/chosen": -4.704793453216553, + "logps/rejected": -72.97203063964844, + "loss": 0.4518013298511505, + "memory(GiB)": 46.74, + "nll_loss": 0.3283430337905884, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07784826308488846, + "rewards/margins": 5.81508731842041, + "rewards/rejected": -5.737238883972168, + "step": 322, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.8123231688148381, + "grad_norm": 0.9543433785438538, + "learning_rate": 0.00017448152886245888, + "logits/chosen": -0.5041825771331787, + "logits/rejected": -0.6829778552055359, + "logps/chosen": -4.729351997375488, + "logps/rejected": -76.31644439697266, + "loss": 0.3558841347694397, + "memory(GiB)": 46.74, + "nll_loss": 0.281302273273468, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.249185249209404, + "rewards/margins": 6.1249003410339355, + "rewards/rejected": -5.875715255737305, + "step": 323, + "train_speed(iter/s)": 0.011173 + }, + { + "epoch": 0.8148381012260295, + "grad_norm": 5.0823750495910645, + "learning_rate": 0.00017429589315763638, + "logits/chosen": -0.4357600510120392, + "logits/rejected": -0.700524091720581, + "logps/chosen": -3.877192258834839, + "logps/rejected": -68.90542602539062, + "loss": 0.5542052984237671, + "memory(GiB)": 46.74, + "nll_loss": 0.41005879640579224, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.02913202904164791, + "rewards/margins": 5.158620357513428, + "rewards/rejected": -5.129488945007324, + "step": 324, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.817353033637221, + "grad_norm": 1.3050035238265991, + "learning_rate": 0.00017410968420941244, + "logits/chosen": -0.3421289622783661, + "logits/rejected": -0.6473272442817688, + "logps/chosen": -5.135888576507568, + "logps/rejected": -74.2640151977539, + "loss": 0.48740285634994507, + "memory(GiB)": 46.74, + "nll_loss": 0.38718196749687195, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.24520042538642883, + "rewards/margins": 5.387969493865967, + "rewards/rejected": -5.1427693367004395, + "step": 325, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.8198679660484125, + "grad_norm": 11.92521858215332, + "learning_rate": 0.00017392290345451592, + "logits/chosen": -0.38754820823669434, + "logits/rejected": -0.607338547706604, + "logps/chosen": -6.677229404449463, + "logps/rejected": -71.38823699951172, + "loss": 0.5474317073822021, + "memory(GiB)": 46.74, + "nll_loss": 0.40050551295280457, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.13461363315582275, + "rewards/margins": 5.393355846405029, + "rewards/rejected": -5.258741855621338, + "step": 326, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.8223828984596039, + "grad_norm": 1.5031628608703613, + "learning_rate": 0.0001737355523340875, + "logits/chosen": -0.5126341581344604, + "logits/rejected": -0.6609553694725037, + "logps/chosen": -3.9447288513183594, + "logps/rejected": -60.175750732421875, + "loss": 0.4619714617729187, + "memory(GiB)": 46.74, + "nll_loss": 0.35617318749427795, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17950919270515442, + "rewards/margins": 4.923368453979492, + "rewards/rejected": -4.743858814239502, + "step": 327, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.8248978308707954, + "grad_norm": 1.3256229162216187, + "learning_rate": 0.0001735476322936686, + "logits/chosen": -0.3694300651550293, + "logits/rejected": -0.6536751389503479, + "logps/chosen": -4.688364505767822, + "logps/rejected": -76.19749450683594, + "loss": 0.4076545834541321, + "memory(GiB)": 46.74, + "nll_loss": 0.325284868478775, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1420704424381256, + "rewards/margins": 5.678350448608398, + "rewards/rejected": -5.536280155181885, + "step": 328, + "train_speed(iter/s)": 0.011174 + }, + { + "epoch": 0.8274127632819868, + "grad_norm": 1.3037030696868896, + "learning_rate": 0.00017335914478319028, + "logits/chosen": -0.41062769293785095, + "logits/rejected": -0.6642740368843079, + "logps/chosen": -5.453380584716797, + "logps/rejected": -74.66622924804688, + "loss": 0.44521886110305786, + "memory(GiB)": 46.74, + "nll_loss": 0.3608228862285614, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.11086025834083557, + "rewards/margins": 5.619003772735596, + "rewards/rejected": -5.508144378662109, + "step": 329, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.8299276956931783, + "grad_norm": 1.4347755908966064, + "learning_rate": 0.00017317009125696208, + "logits/chosen": -0.4672892689704895, + "logits/rejected": -0.6743369698524475, + "logps/chosen": -4.6292266845703125, + "logps/rejected": -73.22108459472656, + "loss": 0.2870956063270569, + "memory(GiB)": 46.74, + "nll_loss": 0.23536254465579987, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.18421661853790283, + "rewards/margins": 6.022341728210449, + "rewards/rejected": -5.838125228881836, + "step": 330, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.8324426281043696, + "grad_norm": 2.6831681728363037, + "learning_rate": 0.00017298047317366061, + "logits/chosen": -0.46991387009620667, + "logits/rejected": -0.6954001784324646, + "logps/chosen": -4.487226963043213, + "logps/rejected": -70.13626861572266, + "loss": 0.5644213557243347, + "memory(GiB)": 46.74, + "nll_loss": 0.4404352903366089, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0725957453250885, + "rewards/margins": 5.4586405754089355, + "rewards/rejected": -5.386044502258301, + "step": 331, + "train_speed(iter/s)": 0.011175 + }, + { + "epoch": 0.8349575605155611, + "grad_norm": 0.9778565764427185, + "learning_rate": 0.00017279029199631857, + "logits/chosen": -0.40650704503059387, + "logits/rejected": -0.6795352697372437, + "logps/chosen": -2.2717714309692383, + "logps/rejected": -77.46300506591797, + "loss": 0.29844945669174194, + "memory(GiB)": 46.74, + "nll_loss": 0.1882498562335968, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16533461213111877, + "rewards/margins": 6.410762310028076, + "rewards/rejected": -6.24542760848999, + "step": 332, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.8374724929267526, + "grad_norm": 2.472064971923828, + "learning_rate": 0.0001725995491923131, + "logits/chosen": -0.4610061049461365, + "logits/rejected": -0.6507254838943481, + "logps/chosen": -11.14199447631836, + "logps/rejected": -77.41361999511719, + "loss": 0.7604877352714539, + "memory(GiB)": 46.74, + "nll_loss": 0.5936329960823059, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.011591670103371143, + "rewards/margins": 6.023037433624268, + "rewards/rejected": -6.034628868103027, + "step": 333, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.839987425337944, + "grad_norm": 4.791397571563721, + "learning_rate": 0.0001724082462333549, + "logits/chosen": -0.4945090115070343, + "logits/rejected": -0.6786508560180664, + "logps/chosen": -4.583111763000488, + "logps/rejected": -70.58711242675781, + "loss": 0.4426124095916748, + "memory(GiB)": 46.74, + "nll_loss": 0.3457377851009369, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.06782704591751099, + "rewards/margins": 5.740070819854736, + "rewards/rejected": -5.672244071960449, + "step": 334, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.8425023577491355, + "grad_norm": 7.941857814788818, + "learning_rate": 0.00017221638459547655, + "logits/chosen": -0.5508913397789001, + "logits/rejected": -0.7254266142845154, + "logps/chosen": -5.333986759185791, + "logps/rejected": -66.42805480957031, + "loss": 0.6611946225166321, + "memory(GiB)": 46.74, + "nll_loss": 0.5266276001930237, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.258667916059494, + "rewards/margins": 5.6560516357421875, + "rewards/rejected": -5.397383689880371, + "step": 335, + "train_speed(iter/s)": 0.011176 + }, + { + "epoch": 0.8450172901603269, + "grad_norm": 1.9054502248764038, + "learning_rate": 0.00017202396575902116, + "logits/chosen": -0.5157940983772278, + "logits/rejected": -0.662640392780304, + "logps/chosen": -9.51154899597168, + "logps/rejected": -77.71415710449219, + "loss": 0.6165091395378113, + "memory(GiB)": 46.74, + "nll_loss": 0.498354434967041, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.054273076355457306, + "rewards/margins": 5.92840576171875, + "rewards/rejected": -5.98267936706543, + "step": 336, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8475322225715184, + "grad_norm": 5.0429582595825195, + "learning_rate": 0.00017183099120863116, + "logits/chosen": -0.4925355017185211, + "logits/rejected": -0.7224594950675964, + "logps/chosen": -3.7566463947296143, + "logps/rejected": -82.92442321777344, + "loss": 0.37845250964164734, + "memory(GiB)": 46.74, + "nll_loss": 0.27013376355171204, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.047135189175605774, + "rewards/margins": 6.651348114013672, + "rewards/rejected": -6.604212760925293, + "step": 337, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8500471549827099, + "grad_norm": 1.3209309577941895, + "learning_rate": 0.0001716374624332365, + "logits/chosen": -0.43844541907310486, + "logits/rejected": -0.7345193028450012, + "logps/chosen": -3.03592848777771, + "logps/rejected": -88.54313659667969, + "loss": 0.3141745626926422, + "memory(GiB)": 46.74, + "nll_loss": 0.24039344489574432, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.189487487077713, + "rewards/margins": 6.713014602661133, + "rewards/rejected": -6.523527145385742, + "step": 338, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8525620873939013, + "grad_norm": 3.733077049255371, + "learning_rate": 0.0001714433809260435, + "logits/chosen": -0.541033923625946, + "logits/rejected": -0.6785414814949036, + "logps/chosen": -13.735635757446289, + "logps/rejected": -80.87117767333984, + "loss": 0.6797083616256714, + "memory(GiB)": 46.74, + "nll_loss": 0.6222865581512451, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15373513102531433, + "rewards/margins": 6.620373249053955, + "rewards/rejected": -6.466638088226318, + "step": 339, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8550770198050928, + "grad_norm": 4.67393159866333, + "learning_rate": 0.0001712487481845231, + "logits/chosen": -0.3521523177623749, + "logits/rejected": -0.6670207977294922, + "logps/chosen": -6.3595123291015625, + "logps/rejected": -96.03783416748047, + "loss": 0.3294995427131653, + "memory(GiB)": 46.74, + "nll_loss": 0.22724169492721558, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.025615770369768143, + "rewards/margins": 7.826894760131836, + "rewards/rejected": -7.801278591156006, + "step": 340, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8575919522162841, + "grad_norm": 0.8247130513191223, + "learning_rate": 0.00017105356571039945, + "logits/chosen": -0.4443299174308777, + "logits/rejected": -0.7191017866134644, + "logps/chosen": -7.833669662475586, + "logps/rejected": -86.58798217773438, + "loss": 0.42601728439331055, + "memory(GiB)": 46.74, + "nll_loss": 0.3831000328063965, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07022079825401306, + "rewards/margins": 6.889365196228027, + "rewards/rejected": -6.8191447257995605, + "step": 341, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8601068846274756, + "grad_norm": 0.8711926341056824, + "learning_rate": 0.00017085783500963824, + "logits/chosen": -0.41609716415405273, + "logits/rejected": -0.7125498652458191, + "logps/chosen": -3.1712937355041504, + "logps/rejected": -77.74504089355469, + "loss": 0.21924850344657898, + "memory(GiB)": 46.74, + "nll_loss": 0.1784050464630127, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20674246549606323, + "rewards/margins": 6.209396839141846, + "rewards/rejected": -6.002654552459717, + "step": 342, + "train_speed(iter/s)": 0.011177 + }, + { + "epoch": 0.8626218170386671, + "grad_norm": 5.732574939727783, + "learning_rate": 0.00017066155759243507, + "logits/chosen": -0.5378013849258423, + "logits/rejected": -0.7252504825592041, + "logps/chosen": -5.074343204498291, + "logps/rejected": -75.13893127441406, + "loss": 0.37449705600738525, + "memory(GiB)": 46.74, + "nll_loss": 0.28582653403282166, + "rewards/accuracies": 0.96875, + "rewards/chosen": -0.06149884685873985, + "rewards/margins": 6.032390117645264, + "rewards/rejected": -6.093888759613037, + "step": 343, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.8651367494498585, + "grad_norm": 1.0474615097045898, + "learning_rate": 0.00017046473497320385, + "logits/chosen": -0.5175920724868774, + "logits/rejected": -0.7067053914070129, + "logps/chosen": -4.754598140716553, + "logps/rejected": -72.19132232666016, + "loss": 0.398180216550827, + "memory(GiB)": 46.74, + "nll_loss": 0.3198007643222809, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.03717122972011566, + "rewards/margins": 5.711915969848633, + "rewards/rejected": -5.674744129180908, + "step": 344, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.86765168186105, + "grad_norm": 1.3896440267562866, + "learning_rate": 0.0001702673686705651, + "logits/chosen": -0.5148175954818726, + "logits/rejected": -0.7209337949752808, + "logps/chosen": -3.864173650741577, + "logps/rejected": -69.10660552978516, + "loss": 0.27238431572914124, + "memory(GiB)": 46.74, + "nll_loss": 0.19631227850914001, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.0761566311120987, + "rewards/margins": 5.610742092132568, + "rewards/rejected": -5.534584999084473, + "step": 345, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.8701666142722414, + "grad_norm": 1.14860999584198, + "learning_rate": 0.00017006946020733425, + "logits/chosen": -0.45313283801078796, + "logits/rejected": -0.6407146453857422, + "logps/chosen": -4.239717483520508, + "logps/rejected": -68.09969329833984, + "loss": 0.29144588112831116, + "memory(GiB)": 46.74, + "nll_loss": 0.22455181181430817, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.2668611407279968, + "rewards/margins": 5.458950042724609, + "rewards/rejected": -5.192089080810547, + "step": 346, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.8726815466834329, + "grad_norm": 1.542136788368225, + "learning_rate": 0.00016987101111050982, + "logits/chosen": -0.48639586567878723, + "logits/rejected": -0.6774500608444214, + "logps/chosen": -6.492711067199707, + "logps/rejected": -60.656402587890625, + "loss": 0.4538443088531494, + "memory(GiB)": 46.74, + "nll_loss": 0.34202519059181213, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.27481359243392944, + "rewards/margins": 4.575342655181885, + "rewards/rejected": -4.3005290031433105, + "step": 347, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.8751964790946243, + "grad_norm": 2.576685667037964, + "learning_rate": 0.00016967202291126173, + "logits/chosen": -0.42330560088157654, + "logits/rejected": -0.7439920902252197, + "logps/chosen": -3.36045241355896, + "logps/rejected": -69.37139129638672, + "loss": 0.34409311413764954, + "memory(GiB)": 46.74, + "nll_loss": 0.26893264055252075, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2507910132408142, + "rewards/margins": 4.905345916748047, + "rewards/rejected": -4.654554843902588, + "step": 348, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.8777114115058158, + "grad_norm": 1.1091530323028564, + "learning_rate": 0.00016947249714491938, + "logits/chosen": -0.4069375991821289, + "logits/rejected": -0.7425306439399719, + "logps/chosen": -3.862551689147949, + "logps/rejected": -80.9285659790039, + "loss": 0.36630380153656006, + "memory(GiB)": 46.74, + "nll_loss": 0.2950875759124756, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15249909460544586, + "rewards/margins": 5.997171401977539, + "rewards/rejected": -5.844672679901123, + "step": 349, + "train_speed(iter/s)": 0.011178 + }, + { + "epoch": 0.8802263439170073, + "grad_norm": 3.5830211639404297, + "learning_rate": 0.00016927243535095997, + "logits/chosen": -0.5977684259414673, + "logits/rejected": -0.7476451992988586, + "logps/chosen": -5.81241512298584, + "logps/rejected": -62.173587799072266, + "loss": 0.6108379364013672, + "memory(GiB)": 46.74, + "nll_loss": 0.4931432604789734, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.10844524204730988, + "rewards/margins": 5.111351013183594, + "rewards/rejected": -5.00290584564209, + "step": 350, + "train_speed(iter/s)": 0.011179 + } + ], + "logging_steps": 1, + "max_steps": 1191, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 50, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9184784764829696e+17, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}