| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 2.0, |
| "eval_steps": 500, |
| "global_step": 68, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.03018867924528302, |
| "grad_norm": 138.31220610930498, |
| "learning_rate": 0.0, |
| "logits/chosen": -2.109619140625, |
| "logits/rejected": -2.166259765625, |
| "logps/chosen": -515.4921875, |
| "logps/rejected": -403.890625, |
| "loss": 0.69140625, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.06037735849056604, |
| "grad_norm": 103.0810019302269, |
| "learning_rate": 7.142857142857142e-08, |
| "logits/chosen": -2.19482421875, |
| "logits/rejected": -2.22802734375, |
| "logps/chosen": -248.8828125, |
| "logps/rejected": -282.2421875, |
| "loss": 0.69140625, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "epoch": 0.09056603773584905, |
| "grad_norm": 110.9946117237372, |
| "learning_rate": 1.4285714285714285e-07, |
| "logits/chosen": -2.11962890625, |
| "logits/rejected": -2.154052734375, |
| "logps/chosen": -385.7265625, |
| "logps/rejected": -336.3046875, |
| "loss": 0.66015625, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": 0.044574737548828125, |
| "rewards/margins": 0.07429122924804688, |
| "rewards/rejected": -0.02971649169921875, |
| "step": 3 |
| }, |
| { |
| "epoch": 0.12075471698113208, |
| "grad_norm": 134.1044963548161, |
| "learning_rate": 2.1428571428571426e-07, |
| "logits/chosen": -2.161865234375, |
| "logits/rejected": -2.15185546875, |
| "logps/chosen": -416.0, |
| "logps/rejected": -410.796875, |
| "loss": 0.7122802734375, |
| "rewards/accuracies": 0.28125, |
| "rewards/chosen": -0.02658843994140625, |
| "rewards/margins": -0.01718902587890625, |
| "rewards/rejected": -0.0093841552734375, |
| "step": 4 |
| }, |
| { |
| "epoch": 0.1509433962264151, |
| "grad_norm": 181.29875886515373, |
| "learning_rate": 2.857142857142857e-07, |
| "logits/chosen": -2.147216796875, |
| "logits/rejected": -2.2158203125, |
| "logps/chosen": -337.8984375, |
| "logps/rejected": -268.15625, |
| "loss": 0.694091796875, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.01094818115234375, |
| "rewards/margins": -0.00392913818359375, |
| "rewards/rejected": -0.007038116455078125, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.1811320754716981, |
| "grad_norm": 127.48792819761363, |
| "learning_rate": 3.5714285714285716e-07, |
| "logits/chosen": -2.21826171875, |
| "logits/rejected": -2.229248046875, |
| "logps/chosen": -425.4140625, |
| "logps/rejected": -476.859375, |
| "loss": 0.672119140625, |
| "rewards/accuracies": 0.40625, |
| "rewards/chosen": 0.019550323486328125, |
| "rewards/margins": 0.055568695068359375, |
| "rewards/rejected": -0.03597259521484375, |
| "step": 6 |
| }, |
| { |
| "epoch": 0.21132075471698114, |
| "grad_norm": 104.85395093542431, |
| "learning_rate": 4.285714285714285e-07, |
| "logits/chosen": -2.1513671875, |
| "logits/rejected": -2.2177734375, |
| "logps/chosen": -324.6328125, |
| "logps/rejected": -291.40625, |
| "loss": 0.690185546875, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.00390625, |
| "rewards/margins": 0.00390625, |
| "rewards/rejected": -0.00782012939453125, |
| "step": 7 |
| }, |
| { |
| "epoch": 0.24150943396226415, |
| "grad_norm": 110.67144826119416, |
| "learning_rate": 5e-07, |
| "logits/chosen": -2.196533203125, |
| "logits/rejected": -2.22265625, |
| "logps/chosen": -309.921875, |
| "logps/rejected": -344.0, |
| "loss": 0.6737060546875, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": 0.043010711669921875, |
| "rewards/margins": 0.04144287109375, |
| "rewards/rejected": 0.00156402587890625, |
| "step": 8 |
| }, |
| { |
| "epoch": 0.27169811320754716, |
| "grad_norm": 113.76857363403872, |
| "learning_rate": 4.996685224712076e-07, |
| "logits/chosen": -2.184326171875, |
| "logits/rejected": -2.20166015625, |
| "logps/chosen": -269.8671875, |
| "logps/rejected": -311.5625, |
| "loss": 0.6883544921875, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.0031280517578125, |
| "rewards/margins": 0.01567840576171875, |
| "rewards/rejected": -0.018768310546875, |
| "step": 9 |
| }, |
| { |
| "epoch": 0.3018867924528302, |
| "grad_norm": 116.528380644011, |
| "learning_rate": 4.986749689036473e-07, |
| "logits/chosen": -2.19580078125, |
| "logits/rejected": -2.22314453125, |
| "logps/chosen": -328.390625, |
| "logps/rejected": -301.921875, |
| "loss": 0.666748046875, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.0203399658203125, |
| "rewards/margins": 0.05944061279296875, |
| "rewards/rejected": -0.03910064697265625, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.3320754716981132, |
| "grad_norm": 114.41976508331743, |
| "learning_rate": 4.970219740227693e-07, |
| "logits/chosen": -2.25244140625, |
| "logits/rejected": -2.267578125, |
| "logps/chosen": -230.1171875, |
| "logps/rejected": -271.609375, |
| "loss": 0.6549072265625, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.014858245849609375, |
| "rewards/margins": 0.08209609985351562, |
| "rewards/rejected": -0.06724929809570312, |
| "step": 11 |
| }, |
| { |
| "epoch": 0.3622641509433962, |
| "grad_norm": 127.78939520989171, |
| "learning_rate": 4.947139212738394e-07, |
| "logits/chosen": -2.19775390625, |
| "logits/rejected": -2.24169921875, |
| "logps/chosen": -368.9609375, |
| "logps/rejected": -386.296875, |
| "loss": 0.6553955078125, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.028934478759765625, |
| "rewards/margins": 0.08996200561523438, |
| "rewards/rejected": -0.06099700927734375, |
| "step": 12 |
| }, |
| { |
| "epoch": 0.39245283018867927, |
| "grad_norm": 160.43709922042387, |
| "learning_rate": 4.917569311978301e-07, |
| "logits/chosen": -2.16552734375, |
| "logits/rejected": -2.205322265625, |
| "logps/chosen": -325.640625, |
| "logps/rejected": -357.0859375, |
| "loss": 0.640380859375, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.03597259521484375, |
| "rewards/margins": 0.13688278198242188, |
| "rewards/rejected": -0.10087966918945312, |
| "step": 13 |
| }, |
| { |
| "epoch": 0.4226415094339623, |
| "grad_norm": 128.59812939912248, |
| "learning_rate": 4.881588452008456e-07, |
| "logits/chosen": -2.119140625, |
| "logits/rejected": -2.1953125, |
| "logps/chosen": -354.609375, |
| "logps/rejected": -295.546875, |
| "loss": 0.63433837890625, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.016429901123046875, |
| "rewards/margins": 0.13383102416992188, |
| "rewards/rejected": -0.11734771728515625, |
| "step": 14 |
| }, |
| { |
| "epoch": 0.4528301886792453, |
| "grad_norm": 120.56983987783293, |
| "learning_rate": 4.839292047601234e-07, |
| "logits/chosen": -2.1904296875, |
| "logits/rejected": -2.263916015625, |
| "logps/chosen": -415.4140625, |
| "logps/rejected": -337.859375, |
| "loss": 0.629638671875, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.0328369140625, |
| "rewards/margins": 0.1298675537109375, |
| "rewards/rejected": -0.16265869140625, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.4830188679245283, |
| "grad_norm": 120.33867564906456, |
| "learning_rate": 4.790792261217512e-07, |
| "logits/chosen": -2.14404296875, |
| "logits/rejected": -2.19482421875, |
| "logps/chosen": -369.359375, |
| "logps/rejected": -369.453125, |
| "loss": 0.635009765625, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.03910064697265625, |
| "rewards/margins": 0.14395904541015625, |
| "rewards/rejected": -0.183013916015625, |
| "step": 16 |
| }, |
| { |
| "epoch": 0.5132075471698113, |
| "grad_norm": 102.63014934519077, |
| "learning_rate": 4.7362177055719886e-07, |
| "logits/chosen": -2.176513671875, |
| "logits/rejected": -2.178466796875, |
| "logps/chosen": -244.3671875, |
| "logps/rejected": -294.7890625, |
| "loss": 0.60015869140625, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.0203399658203125, |
| "rewards/margins": 0.21128082275390625, |
| "rewards/rejected": -0.19091033935546875, |
| "step": 17 |
| }, |
| { |
| "epoch": 0.5433962264150943, |
| "grad_norm": 106.93923521151912, |
| "learning_rate": 4.675713102575388e-07, |
| "logits/chosen": -2.198974609375, |
| "logits/rejected": -2.231689453125, |
| "logps/chosen": -320.1171875, |
| "logps/rejected": -339.6953125, |
| "loss": 0.5753173828125, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.01172637939453125, |
| "rewards/margins": 0.2574005126953125, |
| "rewards/rejected": -0.24556732177734375, |
| "step": 18 |
| }, |
| { |
| "epoch": 0.5735849056603773, |
| "grad_norm": 96.40829818733613, |
| "learning_rate": 4.6094388995579634e-07, |
| "logits/chosen": -2.16357421875, |
| "logits/rejected": -2.264404296875, |
| "logps/chosen": -367.75, |
| "logps/rejected": -296.609375, |
| "loss": 0.61151123046875, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.09149932861328125, |
| "rewards/margins": 0.1924896240234375, |
| "rewards/rejected": -0.2839202880859375, |
| "step": 19 |
| }, |
| { |
| "epoch": 0.6037735849056604, |
| "grad_norm": 104.01103241458425, |
| "learning_rate": 4.537570843792028e-07, |
| "logits/chosen": -2.16064453125, |
| "logits/rejected": -2.170654296875, |
| "logps/chosen": -340.203125, |
| "logps/rejected": -415.078125, |
| "loss": 0.55609130859375, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.032871246337890625, |
| "rewards/margins": 0.328399658203125, |
| "rewards/rejected": -0.36133575439453125, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.6339622641509434, |
| "grad_norm": 102.48543340920472, |
| "learning_rate": 4.460299516441776e-07, |
| "logits/chosen": -2.183837890625, |
| "logits/rejected": -2.28662109375, |
| "logps/chosen": -474.2421875, |
| "logps/rejected": -356.5234375, |
| "loss": 0.54766845703125, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.0742950439453125, |
| "rewards/margins": 0.3504180908203125, |
| "rewards/rejected": -0.42476654052734375, |
| "step": 21 |
| }, |
| { |
| "epoch": 0.6641509433962264, |
| "grad_norm": 101.64915161645655, |
| "learning_rate": 4.3778298271762993e-07, |
| "logits/chosen": -2.17626953125, |
| "logits/rejected": -2.209716796875, |
| "logps/chosen": -303.9921875, |
| "logps/rejected": -357.921875, |
| "loss": 0.52899169921875, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.03277587890625, |
| "rewards/margins": 0.3879852294921875, |
| "rewards/rejected": -0.3552093505859375, |
| "step": 22 |
| }, |
| { |
| "epoch": 0.6943396226415094, |
| "grad_norm": 99.4155872940957, |
| "learning_rate": 4.290380470785983e-07, |
| "logits/chosen": -2.16943359375, |
| "logits/rejected": -2.249267578125, |
| "logps/chosen": -401.28125, |
| "logps/rejected": -417.7734375, |
| "loss": 0.48834228515625, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.004669189453125, |
| "rewards/margins": 0.5399398803710938, |
| "rewards/rejected": -0.5444259643554688, |
| "step": 23 |
| }, |
| { |
| "epoch": 0.7245283018867924, |
| "grad_norm": 90.52873672236639, |
| "learning_rate": 4.198183347243233e-07, |
| "logits/chosen": -2.23193359375, |
| "logits/rejected": -2.27392578125, |
| "logps/chosen": -373.828125, |
| "logps/rejected": -369.609375, |
| "loss": 0.47064208984375, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.044586181640625, |
| "rewards/margins": 0.5651931762695312, |
| "rewards/rejected": -0.6098709106445312, |
| "step": 24 |
| }, |
| { |
| "epoch": 0.7547169811320755, |
| "grad_norm": 89.70939620633418, |
| "learning_rate": 4.101482946745438e-07, |
| "logits/chosen": -2.24365234375, |
| "logits/rejected": -2.263916015625, |
| "logps/chosen": -296.6640625, |
| "logps/rejected": -297.609375, |
| "loss": 0.52313232421875, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.08766937255859375, |
| "rewards/margins": 0.43157958984375, |
| "rewards/rejected": -0.51934814453125, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.7849056603773585, |
| "grad_norm": 106.35938326021778, |
| "learning_rate": 4.000535701370921e-07, |
| "logits/chosen": -2.2880859375, |
| "logits/rejected": -2.29931640625, |
| "logps/chosen": -342.4609375, |
| "logps/rejected": -469.5625, |
| "loss": 0.4821319580078125, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.21507644653320312, |
| "rewards/margins": 0.6137771606445312, |
| "rewards/rejected": -0.8289947509765625, |
| "step": 26 |
| }, |
| { |
| "epoch": 0.8150943396226416, |
| "grad_norm": 89.47071292174518, |
| "learning_rate": 3.895609305067161e-07, |
| "logits/chosen": -2.19677734375, |
| "logits/rejected": -2.234375, |
| "logps/chosen": -310.375, |
| "logps/rejected": -317.515625, |
| "loss": 0.460784912109375, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.15332794189453125, |
| "rewards/margins": 0.607879638671875, |
| "rewards/rejected": -0.76116943359375, |
| "step": 27 |
| }, |
| { |
| "epoch": 0.8452830188679246, |
| "grad_norm": 82.71880874204284, |
| "learning_rate": 3.786982003774577e-07, |
| "logits/chosen": -2.19189453125, |
| "logits/rejected": -2.216064453125, |
| "logps/chosen": -238.6875, |
| "logps/rejected": -300.8359375, |
| "loss": 0.4767303466796875, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.2033233642578125, |
| "rewards/margins": 0.59539794921875, |
| "rewards/rejected": -0.7988433837890625, |
| "step": 28 |
| }, |
| { |
| "epoch": 0.8754716981132076, |
| "grad_norm": 95.70798094262803, |
| "learning_rate": 3.6749418575683e-07, |
| "logits/chosen": -2.20166015625, |
| "logits/rejected": -2.242431640625, |
| "logps/chosen": -298.8203125, |
| "logps/rejected": -324.796875, |
| "loss": 0.4431915283203125, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.1024017333984375, |
| "rewards/margins": 0.679779052734375, |
| "rewards/rejected": -0.7819290161132812, |
| "step": 29 |
| }, |
| { |
| "epoch": 0.9056603773584906, |
| "grad_norm": 99.73199372695603, |
| "learning_rate": 3.559785976774652e-07, |
| "logits/chosen": -2.2548828125, |
| "logits/rejected": -2.306640625, |
| "logps/chosen": -329.125, |
| "logps/rejected": -343.3515625, |
| "loss": 0.4751110076904297, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.26758575439453125, |
| "rewards/margins": 0.6523895263671875, |
| "rewards/rejected": -0.9197998046875, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.9358490566037736, |
| "grad_norm": 84.8861552760989, |
| "learning_rate": 3.4418197340879627e-07, |
| "logits/chosen": -2.24267578125, |
| "logits/rejected": -2.27783203125, |
| "logps/chosen": -388.859375, |
| "logps/rejected": -397.546875, |
| "loss": 0.3660297393798828, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.17905426025390625, |
| "rewards/margins": 1.132171630859375, |
| "rewards/rejected": -1.311431884765625, |
| "step": 31 |
| }, |
| { |
| "epoch": 0.9660377358490566, |
| "grad_norm": 84.17818925735007, |
| "learning_rate": 3.321355954777087e-07, |
| "logits/chosen": -2.186279296875, |
| "logits/rejected": -2.217041015625, |
| "logps/chosen": -303.9296875, |
| "logps/rejected": -323.9375, |
| "loss": 0.410308837890625, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.390289306640625, |
| "rewards/margins": 0.807708740234375, |
| "rewards/rejected": -1.1977081298828125, |
| "step": 32 |
| }, |
| { |
| "epoch": 0.9962264150943396, |
| "grad_norm": 89.71705544962349, |
| "learning_rate": 3.1987140871290236e-07, |
| "logits/chosen": -2.248779296875, |
| "logits/rejected": -2.27880859375, |
| "logps/chosen": -322.6640625, |
| "logps/rejected": -355.671875, |
| "loss": 0.4383659362792969, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.3761749267578125, |
| "rewards/margins": 0.867095947265625, |
| "rewards/rejected": -1.2427978515625, |
| "step": 33 |
| }, |
| { |
| "epoch": 1.0, |
| "grad_norm": 183.46893679118816, |
| "learning_rate": 3.074219355329489e-07, |
| "logits/chosen": -2.00390625, |
| "logits/rejected": -2.03515625, |
| "logps/chosen": -213.0, |
| "logps/rejected": -232.5, |
| "loss": 0.41015625, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.2001953125, |
| "rewards/margins": 0.72607421875, |
| "rewards/rejected": -0.92578125, |
| "step": 34 |
| }, |
| { |
| "epoch": 1.030188679245283, |
| "grad_norm": 53.96764181203498, |
| "learning_rate": 2.948201897026839e-07, |
| "logits/chosen": -2.30712890625, |
| "logits/rejected": -2.344970703125, |
| "logps/chosen": -225.59375, |
| "logps/rejected": -293.5, |
| "loss": 0.3011283874511719, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.03855133056640625, |
| "rewards/margins": 1.488525390625, |
| "rewards/rejected": -1.526702880859375, |
| "step": 35 |
| }, |
| { |
| "epoch": 1.060377358490566, |
| "grad_norm": 62.43795666242199, |
| "learning_rate": 2.8209958878663776e-07, |
| "logits/chosen": -2.279296875, |
| "logits/rejected": -2.30224609375, |
| "logps/chosen": -333.53125, |
| "logps/rejected": -399.6875, |
| "loss": 0.2984161376953125, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.2995033264160156, |
| "rewards/margins": 1.37225341796875, |
| "rewards/rejected": -1.67138671875, |
| "step": 36 |
| }, |
| { |
| "epoch": 1.090566037735849, |
| "grad_norm": 51.16866512137806, |
| "learning_rate": 2.692938655316616e-07, |
| "logits/chosen": -2.2197265625, |
| "logits/rejected": -2.24462890625, |
| "logps/chosen": -274.34375, |
| "logps/rejected": -341.8125, |
| "loss": 0.28313589096069336, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.3532257080078125, |
| "rewards/margins": 1.56646728515625, |
| "rewards/rejected": -1.920501708984375, |
| "step": 37 |
| }, |
| { |
| "epoch": 1.120754716981132, |
| "grad_norm": 66.72467012226434, |
| "learning_rate": 2.5643697841374715e-07, |
| "logits/chosen": -2.1904296875, |
| "logits/rejected": -2.17822265625, |
| "logps/chosen": -216.140625, |
| "logps/rejected": -349.375, |
| "loss": 0.2605581283569336, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.30327606201171875, |
| "rewards/margins": 1.6783447265625, |
| "rewards/rejected": -1.98095703125, |
| "step": 38 |
| }, |
| { |
| "epoch": 1.150943396226415, |
| "grad_norm": 62.738746790894204, |
| "learning_rate": 2.435630215862529e-07, |
| "logits/chosen": -2.218017578125, |
| "logits/rejected": -2.300048828125, |
| "logps/chosen": -327.2109375, |
| "logps/rejected": -246.84375, |
| "loss": 0.3275489807128906, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.29560089111328125, |
| "rewards/margins": 1.1746826171875, |
| "rewards/rejected": -1.4700088500976562, |
| "step": 39 |
| }, |
| { |
| "epoch": 1.181132075471698, |
| "grad_norm": 64.97314574211953, |
| "learning_rate": 2.307061344683384e-07, |
| "logits/chosen": -2.24853515625, |
| "logits/rejected": -2.289794921875, |
| "logps/chosen": -378.265625, |
| "logps/rejected": -409.421875, |
| "loss": 0.26163482666015625, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.325836181640625, |
| "rewards/margins": 1.73931884765625, |
| "rewards/rejected": -2.06549072265625, |
| "step": 40 |
| }, |
| { |
| "epoch": 1.211320754716981, |
| "grad_norm": 70.926273227677, |
| "learning_rate": 2.1790041121336222e-07, |
| "logits/chosen": -2.247314453125, |
| "logits/rejected": -2.322265625, |
| "logps/chosen": -495.953125, |
| "logps/rejected": -395.453125, |
| "loss": 0.3062589168548584, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.44409942626953125, |
| "rewards/margins": 1.86004638671875, |
| "rewards/rejected": -2.304534912109375, |
| "step": 41 |
| }, |
| { |
| "epoch": 1.241509433962264, |
| "grad_norm": 48.82537565235591, |
| "learning_rate": 2.0517981029731612e-07, |
| "logits/chosen": -2.271240234375, |
| "logits/rejected": -2.32568359375, |
| "logps/chosen": -385.1875, |
| "logps/rejected": -355.1875, |
| "loss": 0.22526264190673828, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.21823883056640625, |
| "rewards/margins": 1.83306884765625, |
| "rewards/rejected": -2.051666259765625, |
| "step": 42 |
| }, |
| { |
| "epoch": 1.271698113207547, |
| "grad_norm": 78.04521274122948, |
| "learning_rate": 1.9257806446705112e-07, |
| "logits/chosen": -2.318359375, |
| "logits/rejected": -2.340087890625, |
| "logps/chosen": -283.8671875, |
| "logps/rejected": -317.15625, |
| "loss": 0.27616119384765625, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.51422119140625, |
| "rewards/margins": 1.6151123046875, |
| "rewards/rejected": -2.1287841796875, |
| "step": 43 |
| }, |
| { |
| "epoch": 1.3018867924528301, |
| "grad_norm": 51.25580086600066, |
| "learning_rate": 1.8012859128709764e-07, |
| "logits/chosen": -2.290771484375, |
| "logits/rejected": -2.2978515625, |
| "logps/chosen": -425.6796875, |
| "logps/rejected": -484.390625, |
| "loss": 0.13787996768951416, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5703582763671875, |
| "rewards/margins": 2.62646484375, |
| "rewards/rejected": -3.19580078125, |
| "step": 44 |
| }, |
| { |
| "epoch": 1.3320754716981131, |
| "grad_norm": 58.73013327845229, |
| "learning_rate": 1.678644045222913e-07, |
| "logits/chosen": -2.2548828125, |
| "logits/rejected": -2.3232421875, |
| "logps/chosen": -425.2578125, |
| "logps/rejected": -417.0, |
| "loss": 0.1794424057006836, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3956565856933594, |
| "rewards/margins": 2.46533203125, |
| "rewards/rejected": -2.862060546875, |
| "step": 45 |
| }, |
| { |
| "epoch": 1.3622641509433961, |
| "grad_norm": 62.34773299199525, |
| "learning_rate": 1.5581802659120368e-07, |
| "logits/chosen": -2.2158203125, |
| "logits/rejected": -2.242431640625, |
| "logps/chosen": -410.671875, |
| "logps/rejected": -398.625, |
| "loss": 0.1958155632019043, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.283416748046875, |
| "rewards/margins": 2.351318359375, |
| "rewards/rejected": -2.6341323852539062, |
| "step": 46 |
| }, |
| { |
| "epoch": 1.3924528301886792, |
| "grad_norm": 55.39949009295557, |
| "learning_rate": 1.4402140232253484e-07, |
| "logits/chosen": -2.2275390625, |
| "logits/rejected": -2.26171875, |
| "logps/chosen": -332.5, |
| "logps/rejected": -322.046875, |
| "loss": 0.23157119750976562, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.635040283203125, |
| "rewards/margins": 1.67864990234375, |
| "rewards/rejected": -2.314208984375, |
| "step": 47 |
| }, |
| { |
| "epoch": 1.4226415094339622, |
| "grad_norm": 49.0523957870456, |
| "learning_rate": 1.3250581424317008e-07, |
| "logits/chosen": -2.234375, |
| "logits/rejected": -2.29443359375, |
| "logps/chosen": -404.1796875, |
| "logps/rejected": -373.6875, |
| "loss": 0.1951136589050293, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.49471282958984375, |
| "rewards/margins": 2.03558349609375, |
| "rewards/rejected": -2.53033447265625, |
| "step": 48 |
| }, |
| { |
| "epoch": 1.4528301886792452, |
| "grad_norm": 43.85948277562909, |
| "learning_rate": 1.213017996225424e-07, |
| "logits/chosen": -2.27734375, |
| "logits/rejected": -2.277099609375, |
| "logps/chosen": -396.1328125, |
| "logps/rejected": -428.21875, |
| "loss": 0.15965144336223602, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.4569244384765625, |
| "rewards/margins": 2.53271484375, |
| "rewards/rejected": -2.9884033203125, |
| "step": 49 |
| }, |
| { |
| "epoch": 1.4830188679245282, |
| "grad_norm": 67.30462143329488, |
| "learning_rate": 1.1043906949328385e-07, |
| "logits/chosen": -2.2587890625, |
| "logits/rejected": -2.29833984375, |
| "logps/chosen": -310.765625, |
| "logps/rejected": -351.59375, |
| "loss": 0.31104373931884766, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.7111129760742188, |
| "rewards/margins": 1.73919677734375, |
| "rewards/rejected": -2.45050048828125, |
| "step": 50 |
| }, |
| { |
| "epoch": 1.5132075471698112, |
| "grad_norm": 54.97036033515127, |
| "learning_rate": 9.994642986290797e-08, |
| "logits/chosen": -2.265625, |
| "logits/rejected": -2.28955078125, |
| "logps/chosen": -449.4609375, |
| "logps/rejected": -496.03125, |
| "loss": 0.1978926658630371, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.75494384765625, |
| "rewards/margins": 2.4698486328125, |
| "rewards/rejected": -3.2242431640625, |
| "step": 51 |
| }, |
| { |
| "epoch": 1.5433962264150942, |
| "grad_norm": 58.7410932105306, |
| "learning_rate": 8.985170532545622e-08, |
| "logits/chosen": -2.202880859375, |
| "logits/rejected": -2.2685546875, |
| "logps/chosen": -392.7734375, |
| "logps/rejected": -294.8671875, |
| "loss": 0.21066856384277344, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.3479804992675781, |
| "rewards/margins": 1.939697265625, |
| "rewards/rejected": -2.2872314453125, |
| "step": 52 |
| }, |
| { |
| "epoch": 1.5735849056603772, |
| "grad_norm": 70.52365689566419, |
| "learning_rate": 8.018166527567672e-08, |
| "logits/chosen": -2.240966796875, |
| "logits/rejected": -2.307373046875, |
| "logps/chosen": -342.15625, |
| "logps/rejected": -294.140625, |
| "loss": 0.29566192626953125, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.7627105712890625, |
| "rewards/margins": 1.51177978515625, |
| "rewards/rejected": -2.274658203125, |
| "step": 53 |
| }, |
| { |
| "epoch": 1.6037735849056602, |
| "grad_norm": 91.59109368101038, |
| "learning_rate": 7.096195292140172e-08, |
| "logits/chosen": -2.281494140625, |
| "logits/rejected": -2.3701171875, |
| "logps/chosen": -473.859375, |
| "logps/rejected": -352.1953125, |
| "loss": 0.30013465881347656, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.8317108154296875, |
| "rewards/margins": 1.802398681640625, |
| "rewards/rejected": -2.634796142578125, |
| "step": 54 |
| }, |
| { |
| "epoch": 1.6339622641509433, |
| "grad_norm": 57.582350815239245, |
| "learning_rate": 6.221701728237008e-08, |
| "logits/chosen": -2.204833984375, |
| "logits/rejected": -2.232666015625, |
| "logps/chosen": -314.7265625, |
| "logps/rejected": -399.859375, |
| "loss": 0.2029261589050293, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.4932861328125, |
| "rewards/margins": 2.4166259765625, |
| "rewards/rejected": -2.9102783203125, |
| "step": 55 |
| }, |
| { |
| "epoch": 1.6641509433962263, |
| "grad_norm": 45.97477082609319, |
| "learning_rate": 5.3970048355822416e-08, |
| "logits/chosen": -2.27880859375, |
| "logits/rejected": -2.33447265625, |
| "logps/chosen": -253.046875, |
| "logps/rejected": -313.453125, |
| "loss": 0.2117919921875, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.5623092651367188, |
| "rewards/margins": 1.8668212890625, |
| "rewards/rejected": -2.42919921875, |
| "step": 56 |
| }, |
| { |
| "epoch": 1.6943396226415093, |
| "grad_norm": 44.62211760281082, |
| "learning_rate": 4.6242915620797186e-08, |
| "logits/chosen": -2.330078125, |
| "logits/rejected": -2.33447265625, |
| "logps/chosen": -279.0859375, |
| "logps/rejected": -318.984375, |
| "loss": 0.21802997589111328, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.40540313720703125, |
| "rewards/margins": 2.044219970703125, |
| "rewards/rejected": -2.450164794921875, |
| "step": 57 |
| }, |
| { |
| "epoch": 1.7245283018867923, |
| "grad_norm": 51.2670325277809, |
| "learning_rate": 3.905611004420359e-08, |
| "logits/chosen": -2.32958984375, |
| "logits/rejected": -2.3330078125, |
| "logps/chosen": -352.4296875, |
| "logps/rejected": -507.390625, |
| "loss": 0.20676660537719727, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.807769775390625, |
| "rewards/margins": 2.173065185546875, |
| "rewards/rejected": -2.979888916015625, |
| "step": 58 |
| }, |
| { |
| "epoch": 1.7547169811320755, |
| "grad_norm": 44.04885960141364, |
| "learning_rate": 3.2428689742461187e-08, |
| "logits/chosen": -2.22802734375, |
| "logits/rejected": -2.23876953125, |
| "logps/chosen": -285.890625, |
| "logps/rejected": -427.265625, |
| "loss": 0.16101567447185516, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5487899780273438, |
| "rewards/margins": 2.62451171875, |
| "rewards/rejected": -3.1728515625, |
| "step": 59 |
| }, |
| { |
| "epoch": 1.7849056603773585, |
| "grad_norm": 43.187388393833025, |
| "learning_rate": 2.6378229442801157e-08, |
| "logits/chosen": -2.24658203125, |
| "logits/rejected": -2.252685546875, |
| "logps/chosen": -336.703125, |
| "logps/rejected": -402.09375, |
| "loss": 0.16387367248535156, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5001144409179688, |
| "rewards/margins": 2.2327880859375, |
| "rewards/rejected": -2.732666015625, |
| "step": 60 |
| }, |
| { |
| "epoch": 1.8150943396226416, |
| "grad_norm": 44.49559340775631, |
| "learning_rate": 2.0920773878248837e-08, |
| "logits/chosen": -2.23388671875, |
| "logits/rejected": -2.287353515625, |
| "logps/chosen": -433.109375, |
| "logps/rejected": -437.796875, |
| "loss": 0.17702722549438477, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5224227905273438, |
| "rewards/margins": 2.38623046875, |
| "rewards/rejected": -2.90966796875, |
| "step": 61 |
| }, |
| { |
| "epoch": 1.8452830188679246, |
| "grad_norm": 38.58001237378031, |
| "learning_rate": 1.6070795239876617e-08, |
| "logits/chosen": -2.1708984375, |
| "logits/rejected": -2.24072265625, |
| "logps/chosen": -330.2734375, |
| "logps/rejected": -324.390625, |
| "loss": 0.17849206924438477, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.43164825439453125, |
| "rewards/margins": 2.142822265625, |
| "rewards/rejected": -2.5733642578125, |
| "step": 62 |
| }, |
| { |
| "epoch": 1.8754716981132076, |
| "grad_norm": 47.091837159623296, |
| "learning_rate": 1.1841154799154373e-08, |
| "logits/chosen": -2.34326171875, |
| "logits/rejected": -2.378173828125, |
| "logps/chosen": -282.71875, |
| "logps/rejected": -280.9140625, |
| "loss": 0.20744895935058594, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.43671417236328125, |
| "rewards/margins": 1.98101806640625, |
| "rewards/rejected": -2.41754150390625, |
| "step": 63 |
| }, |
| { |
| "epoch": 1.9056603773584906, |
| "grad_norm": 51.17603737519097, |
| "learning_rate": 8.243068802169905e-09, |
| "logits/chosen": -2.2236328125, |
| "logits/rejected": -2.307861328125, |
| "logps/chosen": -394.0390625, |
| "logps/rejected": -326.078125, |
| "loss": 0.2276473045349121, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.34835052490234375, |
| "rewards/margins": 1.89508056640625, |
| "rewards/rejected": -2.2437744140625, |
| "step": 64 |
| }, |
| { |
| "epoch": 1.9358490566037736, |
| "grad_norm": 55.15415072058887, |
| "learning_rate": 5.286078726160548e-09, |
| "logits/chosen": -2.25244140625, |
| "logits/rejected": -2.32275390625, |
| "logps/chosen": -358.90625, |
| "logps/rejected": -276.796875, |
| "loss": 0.23507118225097656, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": -0.5408477783203125, |
| "rewards/margins": 1.8223876953125, |
| "rewards/rejected": -2.36279296875, |
| "step": 65 |
| }, |
| { |
| "epoch": 1.9660377358490566, |
| "grad_norm": 33.48144189144124, |
| "learning_rate": 2.978025977230736e-09, |
| "logits/chosen": -2.30712890625, |
| "logits/rejected": -2.3076171875, |
| "logps/chosen": -230.609375, |
| "logps/rejected": -380.734375, |
| "loss": 0.1398305892944336, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": -0.37613677978515625, |
| "rewards/margins": 2.4521484375, |
| "rewards/rejected": -2.828125, |
| "step": 66 |
| }, |
| { |
| "epoch": 1.9962264150943396, |
| "grad_norm": 64.2728983817751, |
| "learning_rate": 1.3250310963527355e-09, |
| "logits/chosen": -2.22412109375, |
| "logits/rejected": -2.228515625, |
| "logps/chosen": -281.109375, |
| "logps/rejected": -359.625, |
| "loss": 0.2590675354003906, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.548980712890625, |
| "rewards/margins": 1.7811279296875, |
| "rewards/rejected": -2.331298828125, |
| "step": 67 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 209.37960293088204, |
| "learning_rate": 3.314775287923677e-10, |
| "logits/chosen": -2.42578125, |
| "logits/rejected": -2.3203125, |
| "logps/chosen": -309.0, |
| "logps/rejected": -929.625, |
| "loss": 0.41407907009124756, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.4259033203125, |
| "rewards/margins": 2.6611328125, |
| "rewards/rejected": -3.081298828125, |
| "step": 68 |
| }, |
| { |
| "epoch": 2.0, |
| "step": 68, |
| "total_flos": 119628768952320.0, |
| "train_loss": 0.4046005795107168, |
| "train_runtime": 1834.6485, |
| "train_samples_per_second": 1.154, |
| "train_steps_per_second": 0.037 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 68, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 10, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 119628768952320.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
