| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 150.0, |
| "eval_steps": 500, |
| "global_step": 300, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 2.3850007222248957, |
| "epoch": 5.0, |
| "grad_norm": 5.218133926391602, |
| "learning_rate": 4.85e-06, |
| "logits/chosen": 0.5955334049959196, |
| "logits/rejected": 0.32116526582606625, |
| "logps/chosen": -41.78460889963003, |
| "logps/rejected": -46.04202065101037, |
| "loss": 0.5778924465179444, |
| "mean_token_accuracy": 0.48216897593094754, |
| "num_tokens": 3045.0, |
| "rewards/accuracies": 0.676923076923077, |
| "rewards/chosen": 1.8768298184642425, |
| "rewards/margins": 1.1801037300091524, |
| "rewards/rejected": 0.696726090231767, |
| "step": 10 |
| }, |
| { |
| "entropy": 2.314448336454538, |
| "epoch": 10.0, |
| "grad_norm": 2.3608832359313965, |
| "learning_rate": 4.683333333333334e-06, |
| "logits/chosen": 0.5266339006231595, |
| "logits/rejected": 0.21806011517835058, |
| "logps/chosen": -39.94469804030199, |
| "logps/rejected": -49.08440563495343, |
| "loss": 0.36816518306732177, |
| "mean_token_accuracy": 0.5069385063189726, |
| "num_tokens": 6090.0, |
| "rewards/accuracies": 0.9692307692307692, |
| "rewards/chosen": 2.0608209357811855, |
| "rewards/margins": 1.668333342900643, |
| "rewards/rejected": 0.3924875888579453, |
| "step": 20 |
| }, |
| { |
| "entropy": 2.2045343949244574, |
| "epoch": 15.0, |
| "grad_norm": 1.2208229303359985, |
| "learning_rate": 4.516666666666667e-06, |
| "logits/chosen": 0.42076279344647916, |
| "logits/rejected": 0.0795841027230604, |
| "logps/chosen": -38.261579630925105, |
| "logps/rejected": -52.489193872305066, |
| "loss": 0.25407335758209226, |
| "mean_token_accuracy": 0.5410144303853696, |
| "num_tokens": 9135.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.22913277057501, |
| "rewards/margins": 2.1771240174770354, |
| "rewards/rejected": 0.05200875710982543, |
| "step": 30 |
| }, |
| { |
| "entropy": 2.0643920261126296, |
| "epoch": 20.0, |
| "grad_norm": 1.1566054821014404, |
| "learning_rate": 4.350000000000001e-06, |
| "logits/chosen": 0.2660081956584437, |
| "logits/rejected": -0.11877416602934321, |
| "logps/chosen": -36.715302041860724, |
| "logps/rejected": -56.045902252197266, |
| "loss": 0.18806705474853516, |
| "mean_token_accuracy": 0.5574041231320455, |
| "num_tokens": 12180.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.3837605155431305, |
| "rewards/margins": 2.687422578609907, |
| "rewards/rejected": -0.3036620749733769, |
| "step": 40 |
| }, |
| { |
| "entropy": 1.9192953155590937, |
| "epoch": 25.0, |
| "grad_norm": 1.2787108421325684, |
| "learning_rate": 4.183333333333334e-06, |
| "logits/chosen": 0.06328228399595424, |
| "logits/rejected": -0.36550493553391344, |
| "logps/chosen": -35.36447745103102, |
| "logps/rejected": -59.87469277015099, |
| "loss": 0.1385887861251831, |
| "mean_token_accuracy": 0.5706025022726793, |
| "num_tokens": 15225.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.5188429571115054, |
| "rewards/margins": 3.205384102234474, |
| "rewards/rejected": -0.6865411404806834, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.7868539026150336, |
| "epoch": 30.0, |
| "grad_norm": 0.8686179518699646, |
| "learning_rate": 4.0166666666666675e-06, |
| "logits/chosen": -0.1935140321445184, |
| "logits/rejected": -0.6839043743711061, |
| "logps/chosen": -34.12935148385855, |
| "logps/rejected": -64.12931248591497, |
| "loss": 0.0951196312904358, |
| "mean_token_accuracy": 0.5941899987367484, |
| "num_tokens": 18270.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.64235560802313, |
| "rewards/margins": 3.754358713443463, |
| "rewards/rejected": -1.112003114934151, |
| "step": 60 |
| }, |
| { |
| "entropy": 1.6745153069496155, |
| "epoch": 35.0, |
| "grad_norm": 0.9338397979736328, |
| "learning_rate": 3.85e-06, |
| "logits/chosen": -0.3970043551674374, |
| "logits/rejected": -0.9072793044553101, |
| "logps/chosen": -33.13469376197228, |
| "logps/rejected": -67.47981532170222, |
| "loss": 0.06889303326606751, |
| "mean_token_accuracy": 0.6142469997589405, |
| "num_tokens": 21315.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.7418213651730463, |
| "rewards/margins": 4.188874772878793, |
| "rewards/rejected": -1.4470534209806758, |
| "step": 70 |
| }, |
| { |
| "entropy": 1.5280305674442878, |
| "epoch": 40.0, |
| "grad_norm": 0.33691903948783875, |
| "learning_rate": 3.6833333333333338e-06, |
| "logits/chosen": -0.645527713806289, |
| "logits/rejected": -1.192322842277305, |
| "logps/chosen": -32.71983912541316, |
| "logps/rejected": -71.29921434842623, |
| "loss": 0.05013889670372009, |
| "mean_token_accuracy": 0.6215091819946582, |
| "num_tokens": 24360.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.783306827911964, |
| "rewards/margins": 4.612300166716943, |
| "rewards/rejected": -1.8289933227289181, |
| "step": 80 |
| }, |
| { |
| "entropy": 1.4477157487319066, |
| "epoch": 45.0, |
| "grad_norm": 0.0781475231051445, |
| "learning_rate": 3.516666666666667e-06, |
| "logits/chosen": -0.8208894431606327, |
| "logits/rejected": -1.3862875261002638, |
| "logps/chosen": -32.24270979074331, |
| "logps/rejected": -74.3404776939979, |
| "loss": 0.037910228967666625, |
| "mean_token_accuracy": 0.6225866313164051, |
| "num_tokens": 27405.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.831019756427178, |
| "rewards/margins": 4.964139417501596, |
| "rewards/rejected": -2.133119664513148, |
| "step": 90 |
| }, |
| { |
| "entropy": 1.3916929373374352, |
| "epoch": 50.0, |
| "grad_norm": 0.1870872527360916, |
| "learning_rate": 3.3500000000000005e-06, |
| "logits/chosen": -0.9026928408855097, |
| "logits/rejected": -1.4932921680317246, |
| "logps/chosen": -31.6792912703294, |
| "logps/rejected": -76.95603843102089, |
| "loss": 0.029496246576309205, |
| "mean_token_accuracy": 0.6238097415520595, |
| "num_tokens": 30450.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.8873616236906785, |
| "rewards/margins": 5.282037371855516, |
| "rewards/rejected": -2.3946757453708694, |
| "step": 100 |
| }, |
| { |
| "entropy": 1.3489822713228372, |
| "epoch": 55.0, |
| "grad_norm": 0.15059450268745422, |
| "learning_rate": 3.183333333333334e-06, |
| "logits/chosen": -0.9584337553499183, |
| "logits/rejected": -1.5809670771118987, |
| "logps/chosen": -31.16159066420335, |
| "logps/rejected": -79.13991223848782, |
| "loss": 0.025989437103271486, |
| "mean_token_accuracy": 0.6313372121407436, |
| "num_tokens": 33495.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.9391316624788137, |
| "rewards/margins": 5.552194802577679, |
| "rewards/rejected": -2.6130631179190598, |
| "step": 110 |
| }, |
| { |
| "entropy": 1.3076109152573805, |
| "epoch": 60.0, |
| "grad_norm": 0.4038887619972229, |
| "learning_rate": 3.0166666666666673e-06, |
| "logits/chosen": -1.0143513353856966, |
| "logits/rejected": -1.6518468357354557, |
| "logps/chosen": -30.717048087486855, |
| "logps/rejected": -80.81175912710337, |
| "loss": 0.019894377887248994, |
| "mean_token_accuracy": 0.6305825824920948, |
| "num_tokens": 36540.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 2.983585899609786, |
| "rewards/margins": 5.763833695191604, |
| "rewards/rejected": -2.7802478168589566, |
| "step": 120 |
| }, |
| { |
| "entropy": 1.2777886221042045, |
| "epoch": 65.0, |
| "grad_norm": 0.11832690238952637, |
| "learning_rate": 2.85e-06, |
| "logits/chosen": -1.0513172948777008, |
| "logits/rejected": -1.70667886808449, |
| "logps/chosen": -30.33043086712177, |
| "logps/rejected": -82.26801863450271, |
| "loss": 0.017130233347415924, |
| "mean_token_accuracy": 0.6363077672628256, |
| "num_tokens": 39585.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.022247677582961, |
| "rewards/margins": 5.94812143949362, |
| "rewards/rejected": -2.9258737678711233, |
| "step": 130 |
| }, |
| { |
| "entropy": 1.250800774189142, |
| "epoch": 70.0, |
| "grad_norm": 0.07396625727415085, |
| "learning_rate": 2.683333333333333e-06, |
| "logits/chosen": -1.0829184006557993, |
| "logits/rejected": -1.7563719300860305, |
| "logps/chosen": -30.013790159959058, |
| "logps/rejected": -83.55414663461538, |
| "loss": 0.014281480014324189, |
| "mean_token_accuracy": 0.6392337904526637, |
| "num_tokens": 42630.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.053911742797265, |
| "rewards/margins": 6.1083983182907104, |
| "rewards/rejected": -3.054486560019163, |
| "step": 140 |
| }, |
| { |
| "entropy": 1.223134689606153, |
| "epoch": 75.0, |
| "grad_norm": 0.05753239244222641, |
| "learning_rate": 2.5166666666666666e-06, |
| "logits/chosen": -1.1195608680355456, |
| "logits/rejected": -1.8052698192809788, |
| "logps/chosen": -29.635238119272085, |
| "logps/rejected": -84.57547466571515, |
| "loss": 0.012452676892280579, |
| "mean_token_accuracy": 0.6391755163669586, |
| "num_tokens": 45675.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.091766904867612, |
| "rewards/margins": 6.248386273017297, |
| "rewards/rejected": -3.1566193484343015, |
| "step": 150 |
| }, |
| { |
| "entropy": 1.2027296034189372, |
| "epoch": 80.0, |
| "grad_norm": 0.3149576485157013, |
| "learning_rate": 2.35e-06, |
| "logits/chosen": -1.1491697674350145, |
| "logits/rejected": -1.8513586526299164, |
| "logps/chosen": -29.390566752507137, |
| "logps/rejected": -85.54401256854717, |
| "loss": 0.010251734405755997, |
| "mean_token_accuracy": 0.6281787042434399, |
| "num_tokens": 48720.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1162340833590583, |
| "rewards/margins": 6.369707246927114, |
| "rewards/rejected": -3.253473144081923, |
| "step": 160 |
| }, |
| { |
| "entropy": 1.1893031638402205, |
| "epoch": 85.0, |
| "grad_norm": 0.27565744519233704, |
| "learning_rate": 2.1833333333333333e-06, |
| "logits/chosen": -1.1672777094357023, |
| "logits/rejected": -1.8822302021351611, |
| "logps/chosen": -29.235072796161358, |
| "logps/rejected": -86.43060050377478, |
| "loss": 0.010401710867881775, |
| "mean_token_accuracy": 0.6251530615183023, |
| "num_tokens": 51765.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1317834468988273, |
| "rewards/margins": 6.473915430215689, |
| "rewards/rejected": -3.3421319489295667, |
| "step": 170 |
| }, |
| { |
| "entropy": 1.1756579087330745, |
| "epoch": 90.0, |
| "grad_norm": 0.06536174565553665, |
| "learning_rate": 2.0166666666666667e-06, |
| "logits/chosen": -1.1910031234123657, |
| "logits/rejected": -1.9204785576885457, |
| "logps/chosen": -28.8819152538593, |
| "logps/rejected": -87.1160526569073, |
| "loss": 0.007509545981884002, |
| "mean_token_accuracy": 0.6323185668541835, |
| "num_tokens": 54810.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.167099202596224, |
| "rewards/margins": 6.577776369681725, |
| "rewards/rejected": -3.410677168002495, |
| "step": 180 |
| }, |
| { |
| "entropy": 1.1582091863338764, |
| "epoch": 95.0, |
| "grad_norm": 0.07627511024475098, |
| "learning_rate": 1.85e-06, |
| "logits/chosen": -1.2080450987055713, |
| "logits/rejected": -1.953039308066732, |
| "logps/chosen": -28.834403258103592, |
| "logps/rejected": -87.93590979942908, |
| "loss": 0.006308925151824951, |
| "mean_token_accuracy": 0.6208858558764825, |
| "num_tokens": 57855.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.17185040804056, |
| "rewards/margins": 6.6645133348611685, |
| "rewards/rejected": -3.4926629029787506, |
| "step": 190 |
| }, |
| { |
| "entropy": 1.1551778793334961, |
| "epoch": 100.0, |
| "grad_norm": 0.05331544205546379, |
| "learning_rate": 1.6833333333333335e-06, |
| "logits/chosen": -1.220794750164101, |
| "logits/rejected": -1.981571992290679, |
| "logps/chosen": -28.619847635122447, |
| "logps/rejected": -88.4375508235051, |
| "loss": 0.0059698235243558885, |
| "mean_token_accuracy": 0.6226402415679051, |
| "num_tokens": 60900.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.1933059848271883, |
| "rewards/margins": 6.736132926207322, |
| "rewards/rejected": -3.542826987229861, |
| "step": 200 |
| }, |
| { |
| "entropy": 1.1426296073656816, |
| "epoch": 105.0, |
| "grad_norm": 0.05480801314115524, |
| "learning_rate": 1.5166666666666668e-06, |
| "logits/chosen": -1.231853422725045, |
| "logits/rejected": -2.0020896033092983, |
| "logps/chosen": -28.58630635188176, |
| "logps/rejected": -89.03529874361479, |
| "loss": 0.005416269600391388, |
| "mean_token_accuracy": 0.6206193428773147, |
| "num_tokens": 63945.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.196660072986896, |
| "rewards/margins": 6.799261859747079, |
| "rewards/rejected": -3.6026018018905934, |
| "step": 210 |
| }, |
| { |
| "entropy": 1.1356208879214067, |
| "epoch": 110.0, |
| "grad_norm": 0.1870356947183609, |
| "learning_rate": 1.3500000000000002e-06, |
| "logits/chosen": -1.242897494818351, |
| "logits/rejected": -2.017118331043808, |
| "logps/chosen": -28.472275719275842, |
| "logps/rejected": -89.66436227651742, |
| "loss": 0.0053675461560487745, |
| "mean_token_accuracy": 0.6187149167060852, |
| "num_tokens": 66990.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.208063186131991, |
| "rewards/margins": 6.873571296838613, |
| "rewards/rejected": -3.665508143718426, |
| "step": 220 |
| }, |
| { |
| "entropy": 1.1326277191822345, |
| "epoch": 115.0, |
| "grad_norm": 0.17812301218509674, |
| "learning_rate": 1.1833333333333334e-06, |
| "logits/chosen": -1.2533293434899542, |
| "logits/rejected": -2.039634911561449, |
| "logps/chosen": -28.25326312138484, |
| "logps/rejected": -89.78828330406776, |
| "loss": 0.0052538376301527025, |
| "mean_token_accuracy": 0.6225874048012954, |
| "num_tokens": 70035.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2299644268476047, |
| "rewards/margins": 6.907864669653086, |
| "rewards/rejected": -3.677900252892421, |
| "step": 230 |
| }, |
| { |
| "entropy": 1.1250401950799502, |
| "epoch": 120.0, |
| "grad_norm": 0.15003784000873566, |
| "learning_rate": 1.0166666666666667e-06, |
| "logits/chosen": -1.257569145451956, |
| "logits/rejected": -2.050265735691312, |
| "logps/chosen": -28.221469380305365, |
| "logps/rejected": -90.19944645808293, |
| "loss": 0.004942169412970543, |
| "mean_token_accuracy": 0.6183986746347867, |
| "num_tokens": 73080.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2331438321333663, |
| "rewards/margins": 6.95216036943289, |
| "rewards/rejected": -3.719016566643348, |
| "step": 240 |
| }, |
| { |
| "entropy": 1.1205400155140803, |
| "epoch": 125.0, |
| "grad_norm": 0.042884498834609985, |
| "learning_rate": 8.500000000000001e-07, |
| "logits/chosen": -1.2690723238716755, |
| "logits/rejected": -2.069106913013174, |
| "logps/chosen": -28.168808775681715, |
| "logps/rejected": -90.60292575542744, |
| "loss": 0.004609152302145958, |
| "mean_token_accuracy": 0.6232292890548706, |
| "num_tokens": 76125.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2384098566495454, |
| "rewards/margins": 6.9977743405562185, |
| "rewards/rejected": -3.7593644994955797, |
| "step": 250 |
| }, |
| { |
| "entropy": 1.1178178645097292, |
| "epoch": 130.0, |
| "grad_norm": 0.05744079872965813, |
| "learning_rate": 6.833333333333334e-07, |
| "logits/chosen": -1.2737910903590781, |
| "logits/rejected": -2.07357408100483, |
| "logps/chosen": -28.046851026094878, |
| "logps/rejected": -90.76857217641977, |
| "loss": 0.00424395315349102, |
| "mean_token_accuracy": 0.6194670512126043, |
| "num_tokens": 79170.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2506056253726667, |
| "rewards/margins": 7.0265347260695235, |
| "rewards/rejected": -3.7759291373766386, |
| "step": 260 |
| }, |
| { |
| "entropy": 1.1141802962009724, |
| "epoch": 135.0, |
| "grad_norm": 0.15264171361923218, |
| "learning_rate": 5.166666666666667e-07, |
| "logits/chosen": -1.2837046704770079, |
| "logits/rejected": -2.0861976738101147, |
| "logps/chosen": -28.00541998056265, |
| "logps/rejected": -90.9417841397799, |
| "loss": 0.004570071399211883, |
| "mean_token_accuracy": 0.6172198827450092, |
| "num_tokens": 82215.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.254748777242807, |
| "rewards/margins": 7.047999128928551, |
| "rewards/rejected": -3.793250336096837, |
| "step": 270 |
| }, |
| { |
| "entropy": 1.1152964940437904, |
| "epoch": 140.0, |
| "grad_norm": 0.0695829838514328, |
| "learning_rate": 3.5000000000000004e-07, |
| "logits/chosen": -1.2849921549972774, |
| "logits/rejected": -2.0898888408580025, |
| "logps/chosen": -28.04736768282377, |
| "logps/rejected": -91.21173999492939, |
| "loss": 0.0037465617060661316, |
| "mean_token_accuracy": 0.6219463788546049, |
| "num_tokens": 85260.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2505539692365204, |
| "rewards/margins": 7.070799944950984, |
| "rewards/rejected": -3.820245937200693, |
| "step": 280 |
| }, |
| { |
| "entropy": 1.1116300225257874, |
| "epoch": 145.0, |
| "grad_norm": 0.13623295724391937, |
| "learning_rate": 1.8333333333333336e-07, |
| "logits/chosen": -1.2863314429379384, |
| "logits/rejected": -2.0972242711900613, |
| "logps/chosen": -27.95127751277043, |
| "logps/rejected": -91.2013175377479, |
| "loss": 0.004024703800678253, |
| "mean_token_accuracy": 0.629658029629634, |
| "num_tokens": 88305.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2601629972457884, |
| "rewards/margins": 7.079366680291983, |
| "rewards/rejected": -3.819203683046194, |
| "step": 290 |
| }, |
| { |
| "entropy": 1.1074378174084885, |
| "epoch": 150.0, |
| "grad_norm": 0.05619078502058983, |
| "learning_rate": 1.666666666666667e-08, |
| "logits/chosen": -1.2918823478513337, |
| "logits/rejected": -2.1033061557173203, |
| "logps/chosen": -27.983433268620416, |
| "logps/rejected": -91.42072965181791, |
| "loss": 0.0036425717175006867, |
| "mean_token_accuracy": 0.6180295990063593, |
| "num_tokens": 91350.0, |
| "rewards/accuracies": 1.0, |
| "rewards/chosen": 3.2569474348655114, |
| "rewards/margins": 7.0980922919053295, |
| "rewards/rejected": -3.841144861624791, |
| "step": 300 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 300, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 150, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 839539236556800.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
