diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,3634 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.1254115065057219, + "eval_steps": 500, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.171857863664627, + "epoch": 0.0006270575325286095, + "grad_norm": 36.12955856323242, + "learning_rate": 0.0, + "logits/chosen": -16.618545689498976, + "logits/rejected": -19.895854701940685, + "logps/chosen": -618.4926280975342, + "logps/rejected": -544.1511726379395, + "loss": 1.0, + "mean_token_accuracy": 0.7207779660820961, + "num_tokens": 44742.0, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "entropy": 1.3859521597623825, + "epoch": 0.001254115065057219, + "grad_norm": 35.26237487792969, + "learning_rate": 1e-08, + "logits/chosen": -16.148485680429634, + "logits/rejected": -15.444903538915032, + "logps/chosen": -789.9899635314941, + "logps/rejected": -629.6869964599609, + "loss": 1.0, + "mean_token_accuracy": 0.6869945079088211, + "num_tokens": 96877.0, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 2 + }, + { + "entropy": 1.1979895681142807, + "epoch": 0.0018811725975858284, + "grad_norm": 41.65127944946289, + "learning_rate": 2e-08, + "logits/chosen": -18.29156918013904, + "logits/rejected": -19.65692519211939, + "logps/chosen": -1245.2266998291016, + "logps/rejected": -570.8414497375488, + "loss": 1.0063536167144775, + "mean_token_accuracy": 0.6914148852229118, + "num_tokens": 168920.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": -0.017706198908854276, + "rewards/margins": -0.025545110227540135, + "rewards/rejected": 0.007838911202270538, + "step": 3 + }, + { + "entropy": 1.1795232072472572, + "epoch": 0.002508230130114438, + "grad_norm": 44.819602966308594, + "learning_rate": 3e-08, + "logits/chosen": -20.724100083229605, + "logits/rejected": -19.704480662994772, + "logps/chosen": -1142.84916305542, + "logps/rejected": -649.1872062683105, + "loss": 1.0045382976531982, + "mean_token_accuracy": 0.7210699692368507, + "num_tokens": 243342.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.00683369068428874, + "rewards/margins": -0.01820890378439799, + "rewards/rejected": 0.025042592780664563, + "step": 4 + }, + { + "entropy": 1.2270928248763084, + "epoch": 0.0031352876626430477, + "grad_norm": 32.66579055786133, + "learning_rate": 4e-08, + "logits/chosen": -16.499294980475682, + "logits/rejected": -17.05921506779658, + "logps/chosen": -709.8800754547119, + "logps/rejected": -586.3846015930176, + "loss": 1.0000782012939453, + "mean_token_accuracy": 0.7035035043954849, + "num_tokens": 295956.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.004917382728308439, + "rewards/margins": -0.00028197653591632843, + "rewards/rejected": -0.004635405610315502, + "step": 5 + }, + { + "entropy": 0.8437139950692654, + "epoch": 0.003762345195171657, + "grad_norm": 35.4331169128418, + "learning_rate": 5e-08, + "logits/chosen": -21.37335490682356, + "logits/rejected": -21.437480680261856, + "logps/chosen": -284.04754638671875, + "logps/rejected": -445.4689540863037, + "loss": 0.9983432292938232, + "mean_token_accuracy": 0.8082250654697418, + "num_tokens": 342091.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.01554340252187103, + "rewards/margins": 0.0067013868829235435, + "rewards/rejected": 0.008842015580739826, + "step": 6 + }, + { + "entropy": 1.0479743406176567, + "epoch": 0.004389402727700266, + "grad_norm": 37.445518493652344, + "learning_rate": 6e-08, + "logits/chosen": -18.649493652023672, + "logits/rejected": -18.4066487416478, + "logps/chosen": -805.4484777450562, + "logps/rejected": -680.746976852417, + "loss": 0.9999387264251709, + "mean_token_accuracy": 0.7536595985293388, + "num_tokens": 402856.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.002710643340833485, + "rewards/margins": 0.00014570658095180988, + "rewards/rejected": 0.0025649368471931666, + "step": 7 + }, + { + "entropy": 0.9191301837563515, + "epoch": 0.005016460260228876, + "grad_norm": 38.56687927246094, + "learning_rate": 7e-08, + "logits/chosen": -23.125927299845493, + "logits/rejected": -18.183520901420447, + "logps/chosen": -924.5588388442993, + "logps/rejected": -373.86534118652344, + "loss": 0.9900327324867249, + "mean_token_accuracy": 0.7644752189517021, + "num_tokens": 466828.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.01720759950694628, + "rewards/margins": 0.04020594718167558, + "rewards/rejected": -0.02299834630684927, + "step": 8 + }, + { + "entropy": 0.9575163498520851, + "epoch": 0.005643517792757485, + "grad_norm": 40.65217590332031, + "learning_rate": 8e-08, + "logits/chosen": -21.412901522784708, + "logits/rejected": -21.05138834665733, + "logps/chosen": -816.7314529418945, + "logps/rejected": -663.6353340148926, + "loss": 1.0026217699050903, + "mean_token_accuracy": 0.771336242556572, + "num_tokens": 527777.0, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.005182235909160227, + "rewards/margins": -0.010497396113350987, + "rewards/rejected": 0.005315160844475031, + "step": 9 + }, + { + "entropy": 1.193418636918068, + "epoch": 0.006270575325286095, + "grad_norm": 42.18134689331055, + "learning_rate": 9e-08, + "logits/chosen": -20.675864147312936, + "logits/rejected": -20.81446933923923, + "logps/chosen": -905.0363216400146, + "logps/rejected": -529.3571662902832, + "loss": 1.0143247842788696, + "mean_token_accuracy": 0.7211106270551682, + "num_tokens": 582379.0, + "rewards/accuracies": 0.34375, + "rewards/chosen": -0.04024720628513023, + "rewards/margins": -0.05793563392944634, + "rewards/rejected": 0.01768842909950763, + "step": 10 + }, + { + "entropy": 1.0013729706406593, + "epoch": 0.006897632857814705, + "grad_norm": 26.49334144592285, + "learning_rate": 1e-07, + "logits/chosen": -18.441954576827186, + "logits/rejected": -20.784630543895993, + "logps/chosen": -532.7153358459473, + "logps/rejected": -424.26265144348145, + "loss": 0.9981816411018372, + "mean_token_accuracy": 0.7596741318702698, + "num_tokens": 629715.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.00034768966725096107, + "rewards/margins": 0.007268631597980857, + "rewards/rejected": -0.006920939544215798, + "step": 11 + }, + { + "entropy": 1.0685330666601658, + "epoch": 0.007524690390343314, + "grad_norm": 34.10297393798828, + "learning_rate": 1.0999999999999999e-07, + "logits/chosen": -20.029515190554566, + "logits/rejected": -21.2224080516646, + "logps/chosen": -728.1428518295288, + "logps/rejected": -420.13951873779297, + "loss": 0.9948853850364685, + "mean_token_accuracy": 0.7521882280707359, + "num_tokens": 675406.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.010362100991187617, + "rewards/margins": 0.020601681084372103, + "rewards/rejected": -0.030963782221078873, + "step": 12 + }, + { + "entropy": 1.0431296452879906, + "epoch": 0.008151747922871924, + "grad_norm": 26.776187896728516, + "learning_rate": 1.2e-07, + "logits/chosen": -19.538335123973816, + "logits/rejected": -20.38955248873754, + "logps/chosen": -445.5426654815674, + "logps/rejected": -416.24525260925293, + "loss": 0.997660756111145, + "mean_token_accuracy": 0.7694718763232231, + "num_tokens": 722623.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.010003616625908762, + "rewards/margins": 0.009433707047719508, + "rewards/rejected": 0.0005699098983313888, + "step": 13 + }, + { + "entropy": 1.328450158238411, + "epoch": 0.008778805455400532, + "grad_norm": 40.44959259033203, + "learning_rate": 1.3e-07, + "logits/chosen": -16.50173976340228, + "logits/rejected": -18.966777530094504, + "logps/chosen": -1086.8657112121582, + "logps/rejected": -677.222972869873, + "loss": 0.9905858039855957, + "mean_token_accuracy": 0.6911511719226837, + "num_tokens": 786297.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.018621409428305924, + "rewards/margins": 0.03770919982343912, + "rewards/rejected": -0.01908779090445023, + "step": 14 + }, + { + "entropy": 1.0233074873685837, + "epoch": 0.009405862987929142, + "grad_norm": 28.252763748168945, + "learning_rate": 1.4e-07, + "logits/chosen": -17.86138401016042, + "logits/rejected": -19.831209346381634, + "logps/chosen": -449.9010829925537, + "logps/rejected": -492.7903289794922, + "loss": 1.0015244483947754, + "mean_token_accuracy": 0.7536996155977249, + "num_tokens": 837006.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.0008059862302616239, + "rewards/margins": -0.006120585021562874, + "rewards/rejected": 0.006926572299562395, + "step": 15 + }, + { + "entropy": 1.1438564285635948, + "epoch": 0.010032920520457752, + "grad_norm": 39.57615280151367, + "learning_rate": 1.5e-07, + "logits/chosen": -17.861874499690074, + "logits/rejected": -19.590980097470435, + "logps/chosen": -791.6395130157471, + "logps/rejected": -567.6202964782715, + "loss": 0.9942368268966675, + "mean_token_accuracy": 0.7436006516218185, + "num_tokens": 896266.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.008153757720720023, + "rewards/margins": 0.023181513650342822, + "rewards/rejected": -0.015027755754999816, + "step": 16 + }, + { + "entropy": 1.1797396391630173, + "epoch": 0.010659978052986362, + "grad_norm": 31.603225708007812, + "learning_rate": 1.6e-07, + "logits/chosen": -16.27227861739119, + "logits/rejected": -20.263959422316383, + "logps/chosen": -687.8001708984375, + "logps/rejected": -614.8188095092773, + "loss": 1.00355064868927, + "mean_token_accuracy": 0.7303128838539124, + "num_tokens": 962374.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.0015672160079702735, + "rewards/margins": -0.014266991434851661, + "rewards/rejected": 0.012699775630608201, + "step": 17 + }, + { + "entropy": 0.9821090400218964, + "epoch": 0.01128703558551497, + "grad_norm": 30.358339309692383, + "learning_rate": 1.7000000000000001e-07, + "logits/chosen": -19.800772516773982, + "logits/rejected": -17.959288041851217, + "logps/chosen": -607.3202495574951, + "logps/rejected": -534.6925010681152, + "loss": 1.002077579498291, + "mean_token_accuracy": 0.7688523679971695, + "num_tokens": 1016007.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.0275348040740937, + "rewards/margins": -0.008413461735472083, + "rewards/rejected": -0.019121342920698225, + "step": 18 + }, + { + "entropy": 1.2196400687098503, + "epoch": 0.01191409311804358, + "grad_norm": 37.30662536621094, + "learning_rate": 1.8e-07, + "logits/chosen": -17.72504789763709, + "logits/rejected": -16.12664834863506, + "logps/chosen": -813.8953742980957, + "logps/rejected": -505.71807956695557, + "loss": 1.004115343093872, + "mean_token_accuracy": 0.7162602841854095, + "num_tokens": 1080373.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.006188086466863751, + "rewards/margins": -0.016581419855356216, + "rewards/rejected": 0.010393334203399718, + "step": 19 + }, + { + "entropy": 1.1595325469970703, + "epoch": 0.01254115065057219, + "grad_norm": 44.13706970214844, + "learning_rate": 1.8999999999999998e-07, + "logits/chosen": -19.441370010137216, + "logits/rejected": -19.42652252828183, + "logps/chosen": -1272.5718240737915, + "logps/rejected": -622.7399673461914, + "loss": 1.0018830299377441, + "mean_token_accuracy": 0.7498048022389412, + "num_tokens": 1154517.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.02443491853773594, + "rewards/margins": -0.007660747622139752, + "rewards/rejected": -0.016774169402197003, + "step": 20 + }, + { + "entropy": 1.1010279133915901, + "epoch": 0.013168208183100799, + "grad_norm": 41.25276184082031, + "learning_rate": 2e-07, + "logits/chosen": -21.419456943034934, + "logits/rejected": -18.181673739978688, + "logps/chosen": -728.3854866027832, + "logps/rejected": -525.0471820831299, + "loss": 0.9977896213531494, + "mean_token_accuracy": 0.7623501494526863, + "num_tokens": 1212935.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.010096669750055298, + "rewards/margins": 0.008904347196221352, + "rewards/rejected": 0.0011923184501938522, + "step": 21 + }, + { + "entropy": 1.028150089085102, + "epoch": 0.01379526571562941, + "grad_norm": 36.785099029541016, + "learning_rate": 2.0999999999999997e-07, + "logits/chosen": -21.92485367743462, + "logits/rejected": -18.499413813157435, + "logps/chosen": -951.3307018280029, + "logps/rejected": -426.83284759521484, + "loss": 0.9958165884017944, + "mean_token_accuracy": 0.7534473612904549, + "num_tokens": 1281702.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.011519648949615657, + "rewards/margins": 0.01635956938844174, + "rewards/rejected": -0.027879221714101732, + "step": 22 + }, + { + "entropy": 0.9120204672217369, + "epoch": 0.01442232324815802, + "grad_norm": 36.060447692871094, + "learning_rate": 2.1999999999999998e-07, + "logits/chosen": -20.064261738727723, + "logits/rejected": -21.198644490481797, + "logps/chosen": -380.3253688812256, + "logps/rejected": -429.5263252258301, + "loss": 0.9947471022605896, + "mean_token_accuracy": 0.7963218614459038, + "num_tokens": 1327486.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.007161529851146042, + "rewards/margins": 0.02124298777198419, + "rewards/rejected": -0.014081457338761538, + "step": 23 + }, + { + "entropy": 1.282885067164898, + "epoch": 0.015049380780686628, + "grad_norm": 29.555295944213867, + "learning_rate": 2.3e-07, + "logits/chosen": -14.928810725754646, + "logits/rejected": -15.783012175933775, + "logps/chosen": -594.7181587219238, + "logps/rejected": -425.2836685180664, + "loss": 1.0085357427597046, + "mean_token_accuracy": 0.7083301916718483, + "num_tokens": 1377436.0, + "rewards/accuracies": 0.28125, + "rewards/chosen": -0.018809219647664577, + "rewards/margins": -0.03420144016854465, + "rewards/rejected": 0.015392220113426447, + "step": 24 + }, + { + "entropy": 1.170431300997734, + "epoch": 0.015676438313215236, + "grad_norm": 30.94621467590332, + "learning_rate": 2.4e-07, + "logits/chosen": -15.975754306334057, + "logits/rejected": -18.080566240707103, + "logps/chosen": -616.5587253570557, + "logps/rejected": -574.7409896850586, + "loss": 0.995916485786438, + "mean_token_accuracy": 0.7270784974098206, + "num_tokens": 1424961.0, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.010408571077277884, + "rewards/margins": 0.01637996477074921, + "rewards/rejected": -0.00597139319870621, + "step": 25 + }, + { + "entropy": 0.8303776457905769, + "epoch": 0.016303495845743848, + "grad_norm": 33.599998474121094, + "learning_rate": 2.5e-07, + "logits/chosen": -25.26339476179235, + "logits/rejected": -21.119193724011815, + "logps/chosen": -830.6931796073914, + "logps/rejected": -321.0673656463623, + "loss": 1.0016303062438965, + "mean_token_accuracy": 0.7916704788804054, + "num_tokens": 1479087.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.01703651063144207, + "rewards/margins": -0.007413207786157727, + "rewards/rejected": -0.009623300982639194, + "step": 26 + }, + { + "entropy": 0.991417720913887, + "epoch": 0.016930553378272456, + "grad_norm": 36.997718811035156, + "learning_rate": 2.6e-07, + "logits/chosen": -21.212152458499872, + "logits/rejected": -18.952785907097258, + "logps/chosen": -704.5350914001465, + "logps/rejected": -370.6160888671875, + "loss": 0.9924548864364624, + "mean_token_accuracy": 0.7522578835487366, + "num_tokens": 1545047.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": 0.017328572226688266, + "rewards/margins": 0.03090921661350876, + "rewards/rejected": -0.013580642873421311, + "step": 27 + }, + { + "entropy": 0.9894383996725082, + "epoch": 0.017557610910801064, + "grad_norm": 32.38890838623047, + "learning_rate": 2.7e-07, + "logits/chosen": -15.813927428167577, + "logits/rejected": -17.403900338000618, + "logps/chosen": -491.92512130737305, + "logps/rejected": -495.05503511428833, + "loss": 0.9995163083076477, + "mean_token_accuracy": 0.7497244253754616, + "num_tokens": 1594476.0, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.008761245291680098, + "rewards/margins": 0.001895940862596035, + "rewards/rejected": -0.010657186910975724, + "step": 28 + }, + { + "entropy": 0.9598497748374939, + "epoch": 0.018184668443329676, + "grad_norm": 37.41246795654297, + "learning_rate": 2.8e-07, + "logits/chosen": -19.56551149166531, + "logits/rejected": -19.280089414645445, + "logps/chosen": -857.0342273712158, + "logps/rejected": -602.1430015563965, + "loss": 0.9875982403755188, + "mean_token_accuracy": 0.7813946753740311, + "num_tokens": 1660065.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.013675417518243194, + "rewards/margins": 0.05004646920133382, + "rewards/rejected": -0.03637105316738598, + "step": 29 + }, + { + "entropy": 1.324970230460167, + "epoch": 0.018811725975858284, + "grad_norm": 36.4524040222168, + "learning_rate": 2.9e-07, + "logits/chosen": -21.331314482840543, + "logits/rejected": -18.526820149021674, + "logps/chosen": -1117.674301147461, + "logps/rejected": -583.0690956115723, + "loss": 1.0027996301651, + "mean_token_accuracy": 0.687925897538662, + "num_tokens": 1720616.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.020995987113565207, + "rewards/margins": -0.01269948878325522, + "rewards/rejected": -0.008296501007862389, + "step": 30 + }, + { + "entropy": 1.0734140500426292, + "epoch": 0.019438783508386896, + "grad_norm": 46.66982650756836, + "learning_rate": 3e-07, + "logits/chosen": -16.946001394868276, + "logits/rejected": -16.085758178871664, + "logps/chosen": -1352.3280754089355, + "logps/rejected": -446.2639093399048, + "loss": 0.9967788457870483, + "mean_token_accuracy": 0.7191508114337921, + "num_tokens": 1811101.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.0069078231463208795, + "rewards/margins": 0.012905016890726984, + "rewards/rejected": -0.005997193278744817, + "step": 31 + }, + { + "entropy": 1.323067456483841, + "epoch": 0.020065841040915505, + "grad_norm": 43.25249099731445, + "learning_rate": 3.1e-07, + "logits/chosen": -17.15731645945775, + "logits/rejected": -16.6517620046484, + "logps/chosen": -1334.5013446807861, + "logps/rejected": -645.0475730895996, + "loss": 0.9926663637161255, + "mean_token_accuracy": 0.7025224380195141, + "num_tokens": 1874585.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0028306127060204744, + "rewards/margins": 0.0292252313811332, + "rewards/rejected": -0.032055844203568995, + "step": 32 + }, + { + "entropy": 1.0061679631471634, + "epoch": 0.020692898573444113, + "grad_norm": 31.95186996459961, + "learning_rate": 3.2e-07, + "logits/chosen": -19.57589939389031, + "logits/rejected": -19.90894640965601, + "logps/chosen": -616.9296951293945, + "logps/rejected": -426.7467842102051, + "loss": 1.0076781511306763, + "mean_token_accuracy": 0.7674555331468582, + "num_tokens": 1923915.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": -0.03830282250419259, + "rewards/margins": -0.031945616006851196, + "rewards/rejected": -0.006357205333188176, + "step": 33 + }, + { + "entropy": 0.9171701222658157, + "epoch": 0.021319956105972725, + "grad_norm": 40.395572662353516, + "learning_rate": 3.3e-07, + "logits/chosen": -23.769768694096186, + "logits/rejected": -21.32001457302996, + "logps/chosen": -1274.3026218414307, + "logps/rejected": -430.8449001312256, + "loss": 0.991915225982666, + "mean_token_accuracy": 0.7600029706954956, + "num_tokens": 1992060.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.0150325192080345, + "rewards/margins": 0.03390131541527808, + "rewards/rejected": -0.01886879827361554, + "step": 34 + }, + { + "entropy": 1.1749595329165459, + "epoch": 0.021947013638501333, + "grad_norm": 44.95557403564453, + "learning_rate": 3.4000000000000003e-07, + "logits/chosen": -20.226573322321272, + "logits/rejected": -19.058830108404514, + "logps/chosen": -1253.8970890045166, + "logps/rejected": -554.8555345535278, + "loss": 1.0085124969482422, + "mean_token_accuracy": 0.7034121379256248, + "num_tokens": 2055149.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.031581409042701125, + "rewards/margins": -0.03442110400646925, + "rewards/rejected": 0.002839697408489883, + "step": 35 + }, + { + "entropy": 1.1094688102602959, + "epoch": 0.02257407117102994, + "grad_norm": 40.33211135864258, + "learning_rate": 3.5e-07, + "logits/chosen": -20.24939867823679, + "logits/rejected": -20.34034861023941, + "logps/chosen": -832.8481521606445, + "logps/rejected": -694.2218322753906, + "loss": 0.9992198944091797, + "mean_token_accuracy": 0.7405928075313568, + "num_tokens": 2115443.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.009844484797213227, + "rewards/margins": 0.0029849251732230186, + "rewards/rejected": -0.01282941095996648, + "step": 36 + }, + { + "entropy": 0.9377781078219414, + "epoch": 0.023201128703558553, + "grad_norm": 34.241668701171875, + "learning_rate": 3.6e-07, + "logits/chosen": -23.224412647259843, + "logits/rejected": -21.117019989134523, + "logps/chosen": -721.9840965270996, + "logps/rejected": -388.344877243042, + "loss": 0.99440598487854, + "mean_token_accuracy": 0.7658621892333031, + "num_tokens": 2172629.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.016015969216823578, + "rewards/margins": 0.022238188714254647, + "rewards/rejected": -0.0062222188571467996, + "step": 37 + }, + { + "entropy": 1.0614213570952415, + "epoch": 0.02382818623608716, + "grad_norm": 34.88137435913086, + "learning_rate": 3.7e-07, + "logits/chosen": -21.41548795553992, + "logits/rejected": -18.69351990419123, + "logps/chosen": -1103.8941230773926, + "logps/rejected": -619.8993282318115, + "loss": 1.011991024017334, + "mean_token_accuracy": 0.7375566810369492, + "num_tokens": 2243025.0, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.0650398419238627, + "rewards/margins": -0.04807211959268898, + "rewards/rejected": -0.016967719770036638, + "step": 38 + }, + { + "entropy": 1.2264064773917198, + "epoch": 0.02445524376861577, + "grad_norm": 32.48948669433594, + "learning_rate": 3.7999999999999996e-07, + "logits/chosen": -18.42769778338538, + "logits/rejected": -16.890675790644988, + "logps/chosen": -775.1542854309082, + "logps/rejected": -521.0977792739868, + "loss": 1.0028043985366821, + "mean_token_accuracy": 0.7075130566954613, + "num_tokens": 2294828.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.009019852615892887, + "rewards/margins": -0.011407412588596344, + "rewards/rejected": 0.002387559274211526, + "step": 39 + }, + { + "entropy": 1.2194309085607529, + "epoch": 0.02508230130114438, + "grad_norm": 35.85207748413086, + "learning_rate": 3.8999999999999997e-07, + "logits/chosen": -18.869462390156798, + "logits/rejected": -17.76926630663928, + "logps/chosen": -866.6771221160889, + "logps/rejected": -515.5025405883789, + "loss": 1.0078489780426025, + "mean_token_accuracy": 0.7282191216945648, + "num_tokens": 2359584.0, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.019920013728551567, + "rewards/margins": -0.03206599899567664, + "rewards/rejected": 0.012145984219387174, + "step": 40 + }, + { + "entropy": 0.9393897280097008, + "epoch": 0.02570935883367299, + "grad_norm": 29.508663177490234, + "learning_rate": 4e-07, + "logits/chosen": -19.542580737616923, + "logits/rejected": -17.785255602711995, + "logps/chosen": -440.89880752563477, + "logps/rejected": -531.4820680618286, + "loss": 0.9985625147819519, + "mean_token_accuracy": 0.7809053212404251, + "num_tokens": 2407189.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0036070493515580893, + "rewards/margins": 0.005761435255408287, + "rewards/rejected": -0.002154385729227215, + "step": 41 + }, + { + "entropy": 1.1858881711959839, + "epoch": 0.026336416366201598, + "grad_norm": 34.17562484741211, + "learning_rate": 4.0999999999999994e-07, + "logits/chosen": -20.121897634772164, + "logits/rejected": -19.622514667746742, + "logps/chosen": -804.0400094985962, + "logps/rejected": -586.3046684265137, + "loss": 0.9875714778900146, + "mean_token_accuracy": 0.7288667857646942, + "num_tokens": 2459432.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.0406644003232941, + "rewards/margins": 0.05225609429180622, + "rewards/rejected": -0.011591696529649198, + "step": 42 + }, + { + "entropy": 0.8192418366670609, + "epoch": 0.02696347389873021, + "grad_norm": 43.9438591003418, + "learning_rate": 4.1999999999999995e-07, + "logits/chosen": -24.092886788439017, + "logits/rejected": -23.664276651461066, + "logps/chosen": -584.1518545150757, + "logps/rejected": -503.76978302001953, + "loss": 0.9815411567687988, + "mean_token_accuracy": 0.8075317144393921, + "num_tokens": 2512702.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04232085426338017, + "rewards/margins": 0.07526338030584157, + "rewards/rejected": -0.032942528603598475, + "step": 43 + }, + { + "entropy": 0.9453827887773514, + "epoch": 0.02759053143125882, + "grad_norm": 35.92549514770508, + "learning_rate": 4.2999999999999996e-07, + "logits/chosen": -18.338003960845416, + "logits/rejected": -19.673824728737838, + "logps/chosen": -925.3223133087158, + "logps/rejected": -475.0994644165039, + "loss": 0.9924615025520325, + "mean_token_accuracy": 0.769691713154316, + "num_tokens": 2575249.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.02094321296317503, + "rewards/margins": 0.03026525373570621, + "rewards/rejected": -0.009322041762061417, + "step": 44 + }, + { + "entropy": 0.8736219257116318, + "epoch": 0.028217588963787427, + "grad_norm": 33.97110366821289, + "learning_rate": 4.3999999999999997e-07, + "logits/chosen": -22.332301111125155, + "logits/rejected": -20.832310241019115, + "logps/chosen": -462.5166606903076, + "logps/rejected": -383.00551986694336, + "loss": 1.0048575401306152, + "mean_token_accuracy": 0.7859590947628021, + "num_tokens": 2621631.0, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.009766561212018132, + "rewards/margins": -0.019437916460447013, + "rewards/rejected": 0.009671354899182916, + "step": 45 + }, + { + "entropy": 1.02370435744524, + "epoch": 0.02884464649631604, + "grad_norm": 31.75177764892578, + "learning_rate": 4.5e-07, + "logits/chosen": -18.283618627987572, + "logits/rejected": -18.815319615630727, + "logps/chosen": -466.81629753112793, + "logps/rejected": -454.2089099884033, + "loss": 0.9951180219650269, + "mean_token_accuracy": 0.7507254704833031, + "num_tokens": 2674748.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.001771716313669458, + "rewards/margins": 0.019629769725725055, + "rewards/rejected": -0.01785805242252536, + "step": 46 + }, + { + "entropy": 0.9373743236064911, + "epoch": 0.029471704028844647, + "grad_norm": 42.6140251159668, + "learning_rate": 4.6e-07, + "logits/chosen": -25.528625869209804, + "logits/rejected": -21.64687665278925, + "logps/chosen": -1166.7054986953735, + "logps/rejected": -438.6943473815918, + "loss": 0.9925483465194702, + "mean_token_accuracy": 0.759131945669651, + "num_tokens": 2744660.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.005512146046385169, + "rewards/margins": 0.03011180693283677, + "rewards/rejected": -0.024599659722298384, + "step": 47 + }, + { + "entropy": 0.9783306047320366, + "epoch": 0.030098761561373255, + "grad_norm": 35.963809967041016, + "learning_rate": 4.6999999999999995e-07, + "logits/chosen": -19.222043644762906, + "logits/rejected": -18.896907946229895, + "logps/chosen": -515.0348672866821, + "logps/rejected": -419.7729835510254, + "loss": 0.9998578429222107, + "mean_token_accuracy": 0.792612262070179, + "num_tokens": 2790068.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.02343193959677592, + "rewards/margins": 0.0005922880955040455, + "rewards/rejected": -0.024024227866902947, + "step": 48 + }, + { + "entropy": 1.10813407599926, + "epoch": 0.030725819093901867, + "grad_norm": 35.080543518066406, + "learning_rate": 4.8e-07, + "logits/chosen": -22.976597678281987, + "logits/rejected": -20.831361335827353, + "logps/chosen": -826.8215522766113, + "logps/rejected": -406.26672554016113, + "loss": 1.008866310119629, + "mean_token_accuracy": 0.7337941229343414, + "num_tokens": 2845743.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.02741124981548637, + "rewards/margins": -0.03900438791606575, + "rewards/rejected": 0.011593140079639852, + "step": 49 + }, + { + "entropy": 1.3011442199349403, + "epoch": 0.03135287662643047, + "grad_norm": 32.856292724609375, + "learning_rate": 4.9e-07, + "logits/chosen": -14.496447176077387, + "logits/rejected": -17.84699543352937, + "logps/chosen": -565.9537868499756, + "logps/rejected": -459.75712490081787, + "loss": 0.9952901005744934, + "mean_token_accuracy": 0.7119247242808342, + "num_tokens": 2899233.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.004313211014959961, + "rewards/margins": 0.018873692606575787, + "rewards/rejected": -0.014560480834916234, + "step": 50 + }, + { + "entropy": 1.2501383051276207, + "epoch": 0.031979934158959084, + "grad_norm": 36.4530143737793, + "learning_rate": 5e-07, + "logits/chosen": -21.17299309242602, + "logits/rejected": -22.865218106484676, + "logps/chosen": -866.4957427978516, + "logps/rejected": -508.9821949005127, + "loss": 1.0008279085159302, + "mean_token_accuracy": 0.7066351845860481, + "num_tokens": 2952150.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.0076497383415699005, + "rewards/margins": -0.003307956736534834, + "rewards/rejected": -0.004341780979302712, + "step": 51 + }, + { + "entropy": 1.1913169473409653, + "epoch": 0.032606991691487695, + "grad_norm": 40.79720687866211, + "learning_rate": 5.1e-07, + "logits/chosen": -17.331403938671592, + "logits/rejected": -16.946636435404315, + "logps/chosen": -760.1813850402832, + "logps/rejected": -703.0359497070312, + "loss": 1.0072510242462158, + "mean_token_accuracy": 0.7222515121102333, + "num_tokens": 3005503.0, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.013277458609081805, + "rewards/margins": -0.029311579186469316, + "rewards/rejected": 0.01603412051917985, + "step": 52 + }, + { + "entropy": 1.3226243034005165, + "epoch": 0.0332340492240163, + "grad_norm": 37.33218002319336, + "learning_rate": 5.2e-07, + "logits/chosen": -19.339108194716523, + "logits/rejected": -17.709144071414954, + "logps/chosen": -1036.9900550842285, + "logps/rejected": -671.3515205383301, + "loss": 1.0005102157592773, + "mean_token_accuracy": 0.699407085776329, + "num_tokens": 3062231.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.01915261917747557, + "rewards/margins": -0.002298903651535511, + "rewards/rejected": -0.016853714711032808, + "step": 53 + }, + { + "entropy": 1.2621377930045128, + "epoch": 0.03386110675654491, + "grad_norm": 36.86682891845703, + "learning_rate": 5.3e-07, + "logits/chosen": -23.098386869669465, + "logits/rejected": -18.559492736951306, + "logps/chosen": -879.0279788970947, + "logps/rejected": -472.2311248779297, + "loss": 0.9976851940155029, + "mean_token_accuracy": 0.687630370259285, + "num_tokens": 3118527.0, + "rewards/accuracies": 0.53125, + "rewards/chosen": -0.01943317288532853, + "rewards/margins": 0.009305761312134564, + "rewards/rejected": -0.028738934081047773, + "step": 54 + }, + { + "entropy": 1.0990462750196457, + "epoch": 0.034488164289073524, + "grad_norm": 29.28276824951172, + "learning_rate": 5.4e-07, + "logits/chosen": -17.590637043087355, + "logits/rejected": -16.153945977490892, + "logps/chosen": -548.3314371109009, + "logps/rejected": -324.43446254730225, + "loss": 0.9927998781204224, + "mean_token_accuracy": 0.7380417436361313, + "num_tokens": 3164828.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.007261359598487616, + "rewards/margins": 0.02887870534323156, + "rewards/rejected": -0.0216173455119133, + "step": 55 + }, + { + "entropy": 1.2540519461035728, + "epoch": 0.03511522182160213, + "grad_norm": 36.696895599365234, + "learning_rate": 5.5e-07, + "logits/chosen": -15.279084980657903, + "logits/rejected": -15.824559065606763, + "logps/chosen": -903.7292518615723, + "logps/rejected": -541.010986328125, + "loss": 0.9985660314559937, + "mean_token_accuracy": 0.6998207569122314, + "num_tokens": 3233270.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.016322331794071943, + "rewards/margins": 0.0056688676122576, + "rewards/rejected": -0.021991199755575508, + "step": 56 + }, + { + "entropy": 1.0729844830930233, + "epoch": 0.03574227935413074, + "grad_norm": 26.47355842590332, + "learning_rate": 5.6e-07, + "logits/chosen": -19.623318897458958, + "logits/rejected": -20.258368385656922, + "logps/chosen": -469.5618600845337, + "logps/rejected": -361.1621437072754, + "loss": 1.0013068914413452, + "mean_token_accuracy": 0.7484212070703506, + "num_tokens": 3268958.0, + "rewards/accuracies": 0.4375, + "rewards/chosen": 0.004558162530884147, + "rewards/margins": -0.005245101172477007, + "rewards/rejected": 0.009803264052607119, + "step": 57 + }, + { + "entropy": 0.9888226762413979, + "epoch": 0.03636933688665935, + "grad_norm": 28.04119873046875, + "learning_rate": 5.699999999999999e-07, + "logits/chosen": -18.97722177970246, + "logits/rejected": -17.818980544574217, + "logps/chosen": -490.36932277679443, + "logps/rejected": -353.6288299560547, + "loss": 0.9960744976997375, + "mean_token_accuracy": 0.763416476547718, + "num_tokens": 3318736.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.0007263210136443377, + "rewards/margins": 0.015722022857517004, + "rewards/rejected": -0.016448343638330698, + "step": 58 + }, + { + "entropy": 1.2886198982596397, + "epoch": 0.03699639441918796, + "grad_norm": 43.716182708740234, + "learning_rate": 5.8e-07, + "logits/chosen": -20.97464568747773, + "logits/rejected": -18.98481146897743, + "logps/chosen": -1585.0814723968506, + "logps/rejected": -528.0391731262207, + "loss": 0.9945598840713501, + "mean_token_accuracy": 0.7001667320728302, + "num_tokens": 3386147.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.019920118735171854, + "rewards/margins": 0.021774652879685163, + "rewards/rejected": -0.0018545325146988034, + "step": 59 + }, + { + "entropy": 1.039273351430893, + "epoch": 0.03762345195171657, + "grad_norm": 27.81415367126465, + "learning_rate": 5.9e-07, + "logits/chosen": -18.662482620406607, + "logits/rejected": -20.1330546037938, + "logps/chosen": -365.54387283325195, + "logps/rejected": -373.88531494140625, + "loss": 0.996242105960846, + "mean_token_accuracy": 0.7572688236832619, + "num_tokens": 3424538.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.006609153002500534, + "rewards/margins": 0.015012014424428344, + "rewards/rejected": -0.02162116818362847, + "step": 60 + }, + { + "entropy": 1.2042050436139107, + "epoch": 0.03825050948424518, + "grad_norm": 36.16862869262695, + "learning_rate": 6e-07, + "logits/chosen": -20.75980830973855, + "logits/rejected": -20.60451823594014, + "logps/chosen": -1099.3400793075562, + "logps/rejected": -421.7832374572754, + "loss": 1.0015130043029785, + "mean_token_accuracy": 0.7048044949769974, + "num_tokens": 3491026.0, + "rewards/accuracies": 0.46875, + "rewards/chosen": 0.00036664726212620735, + "rewards/margins": -0.005983836483210325, + "rewards/rejected": 0.00635048293042928, + "step": 61 + }, + { + "entropy": 1.0666373148560524, + "epoch": 0.03887756701677379, + "grad_norm": 43.32762908935547, + "learning_rate": 6.1e-07, + "logits/chosen": -23.173134701752, + "logits/rejected": -20.672318471013675, + "logps/chosen": -1242.8998069763184, + "logps/rejected": -464.8951187133789, + "loss": 0.9945209622383118, + "mean_token_accuracy": 0.7305290177464485, + "num_tokens": 3555810.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.005398005829192698, + "rewards/margins": 0.022236518329009414, + "rewards/rejected": -0.016838514362461865, + "step": 62 + }, + { + "entropy": 0.7979357466101646, + "epoch": 0.0395046245493024, + "grad_norm": 30.137588500976562, + "learning_rate": 6.2e-07, + "logits/chosen": -22.89783292156173, + "logits/rejected": -22.199726863036197, + "logps/chosen": -536.3147115707397, + "logps/rejected": -364.88674545288086, + "loss": 0.9864821434020996, + "mean_token_accuracy": 0.8112821727991104, + "num_tokens": 3604824.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.03518715803511441, + "rewards/margins": 0.056509705260396004, + "rewards/rejected": -0.02132254181196913, + "step": 63 + }, + { + "entropy": 1.1044994071125984, + "epoch": 0.04013168208183101, + "grad_norm": 29.391481399536133, + "learning_rate": 6.3e-07, + "logits/chosen": -24.507481146010182, + "logits/rejected": -22.21625134174552, + "logps/chosen": -702.180121421814, + "logps/rejected": -308.05684661865234, + "loss": 0.9998461604118347, + "mean_token_accuracy": 0.7285795137286186, + "num_tokens": 3651250.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.004402894992381334, + "rewards/margins": -0.0003644675016403198, + "rewards/rejected": -0.004038428800413385, + "step": 64 + }, + { + "entropy": 0.8255317062139511, + "epoch": 0.04075873961435962, + "grad_norm": 36.75194549560547, + "learning_rate": 6.4e-07, + "logits/chosen": -23.442134502527015, + "logits/rejected": -21.24577292120065, + "logps/chosen": -481.1418218612671, + "logps/rejected": -390.04198455810547, + "loss": 0.9808884859085083, + "mean_token_accuracy": 0.8040206730365753, + "num_tokens": 3700811.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.03725379565730691, + "rewards/margins": 0.0775255209300667, + "rewards/rejected": -0.040271724574267864, + "step": 65 + }, + { + "entropy": 0.9590631648898125, + "epoch": 0.041385797146888226, + "grad_norm": 31.754987716674805, + "learning_rate": 6.5e-07, + "logits/chosen": -23.604474652863118, + "logits/rejected": -21.18440918134874, + "logps/chosen": -892.4836721420288, + "logps/rejected": -423.81532859802246, + "loss": 0.9878406524658203, + "mean_token_accuracy": 0.7679838165640831, + "num_tokens": 3752809.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": 0.035222190992499236, + "rewards/margins": 0.05575526849133894, + "rewards/rejected": -0.020533079630695283, + "step": 66 + }, + { + "entropy": 0.9526357278227806, + "epoch": 0.04201285467941684, + "grad_norm": 38.09070587158203, + "learning_rate": 6.6e-07, + "logits/chosen": -19.300392282829993, + "logits/rejected": -20.892184024152968, + "logps/chosen": -767.138952255249, + "logps/rejected": -475.6669750213623, + "loss": 0.9936155080795288, + "mean_token_accuracy": 0.7684061825275421, + "num_tokens": 3817303.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.02726819575764239, + "rewards/margins": 0.025456703966483474, + "rewards/rejected": 0.0018114912672899663, + "step": 67 + }, + { + "entropy": 0.7557753957808018, + "epoch": 0.04263991221194545, + "grad_norm": 31.207439422607422, + "learning_rate": 6.7e-07, + "logits/chosen": -22.31144983616734, + "logits/rejected": -22.303479200481075, + "logps/chosen": -516.0994625091553, + "logps/rejected": -432.821720123291, + "loss": 0.993211030960083, + "mean_token_accuracy": 0.8136529326438904, + "num_tokens": 3869975.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.008595484774559736, + "rewards/margins": 0.027202091412618756, + "rewards/rejected": -0.018606607103720307, + "step": 68 + }, + { + "entropy": 1.078584998846054, + "epoch": 0.043266969744474054, + "grad_norm": 35.849456787109375, + "learning_rate": 6.800000000000001e-07, + "logits/chosen": -20.188257880122976, + "logits/rejected": -19.395488505188574, + "logps/chosen": -778.6550903320312, + "logps/rejected": -460.7582092285156, + "loss": 0.9967914819717407, + "mean_token_accuracy": 0.739520289003849, + "num_tokens": 3917591.0, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.0018483520834706724, + "rewards/margins": 0.013088171719573438, + "rewards/rejected": -0.014936523628421128, + "step": 69 + }, + { + "entropy": 1.0627146326005459, + "epoch": 0.043894027277002666, + "grad_norm": 33.957401275634766, + "learning_rate": 6.9e-07, + "logits/chosen": -19.09365423111348, + "logits/rejected": -19.238957994462496, + "logps/chosen": -625.0276880264282, + "logps/rejected": -460.5406036376953, + "loss": 0.9846621155738831, + "mean_token_accuracy": 0.7464545965194702, + "num_tokens": 3963982.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.005297277413774282, + "rewards/margins": 0.06182992341928184, + "rewards/rejected": -0.056532644899562, + "step": 70 + }, + { + "entropy": 1.0039971619844437, + "epoch": 0.04452108480953128, + "grad_norm": 36.00166702270508, + "learning_rate": 7e-07, + "logits/chosen": -21.96269636469623, + "logits/rejected": -21.29580632185736, + "logps/chosen": -772.8854808807373, + "logps/rejected": -615.0897903442383, + "loss": 0.9930751323699951, + "mean_token_accuracy": 0.7512786686420441, + "num_tokens": 4028293.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0168970461236313, + "rewards/margins": 0.02881601534318179, + "rewards/rejected": -0.011918970150873065, + "step": 71 + }, + { + "entropy": 1.302395537495613, + "epoch": 0.04514814234205988, + "grad_norm": 39.504661560058594, + "learning_rate": 7.1e-07, + "logits/chosen": -19.878782312586424, + "logits/rejected": -18.000410834982972, + "logps/chosen": -1210.3033666610718, + "logps/rejected": -607.6138458251953, + "loss": 0.9891315698623657, + "mean_token_accuracy": 0.703778937458992, + "num_tokens": 4089535.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.0096849100664258, + "rewards/margins": 0.043962169205769897, + "rewards/rejected": -0.034277260303497314, + "step": 72 + }, + { + "entropy": 0.9964236989617348, + "epoch": 0.045775199874588494, + "grad_norm": 33.938472747802734, + "learning_rate": 7.2e-07, + "logits/chosen": -20.415109094760414, + "logits/rejected": -17.26682495783373, + "logps/chosen": -721.8188791275024, + "logps/rejected": -408.42410469055176, + "loss": 0.995058536529541, + "mean_token_accuracy": 0.7626092284917831, + "num_tokens": 4140075.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.015723853604868054, + "rewards/margins": 0.019197183661162853, + "rewards/rejected": -0.03492103505413979, + "step": 73 + }, + { + "entropy": 1.017537921667099, + "epoch": 0.046402257407117106, + "grad_norm": 26.6578311920166, + "learning_rate": 7.3e-07, + "logits/chosen": -15.80474927414226, + "logits/rejected": -16.128822298593438, + "logps/chosen": -515.7086997032166, + "logps/rejected": -239.00895309448242, + "loss": 0.9995874762535095, + "mean_token_accuracy": 0.7658038064837456, + "num_tokens": 4194003.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.008220234914915636, + "rewards/margins": 0.0016142040840350091, + "rewards/rejected": -0.009834438504185528, + "step": 74 + }, + { + "entropy": 0.9735286235809326, + "epoch": 0.04702931493964571, + "grad_norm": 33.58881759643555, + "learning_rate": 7.4e-07, + "logits/chosen": -20.946241053605, + "logits/rejected": -19.067083934619777, + "logps/chosen": -674.0198707580566, + "logps/rejected": -419.6971263885498, + "loss": 0.991060733795166, + "mean_token_accuracy": 0.7505866810679436, + "num_tokens": 4244222.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.015050875255838037, + "rewards/margins": 0.03578460752032697, + "rewards/rejected": -0.020733732322696596, + "step": 75 + }, + { + "entropy": 1.137626238167286, + "epoch": 0.04765637247217432, + "grad_norm": 38.98091125488281, + "learning_rate": 7.5e-07, + "logits/chosen": -22.829858763000864, + "logits/rejected": -19.879969265793594, + "logps/chosen": -1177.3832550048828, + "logps/rejected": -511.0323143005371, + "loss": 0.9940009713172913, + "mean_token_accuracy": 0.7326341941952705, + "num_tokens": 4313717.0, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.021500761155039072, + "rewards/margins": 0.023731452063657343, + "rewards/rejected": -0.0452322136843577, + "step": 76 + }, + { + "entropy": 1.2288251742720604, + "epoch": 0.048283430004702935, + "grad_norm": 28.220809936523438, + "learning_rate": 7.599999999999999e-07, + "logits/chosen": -17.088404479801554, + "logits/rejected": -18.424581121455233, + "logps/chosen": -600.109245300293, + "logps/rejected": -335.4466133117676, + "loss": 1.0012062788009644, + "mean_token_accuracy": 0.7148683369159698, + "num_tokens": 4358615.0, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.014499580138362944, + "rewards/margins": -0.004860305110923946, + "rewards/rejected": -0.00963927514385432, + "step": 77 + }, + { + "entropy": 1.0265920907258987, + "epoch": 0.04891048753723154, + "grad_norm": 34.03989791870117, + "learning_rate": 7.699999999999999e-07, + "logits/chosen": -22.126280957342644, + "logits/rejected": -21.625379783333514, + "logps/chosen": -804.5845255851746, + "logps/rejected": -453.7879638671875, + "loss": 0.9970452785491943, + "mean_token_accuracy": 0.7403309643268585, + "num_tokens": 4415873.0, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.008073777018580586, + "rewards/margins": 0.011700771981850266, + "rewards/rejected": -0.019774550921283662, + "step": 78 + }, + { + "entropy": 1.262473076581955, + "epoch": 0.04953754506976015, + "grad_norm": 36.033348083496094, + "learning_rate": 7.799999999999999e-07, + "logits/chosen": -18.124917388062837, + "logits/rejected": -19.211221946008383, + "logps/chosen": -795.0781173706055, + "logps/rejected": -577.8308029174805, + "loss": 0.9934069514274597, + "mean_token_accuracy": 0.7068465352058411, + "num_tokens": 4477334.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.004607248993124813, + "rewards/margins": 0.026270719012245536, + "rewards/rejected": -0.030877968529239297, + "step": 79 + }, + { + "entropy": 0.9185331761837006, + "epoch": 0.05016460260228876, + "grad_norm": 36.832279205322266, + "learning_rate": 7.9e-07, + "logits/chosen": -20.603378074573, + "logits/rejected": -17.056396975980135, + "logps/chosen": -758.8550605773926, + "logps/rejected": -412.65998458862305, + "loss": 0.9883898496627808, + "mean_token_accuracy": 0.7726860344409943, + "num_tokens": 4533455.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.0016075177991297096, + "rewards/margins": 0.04657475184649229, + "rewards/rejected": -0.04818226984934881, + "step": 80 + }, + { + "entropy": 1.121558502316475, + "epoch": 0.05079166013481737, + "grad_norm": 36.56794738769531, + "learning_rate": 8e-07, + "logits/chosen": -20.37862110155874, + "logits/rejected": -18.992690204889037, + "logps/chosen": -1184.8006420135498, + "logps/rejected": -610.0655975341797, + "loss": 0.9853536486625671, + "mean_token_accuracy": 0.7357284799218178, + "num_tokens": 4596652.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.02295642625540495, + "rewards/margins": 0.059380507678724825, + "rewards/rejected": -0.03642408235464245, + "step": 81 + }, + { + "entropy": 1.0517073720693588, + "epoch": 0.05141871766734598, + "grad_norm": 39.78794860839844, + "learning_rate": 8.1e-07, + "logits/chosen": -23.220199460872006, + "logits/rejected": -21.76251201594231, + "logps/chosen": -901.9024753570557, + "logps/rejected": -581.7087097167969, + "loss": 0.9863370656967163, + "mean_token_accuracy": 0.7483935281634331, + "num_tokens": 4660574.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.005605014972388744, + "rewards/margins": 0.05513616371899843, + "rewards/rejected": -0.04953114781528711, + "step": 82 + }, + { + "entropy": 1.031716726720333, + "epoch": 0.05204577519987459, + "grad_norm": 34.147884368896484, + "learning_rate": 8.199999999999999e-07, + "logits/chosen": -17.43597000771303, + "logits/rejected": -17.040958005172556, + "logps/chosen": -611.6086521148682, + "logps/rejected": -463.6491508483887, + "loss": 0.9925582408905029, + "mean_token_accuracy": 0.7591038644313812, + "num_tokens": 4726456.0, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.009094578330405056, + "rewards/margins": 0.029864652664400637, + "rewards/rejected": -0.03895922936499119, + "step": 83 + }, + { + "entropy": 1.1349261552095413, + "epoch": 0.052672832732403196, + "grad_norm": 32.638126373291016, + "learning_rate": 8.299999999999999e-07, + "logits/chosen": -17.404521363601013, + "logits/rejected": -19.98172508139376, + "logps/chosen": -767.6361827850342, + "logps/rejected": -448.48921966552734, + "loss": 0.9871995449066162, + "mean_token_accuracy": 0.7235324308276176, + "num_tokens": 4792314.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.017612980911508203, + "rewards/margins": 0.05156451647053473, + "rewards/rejected": -0.033951534889638424, + "step": 84 + }, + { + "entropy": 1.1050493195652962, + "epoch": 0.05329989026493181, + "grad_norm": 36.08334732055664, + "learning_rate": 8.399999999999999e-07, + "logits/chosen": -16.500999504676678, + "logits/rejected": -18.25587297431137, + "logps/chosen": -675.2915000915527, + "logps/rejected": -561.0027389526367, + "loss": 0.9919638633728027, + "mean_token_accuracy": 0.741675615310669, + "num_tokens": 4849066.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.0004984733532182872, + "rewards/margins": 0.03216184466145933, + "rewards/rejected": -0.03166337008588016, + "step": 85 + }, + { + "entropy": 1.220378190279007, + "epoch": 0.05392694779746042, + "grad_norm": 40.76183319091797, + "learning_rate": 8.499999999999999e-07, + "logits/chosen": -23.90722501767462, + "logits/rejected": -22.38695916119651, + "logps/chosen": -1048.8524017333984, + "logps/rejected": -473.95777893066406, + "loss": 0.9880315661430359, + "mean_token_accuracy": 0.7109057381749153, + "num_tokens": 4923190.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.0222030496224761, + "rewards/margins": 0.04874912742525339, + "rewards/rejected": -0.07095217774622142, + "step": 86 + }, + { + "entropy": 1.2091117054224014, + "epoch": 0.054554005329989025, + "grad_norm": 35.72580337524414, + "learning_rate": 8.599999999999999e-07, + "logits/chosen": -20.30170921476879, + "logits/rejected": -21.20954152859497, + "logps/chosen": -951.2183666229248, + "logps/rejected": -513.4668788909912, + "loss": 0.9863914251327515, + "mean_token_accuracy": 0.7123682126402855, + "num_tokens": 4974868.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.026849399670027196, + "rewards/margins": 0.05532143288291991, + "rewards/rejected": -0.028472037287428975, + "step": 87 + }, + { + "entropy": 1.073004886507988, + "epoch": 0.05518106286251764, + "grad_norm": 36.27262496948242, + "learning_rate": 8.699999999999999e-07, + "logits/chosen": -19.685747925382, + "logits/rejected": -17.267068163974074, + "logps/chosen": -578.3146362304688, + "logps/rejected": -475.0130271911621, + "loss": 0.9875953197479248, + "mean_token_accuracy": 0.7410635352134705, + "num_tokens": 5037188.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.014228225103579462, + "rewards/margins": 0.04966498585417867, + "rewards/rejected": -0.03543675900436938, + "step": 88 + }, + { + "entropy": 1.056903399527073, + "epoch": 0.05580812039504625, + "grad_norm": 35.787532806396484, + "learning_rate": 8.799999999999999e-07, + "logits/chosen": -20.64660484271999, + "logits/rejected": -19.71323913593022, + "logps/chosen": -527.9435653686523, + "logps/rejected": -577.5134410858154, + "loss": 0.9897867441177368, + "mean_token_accuracy": 0.7574738562107086, + "num_tokens": 5086966.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.00012873177183791995, + "rewards/margins": 0.04125080001540482, + "rewards/rejected": -0.04112206675927155, + "step": 89 + }, + { + "entropy": 0.9986204952001572, + "epoch": 0.05643517792757485, + "grad_norm": 33.426368713378906, + "learning_rate": 8.9e-07, + "logits/chosen": -18.57544006926955, + "logits/rejected": -19.12848221709334, + "logps/chosen": -766.3484830856323, + "logps/rejected": -476.17732429504395, + "loss": 0.9895883798599243, + "mean_token_accuracy": 0.7547919675707817, + "num_tokens": 5144107.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.013297693920321763, + "rewards/margins": 0.04271816968685016, + "rewards/rejected": -0.029420473758364096, + "step": 90 + }, + { + "entropy": 1.1071998253464699, + "epoch": 0.057062235460103465, + "grad_norm": 46.52241516113281, + "learning_rate": 9e-07, + "logits/chosen": -20.750110720341095, + "logits/rejected": -20.73160492144149, + "logps/chosen": -1105.2754192352295, + "logps/rejected": -582.0479850769043, + "loss": 0.9801906943321228, + "mean_token_accuracy": 0.7243269085884094, + "num_tokens": 5211621.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.019272988894954324, + "rewards/margins": 0.0801434232853353, + "rewards/rejected": -0.06087043380830437, + "step": 91 + }, + { + "entropy": 1.1382925510406494, + "epoch": 0.05768929299263208, + "grad_norm": 37.77758026123047, + "learning_rate": 9.1e-07, + "logits/chosen": -20.904564962987713, + "logits/rejected": -21.018963758425333, + "logps/chosen": -640.5741844177246, + "logps/rejected": -708.7987327575684, + "loss": 0.9877656102180481, + "mean_token_accuracy": 0.7462773993611336, + "num_tokens": 5263236.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.032095869741169736, + "rewards/margins": 0.04855327168479562, + "rewards/rejected": -0.08064913935959339, + "step": 92 + }, + { + "entropy": 1.1960504204034805, + "epoch": 0.05831635052516068, + "grad_norm": 27.9012508392334, + "learning_rate": 9.2e-07, + "logits/chosen": -17.198728228506177, + "logits/rejected": -19.989566953539725, + "logps/chosen": -581.2822170257568, + "logps/rejected": -365.0048427581787, + "loss": 0.9875739812850952, + "mean_token_accuracy": 0.7414722666144371, + "num_tokens": 5301490.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0007973910542204976, + "rewards/margins": 0.04991468833759427, + "rewards/rejected": -0.04911729716695845, + "step": 93 + }, + { + "entropy": 0.9865989461541176, + "epoch": 0.058943408057689294, + "grad_norm": 31.25222396850586, + "learning_rate": 9.3e-07, + "logits/chosen": -18.790141123207366, + "logits/rejected": -16.14887539333265, + "logps/chosen": -577.1131019592285, + "logps/rejected": -340.10778999328613, + "loss": 0.9858304262161255, + "mean_token_accuracy": 0.7623919099569321, + "num_tokens": 5361142.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0011998027912341058, + "rewards/margins": 0.057147986139170825, + "rewards/rejected": -0.05594818387180567, + "step": 94 + }, + { + "entropy": 1.044435366988182, + "epoch": 0.059570465590217905, + "grad_norm": 26.22032928466797, + "learning_rate": 9.399999999999999e-07, + "logits/chosen": -17.09738355288199, + "logits/rejected": -17.458014048093904, + "logps/chosen": -388.39873027801514, + "logps/rejected": -326.0696334838867, + "loss": 0.9898619055747986, + "mean_token_accuracy": 0.7581661641597748, + "num_tokens": 5408748.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.0021333397598937154, + "rewards/margins": 0.04059030464850366, + "rewards/rejected": -0.04272364475764334, + "step": 95 + }, + { + "entropy": 0.8697420880198479, + "epoch": 0.06019752312274651, + "grad_norm": 42.353424072265625, + "learning_rate": 9.499999999999999e-07, + "logits/chosen": -23.41937699390698, + "logits/rejected": -21.70754765415036, + "logps/chosen": -1428.7145009040833, + "logps/rejected": -443.71881103515625, + "loss": 0.9914920926094055, + "mean_token_accuracy": 0.7912928014993668, + "num_tokens": 5483101.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.011147335055284202, + "rewards/margins": 0.045076546957716346, + "rewards/rejected": -0.03392921155318618, + "step": 96 + }, + { + "entropy": 1.1013405546545982, + "epoch": 0.06082458065527512, + "grad_norm": 36.10636520385742, + "learning_rate": 9.6e-07, + "logits/chosen": -19.821280825841903, + "logits/rejected": -19.578973804909, + "logps/chosen": -985.1940336227417, + "logps/rejected": -473.689453125, + "loss": 0.983747124671936, + "mean_token_accuracy": 0.7495378330349922, + "num_tokens": 5534649.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.0021885630558244884, + "rewards/margins": 0.06537919119000435, + "rewards/rejected": -0.06756775546818972, + "step": 97 + }, + { + "entropy": 1.2237722724676132, + "epoch": 0.061451638187803734, + "grad_norm": 41.582496643066406, + "learning_rate": 9.7e-07, + "logits/chosen": -20.153029153130383, + "logits/rejected": -17.664550339329576, + "logps/chosen": -1082.3712921142578, + "logps/rejected": -649.8507308959961, + "loss": 0.9809252023696899, + "mean_token_accuracy": 0.718035951256752, + "num_tokens": 5612387.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.02365162328351289, + "rewards/margins": 0.07657872815616429, + "rewards/rejected": -0.052927102777175605, + "step": 98 + }, + { + "entropy": 1.0099660605192184, + "epoch": 0.06207869572033234, + "grad_norm": 30.545495986938477, + "learning_rate": 9.8e-07, + "logits/chosen": -19.286327351085962, + "logits/rejected": -19.547563799363445, + "logps/chosen": -635.9861097335815, + "logps/rejected": -379.7309799194336, + "loss": 0.9843310117721558, + "mean_token_accuracy": 0.7559159845113754, + "num_tokens": 5664396.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.005910164851229638, + "rewards/margins": 0.06448549684137106, + "rewards/rejected": -0.07039566081948578, + "step": 99 + }, + { + "entropy": 1.1891107335686684, + "epoch": 0.06270575325286094, + "grad_norm": 30.441099166870117, + "learning_rate": 9.9e-07, + "logits/chosen": -16.462167828377467, + "logits/rejected": -17.668335453566645, + "logps/chosen": -774.8042068481445, + "logps/rejected": -352.1152057647705, + "loss": 0.9829254150390625, + "mean_token_accuracy": 0.723647378385067, + "num_tokens": 5712927.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.0013654606882482767, + "rewards/margins": 0.0686984455678612, + "rewards/rejected": -0.07006390765309334, + "step": 100 + }, + { + "entropy": 1.0059744790196419, + "epoch": 0.06333281078538956, + "grad_norm": 34.044551849365234, + "learning_rate": 1e-06, + "logits/chosen": -22.569647432957222, + "logits/rejected": -20.571769885244017, + "logps/chosen": -737.9557514190674, + "logps/rejected": -456.0193290710449, + "loss": 0.9836858510971069, + "mean_token_accuracy": 0.7708024978637695, + "num_tokens": 5762779.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.02014523115940392, + "rewards/margins": 0.0654695084085688, + "rewards/rejected": -0.045324277016334236, + "step": 101 + }, + { + "entropy": 1.1221561804413795, + "epoch": 0.06395986831791817, + "grad_norm": 35.60519790649414, + "learning_rate": 9.999988960301596e-07, + "logits/chosen": -17.63612561400455, + "logits/rejected": -21.225849866079535, + "logps/chosen": -851.1271896362305, + "logps/rejected": -731.4087066650391, + "loss": 0.9757521152496338, + "mean_token_accuracy": 0.7322330251336098, + "num_tokens": 5820523.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.011595683405175805, + "rewards/margins": 0.09945710864849389, + "rewards/rejected": -0.08786143315955997, + "step": 102 + }, + { + "entropy": 1.033433958888054, + "epoch": 0.06458692585044677, + "grad_norm": 33.74344253540039, + "learning_rate": 9.999955841255138e-07, + "logits/chosen": -21.6586000014676, + "logits/rejected": -20.241547622800056, + "logps/chosen": -748.6037483215332, + "logps/rejected": -495.99595642089844, + "loss": 0.9870611429214478, + "mean_token_accuracy": 0.7368155121803284, + "num_tokens": 5870733.0, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.03833825106266886, + "rewards/margins": 0.050038286950439215, + "rewards/rejected": -0.08837653254158795, + "step": 103 + }, + { + "entropy": 1.118708185851574, + "epoch": 0.06521398338297539, + "grad_norm": 43.72235107421875, + "learning_rate": 9.999900643006873e-07, + "logits/chosen": -22.952759755082543, + "logits/rejected": -18.112035021982514, + "logps/chosen": -1293.3799285888672, + "logps/rejected": -614.5923004150391, + "loss": 0.9773090481758118, + "mean_token_accuracy": 0.7410418093204498, + "num_tokens": 5941283.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.02265817968873307, + "rewards/margins": 0.09182942099869251, + "rewards/rejected": -0.06917124305618927, + "step": 104 + }, + { + "entropy": 1.3742387741804123, + "epoch": 0.065841040915504, + "grad_norm": 39.11056137084961, + "learning_rate": 9.99982336580055e-07, + "logits/chosen": -16.713279247073885, + "logits/rejected": -16.98860342760087, + "logps/chosen": -950.8361873626709, + "logps/rejected": -464.50708770751953, + "loss": 0.9773339629173279, + "mean_token_accuracy": 0.6922967359423637, + "num_tokens": 6000343.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.006588876945897937, + "rewards/margins": 0.09067038784269243, + "rewards/rejected": -0.08408151054754853, + "step": 105 + }, + { + "entropy": 1.1545687764883041, + "epoch": 0.0664680984480326, + "grad_norm": 34.153663635253906, + "learning_rate": 9.99972400997742e-07, + "logits/chosen": -20.552279910445396, + "logits/rejected": -18.52949584159018, + "logps/chosen": -710.0431318283081, + "logps/rejected": -350.6925792694092, + "loss": 0.982926070690155, + "mean_token_accuracy": 0.7238080576062202, + "num_tokens": 6047719.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.01743771624751389, + "rewards/margins": 0.06860713846981525, + "rewards/rejected": -0.05116942140739411, + "step": 106 + }, + { + "entropy": 0.8841631710529327, + "epoch": 0.06709515598056122, + "grad_norm": 30.25786018371582, + "learning_rate": 9.999602575976219e-07, + "logits/chosen": -16.81424879838689, + "logits/rejected": -16.14855911409956, + "logps/chosen": -426.7861785888672, + "logps/rejected": -396.69312858581543, + "loss": 0.979150652885437, + "mean_token_accuracy": 0.8006457611918449, + "num_tokens": 6096830.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.007483278808649629, + "rewards/margins": 0.0836036205291748, + "rewards/rejected": -0.07612034154590219, + "step": 107 + }, + { + "entropy": 1.0404388830065727, + "epoch": 0.06772221351308982, + "grad_norm": 33.5556640625, + "learning_rate": 9.999459064333188e-07, + "logits/chosen": -15.91254721701439, + "logits/rejected": -16.29134417675286, + "logps/chosen": -601.9646797180176, + "logps/rejected": -457.3084487915039, + "loss": 0.9777726531028748, + "mean_token_accuracy": 0.7538063228130341, + "num_tokens": 6145266.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.019957647193223238, + "rewards/margins": 0.08928463887423277, + "rewards/rejected": -0.0693269899347797, + "step": 108 + }, + { + "entropy": 1.2834724336862564, + "epoch": 0.06834927104561843, + "grad_norm": 34.99026870727539, + "learning_rate": 9.999293475682062e-07, + "logits/chosen": -17.259286736211884, + "logits/rejected": -17.444496267757785, + "logps/chosen": -667.7793045043945, + "logps/rejected": -577.6125526428223, + "loss": 0.9799240231513977, + "mean_token_accuracy": 0.6992196813225746, + "num_tokens": 6196039.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.005005566752515733, + "rewards/margins": 0.08093663223553449, + "rewards/rejected": -0.08594219759106636, + "step": 109 + }, + { + "entropy": 1.055838204920292, + "epoch": 0.06897632857814705, + "grad_norm": 33.32623291015625, + "learning_rate": 9.999105810754053e-07, + "logits/chosen": -18.407450521012187, + "logits/rejected": -18.873402077141666, + "logps/chosen": -799.7204494476318, + "logps/rejected": -477.783242225647, + "loss": 0.9677953720092773, + "mean_token_accuracy": 0.7510818913578987, + "num_tokens": 6263445.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.052083960617892444, + "rewards/margins": 0.1307980790734291, + "rewards/rejected": -0.07871411880478263, + "step": 110 + }, + { + "entropy": 0.9602865129709244, + "epoch": 0.06960338611067565, + "grad_norm": 28.93109130859375, + "learning_rate": 9.99889607037787e-07, + "logits/chosen": -16.415104591720436, + "logits/rejected": -18.58728532903226, + "logps/chosen": -381.7690181732178, + "logps/rejected": -416.77470779418945, + "loss": 0.9900650382041931, + "mean_token_accuracy": 0.7899875342845917, + "num_tokens": 6311526.0, + "rewards/accuracies": 0.59375, + "rewards/chosen": -0.004977155942469835, + "rewards/margins": 0.039845253340899944, + "rewards/rejected": -0.044822409050539136, + "step": 111 + }, + { + "entropy": 0.8192609176039696, + "epoch": 0.07023044364320426, + "grad_norm": 39.31092071533203, + "learning_rate": 9.998664255479704e-07, + "logits/chosen": -25.072994460556252, + "logits/rejected": -22.42456335487019, + "logps/chosen": -1064.658107995987, + "logps/rejected": -414.81433868408203, + "loss": 0.9966785311698914, + "mean_token_accuracy": 0.8144989609718323, + "num_tokens": 6373673.0, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.06160880299285054, + "rewards/margins": 0.012114129436668009, + "rewards/rejected": -0.0737229329533875, + "step": 112 + }, + { + "entropy": 0.9448361806571484, + "epoch": 0.07085750117573288, + "grad_norm": 39.90821075439453, + "learning_rate": 9.99841036708322e-07, + "logits/chosen": -23.766030290287702, + "logits/rejected": -25.34154914444923, + "logps/chosen": -1000.321138381958, + "logps/rejected": -514.6866340637207, + "loss": 0.9712129831314087, + "mean_token_accuracy": 0.7757420614361763, + "num_tokens": 6428282.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.034964483114890754, + "rewards/margins": 0.11884536314755678, + "rewards/rejected": -0.08388087863568217, + "step": 113 + }, + { + "entropy": 0.8173889517784119, + "epoch": 0.07148455870826148, + "grad_norm": 30.850637435913086, + "learning_rate": 9.998134406309553e-07, + "logits/chosen": -25.008434498378293, + "logits/rejected": -22.165182027384386, + "logps/chosen": -702.1321802139282, + "logps/rejected": -378.8375816345215, + "loss": 0.9969829320907593, + "mean_token_accuracy": 0.8100090399384499, + "num_tokens": 6478216.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.0617737959837541, + "rewards/margins": 0.0021084430627524853, + "rewards/rejected": -0.06388223776593804, + "step": 114 + }, + { + "entropy": 1.0738236010074615, + "epoch": 0.07211161624079009, + "grad_norm": 37.23641586303711, + "learning_rate": 9.997836374377318e-07, + "logits/chosen": -21.598642905098632, + "logits/rejected": -23.624653024193194, + "logps/chosen": -677.4854145050049, + "logps/rejected": -513.8848571777344, + "loss": 0.9803842306137085, + "mean_token_accuracy": 0.7514287456870079, + "num_tokens": 6522645.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0021543916955124587, + "rewards/margins": 0.0788510333513841, + "rewards/rejected": -0.07669664057902992, + "step": 115 + }, + { + "entropy": 1.0687246397137642, + "epoch": 0.0727386737733187, + "grad_norm": 28.08934783935547, + "learning_rate": 9.997516272602588e-07, + "logits/chosen": -18.0843256562564, + "logits/rejected": -17.27698922154224, + "logps/chosen": -591.8647804260254, + "logps/rejected": -329.94952392578125, + "loss": 0.9886791706085205, + "mean_token_accuracy": 0.7329046651721001, + "num_tokens": 6565380.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.014423841843381524, + "rewards/margins": 0.045475234859623015, + "rewards/rejected": -0.05989907821640372, + "step": 116 + }, + { + "entropy": 1.2525769509375095, + "epoch": 0.07336573130584731, + "grad_norm": 32.46056365966797, + "learning_rate": 9.997174102398892e-07, + "logits/chosen": -19.069984483617883, + "logits/rejected": -18.31701463203426, + "logps/chosen": -857.891770362854, + "logps/rejected": -576.6237850189209, + "loss": 0.9878202676773071, + "mean_token_accuracy": 0.7090706676244736, + "num_tokens": 6621385.0, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.012965936097316444, + "rewards/margins": 0.04882583348080516, + "rewards/rejected": -0.06179177016019821, + "step": 117 + }, + { + "entropy": 1.0708193145692348, + "epoch": 0.07399278883837591, + "grad_norm": 33.78789138793945, + "learning_rate": 9.996809865277214e-07, + "logits/chosen": -17.575822411940983, + "logits/rejected": -15.71341222924891, + "logps/chosen": -628.3108444213867, + "logps/rejected": -381.9364585876465, + "loss": 0.984896183013916, + "mean_token_accuracy": 0.7500499114394188, + "num_tokens": 6671308.0, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.02526119421236217, + "rewards/margins": 0.060638573253527284, + "rewards/rejected": -0.0858997656032443, + "step": 118 + }, + { + "entropy": 1.1464358419179916, + "epoch": 0.07461984637090453, + "grad_norm": 29.262676239013672, + "learning_rate": 9.996423562845978e-07, + "logits/chosen": -18.950197207830822, + "logits/rejected": -17.89534831083926, + "logps/chosen": -499.1498966217041, + "logps/rejected": -397.74127197265625, + "loss": 0.9772639274597168, + "mean_token_accuracy": 0.7390127554535866, + "num_tokens": 6707362.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.014790326356887817, + "rewards/margins": 0.09125107247382402, + "rewards/rejected": -0.07646074634976685, + "step": 119 + }, + { + "entropy": 1.1956902518868446, + "epoch": 0.07524690390343314, + "grad_norm": 31.247859954833984, + "learning_rate": 9.996015196811054e-07, + "logits/chosen": -16.430566598689513, + "logits/rejected": -16.301428702284802, + "logps/chosen": -712.8943672180176, + "logps/rejected": -467.7664604187012, + "loss": 0.9875868558883667, + "mean_token_accuracy": 0.7251445576548576, + "num_tokens": 6761518.0, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.00015118438750505447, + "rewards/margins": 0.05001709656789899, + "rewards/rejected": -0.04986591334454715, + "step": 120 + }, + { + "entropy": 1.04892847687006, + "epoch": 0.07587396143596174, + "grad_norm": 36.938690185546875, + "learning_rate": 9.995584768975734e-07, + "logits/chosen": -20.353520038495564, + "logits/rejected": -18.411114375084413, + "logps/chosen": -787.6422328948975, + "logps/rejected": -501.9888114929199, + "loss": 0.9638845920562744, + "mean_token_accuracy": 0.7510411590337753, + "num_tokens": 6814204.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.037587814789731055, + "rewards/margins": 0.1457526022568345, + "rewards/rejected": -0.10816478868946433, + "step": 121 + }, + { + "entropy": 0.911387711763382, + "epoch": 0.07650101896849036, + "grad_norm": 30.251392364501953, + "learning_rate": 9.995132281240734e-07, + "logits/chosen": -19.869032639332318, + "logits/rejected": -18.91703122960338, + "logps/chosen": -582.7686996459961, + "logps/rejected": -366.2984085083008, + "loss": 0.9788750410079956, + "mean_token_accuracy": 0.7888387069106102, + "num_tokens": 6861531.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.003442022774834186, + "rewards/margins": 0.0853227588813752, + "rewards/rejected": -0.08188073709607124, + "step": 122 + }, + { + "entropy": 1.0427976697683334, + "epoch": 0.07712807650101897, + "grad_norm": 36.89279556274414, + "learning_rate": 9.994657735604188e-07, + "logits/chosen": -24.044872176023844, + "logits/rejected": -22.597699453222447, + "logps/chosen": -966.8881568908691, + "logps/rejected": -505.8887948989868, + "loss": 0.9708135724067688, + "mean_token_accuracy": 0.7332872226834297, + "num_tokens": 6921458.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.012047640164382756, + "rewards/margins": 0.11771579552441835, + "rewards/rejected": -0.10566815291531384, + "step": 123 + }, + { + "entropy": 1.0259113758802414, + "epoch": 0.07775513403354758, + "grad_norm": 37.096275329589844, + "learning_rate": 9.994161134161632e-07, + "logits/chosen": -19.591871441281974, + "logits/rejected": -20.936406703396575, + "logps/chosen": -818.1269016265869, + "logps/rejected": -639.8456077575684, + "loss": 0.9665595293045044, + "mean_token_accuracy": 0.7770890146493912, + "num_tokens": 6982937.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.020557800424285233, + "rewards/margins": 0.13559715449810028, + "rewards/rejected": -0.11503935337532312, + "step": 124 + }, + { + "entropy": 1.1710731238126755, + "epoch": 0.07838219156607619, + "grad_norm": 42.78230667114258, + "learning_rate": 9.993642479105997e-07, + "logits/chosen": -24.393052790442358, + "logits/rejected": -18.192713037735032, + "logps/chosen": -1177.7320861816406, + "logps/rejected": -522.7479839324951, + "loss": 0.9867607355117798, + "mean_token_accuracy": 0.712364636361599, + "num_tokens": 7046947.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.02724925708025694, + "rewards/margins": 0.052655004314146936, + "rewards/rejected": -0.07990426244214177, + "step": 125 + }, + { + "entropy": 1.1666254326701164, + "epoch": 0.0790092490986048, + "grad_norm": 32.3548469543457, + "learning_rate": 9.993101772727601e-07, + "logits/chosen": -17.63798263385176, + "logits/rejected": -17.47313740458377, + "logps/chosen": -606.1291179656982, + "logps/rejected": -430.56521797180176, + "loss": 0.9682327508926392, + "mean_token_accuracy": 0.7458649724721909, + "num_tokens": 7102060.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.0330769574502483, + "rewards/margins": 0.13182413554750383, + "rewards/rejected": -0.09874717495404184, + "step": 126 + }, + { + "entropy": 1.1291334107518196, + "epoch": 0.07963630663113341, + "grad_norm": 31.756563186645508, + "learning_rate": 9.99253901741414e-07, + "logits/chosen": -21.307392332824012, + "logits/rejected": -20.84085959531398, + "logps/chosen": -732.371265411377, + "logps/rejected": -503.3115940093994, + "loss": 0.9792959690093994, + "mean_token_accuracy": 0.7251338735222816, + "num_tokens": 7157801.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.018919202295364812, + "rewards/margins": 0.08486578240990639, + "rewards/rejected": -0.06594657967798412, + "step": 127 + }, + { + "entropy": 1.1786841079592705, + "epoch": 0.08026336416366202, + "grad_norm": 40.29298782348633, + "learning_rate": 9.99195421565067e-07, + "logits/chosen": -17.529350354706732, + "logits/rejected": -20.833092058953017, + "logps/chosen": -1033.621503829956, + "logps/rejected": -691.7955369949341, + "loss": 0.9725984334945679, + "mean_token_accuracy": 0.7169393450021744, + "num_tokens": 7224323.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.006326087401248515, + "rewards/margins": 0.11025936878286302, + "rewards/rejected": -0.11658545769751072, + "step": 128 + }, + { + "entropy": 1.1065169423818588, + "epoch": 0.08089042169619062, + "grad_norm": 30.228065490722656, + "learning_rate": 9.991347370019609e-07, + "logits/chosen": -18.675428265484687, + "logits/rejected": -18.02665901699452, + "logps/chosen": -738.0616884231567, + "logps/rejected": -413.77689933776855, + "loss": 0.9663163423538208, + "mean_token_accuracy": 0.7391728013753891, + "num_tokens": 7273940.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.04217634559608996, + "rewards/margins": 0.1416997853666544, + "rewards/rejected": -0.09952343860641122, + "step": 129 + }, + { + "entropy": 1.0223164483904839, + "epoch": 0.08151747922871924, + "grad_norm": 36.71884536743164, + "learning_rate": 9.990718483200711e-07, + "logits/chosen": -24.017658505382084, + "logits/rejected": -19.894383210249206, + "logps/chosen": -853.0483455657959, + "logps/rejected": -410.2704782485962, + "loss": 0.9696524739265442, + "mean_token_accuracy": 0.7612569332122803, + "num_tokens": 7330670.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.022751914570108056, + "rewards/margins": 0.12564924580510706, + "rewards/rejected": -0.10289733344689012, + "step": 130 + }, + { + "entropy": 1.188691645860672, + "epoch": 0.08214453676124785, + "grad_norm": 45.50562286376953, + "learning_rate": 9.990067557971066e-07, + "logits/chosen": -20.96738395725814, + "logits/rejected": -18.577314710392834, + "logps/chosen": -1276.1438827514648, + "logps/rejected": -671.3540458679199, + "loss": 0.9635441303253174, + "mean_token_accuracy": 0.7195305675268173, + "num_tokens": 7400528.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.027424347819760442, + "rewards/margins": 0.14744434040039778, + "rewards/rejected": -0.1200199886225164, + "step": 131 + }, + { + "entropy": 0.9636916220188141, + "epoch": 0.08277159429377645, + "grad_norm": 28.21497344970703, + "learning_rate": 9.989394597205082e-07, + "logits/chosen": -17.83059240747361, + "logits/rejected": -17.95696408960033, + "logps/chosen": -485.29137802124023, + "logps/rejected": -400.46766471862793, + "loss": 0.976447343826294, + "mean_token_accuracy": 0.7697746828198433, + "num_tokens": 7462922.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.010323267662897706, + "rewards/margins": 0.09466969268396497, + "rewards/rejected": -0.08434642176143825, + "step": 132 + }, + { + "entropy": 0.9702051728963852, + "epoch": 0.08339865182630507, + "grad_norm": 35.21070861816406, + "learning_rate": 9.98869960387447e-07, + "logits/chosen": -24.15228929902058, + "logits/rejected": -17.62097141810482, + "logps/chosen": -707.7485370635986, + "logps/rejected": -328.29246520996094, + "loss": 0.9750052690505981, + "mean_token_accuracy": 0.7517875507473946, + "num_tokens": 7516053.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.020313383429311216, + "rewards/margins": 0.10041370801627636, + "rewards/rejected": -0.08010032417951152, + "step": 133 + }, + { + "entropy": 1.080765277147293, + "epoch": 0.08402570935883368, + "grad_norm": 33.306419372558594, + "learning_rate": 9.98798258104824e-07, + "logits/chosen": -16.50647893746725, + "logits/rejected": -18.638188273587172, + "logps/chosen": -394.3768768310547, + "logps/rejected": -530.8435726165771, + "loss": 0.9684998393058777, + "mean_token_accuracy": 0.7539431154727936, + "num_tokens": 7560146.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.008416369266342372, + "rewards/margins": 0.12770695355720818, + "rewards/rejected": -0.13612332614138722, + "step": 134 + }, + { + "entropy": 1.0699984654784203, + "epoch": 0.08465276689136228, + "grad_norm": 29.02651596069336, + "learning_rate": 9.987243531892676e-07, + "logits/chosen": -17.323968607265364, + "logits/rejected": -17.327844017915886, + "logps/chosen": -423.5040798187256, + "logps/rejected": -401.37097549438477, + "loss": 0.9706511497497559, + "mean_token_accuracy": 0.7508202418684959, + "num_tokens": 7604430.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.010786252591060475, + "rewards/margins": 0.11822709790430963, + "rewards/rejected": -0.10744084196630865, + "step": 135 + }, + { + "entropy": 1.1741472780704498, + "epoch": 0.0852798244238909, + "grad_norm": 29.518667221069336, + "learning_rate": 9.98648245967133e-07, + "logits/chosen": -17.057511505199, + "logits/rejected": -15.816799192033697, + "logps/chosen": -578.4385671615601, + "logps/rejected": -470.2986297607422, + "loss": 0.9768874645233154, + "mean_token_accuracy": 0.7322021424770355, + "num_tokens": 7661280.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.018278606235980988, + "rewards/margins": 0.09282787260599434, + "rewards/rejected": -0.07454926625359803, + "step": 136 + }, + { + "entropy": 1.1715349406003952, + "epoch": 0.0859068819564195, + "grad_norm": 35.8231086730957, + "learning_rate": 9.985699367745007e-07, + "logits/chosen": -20.207632968359622, + "logits/rejected": -18.918033251926897, + "logps/chosen": -1063.0584182739258, + "logps/rejected": -546.3697662353516, + "loss": 0.9809889793395996, + "mean_token_accuracy": 0.7140867561101913, + "num_tokens": 7723139.0, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.016168745700269938, + "rewards/margins": 0.07651804899796844, + "rewards/rejected": -0.09268679865635931, + "step": 137 + }, + { + "entropy": 1.0441111102700233, + "epoch": 0.08653393948894811, + "grad_norm": 32.78045654296875, + "learning_rate": 9.984894259571743e-07, + "logits/chosen": -19.221359907309978, + "logits/rejected": -18.10666979652929, + "logps/chosen": -489.88498306274414, + "logps/rejected": -398.86695098876953, + "loss": 0.9633641242980957, + "mean_token_accuracy": 0.7486372217535973, + "num_tokens": 7766818.0, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.011141044669784606, + "rewards/margins": 0.14769388129934669, + "rewards/rejected": -0.1365528372116387, + "step": 138 + }, + { + "entropy": 1.1274549514055252, + "epoch": 0.08716099702147673, + "grad_norm": 28.61403465270996, + "learning_rate": 9.984067138706801e-07, + "logits/chosen": -20.09536989108541, + "logits/rejected": -19.089721130331522, + "logps/chosen": -668.453311920166, + "logps/rejected": -400.32433700561523, + "loss": 0.9768990278244019, + "mean_token_accuracy": 0.7357787117362022, + "num_tokens": 7805514.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": 0.008357231738045812, + "rewards/margins": 0.09283104329369962, + "rewards/rejected": -0.08447381528094411, + "step": 139 + }, + { + "entropy": 1.072766751050949, + "epoch": 0.08778805455400533, + "grad_norm": 36.328330993652344, + "learning_rate": 9.983218008802647e-07, + "logits/chosen": -16.597205351543526, + "logits/rejected": -17.38877890508226, + "logps/chosen": -845.426420211792, + "logps/rejected": -446.5038776397705, + "loss": 0.969933271408081, + "mean_token_accuracy": 0.7438737154006958, + "num_tokens": 7867552.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.004711989313364029, + "rewards/margins": 0.12162661645561457, + "rewards/rejected": -0.12633860250934958, + "step": 140 + }, + { + "entropy": 1.1710950955748558, + "epoch": 0.08841511208653394, + "grad_norm": 35.29788589477539, + "learning_rate": 9.982346873608936e-07, + "logits/chosen": -17.457327570012882, + "logits/rejected": -17.907635026265908, + "logps/chosen": -594.5470962524414, + "logps/rejected": -608.6735458374023, + "loss": 0.9617944359779358, + "mean_token_accuracy": 0.7367723286151886, + "num_tokens": 7913056.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.0014770900015719235, + "rewards/margins": 0.15395231172442436, + "rewards/rejected": -0.15247522108256817, + "step": 141 + }, + { + "entropy": 1.0754098296165466, + "epoch": 0.08904216961906256, + "grad_norm": 35.45695877075195, + "learning_rate": 9.981453736972495e-07, + "logits/chosen": -19.118562752807563, + "logits/rejected": -16.91145482295967, + "logps/chosen": -698.3301868438721, + "logps/rejected": -497.7921562194824, + "loss": 0.9720945954322815, + "mean_token_accuracy": 0.7595574334263802, + "num_tokens": 7980591.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.01288341250619851, + "rewards/margins": 0.11340800416655838, + "rewards/rejected": -0.10052459384314716, + "step": 142 + }, + { + "entropy": 0.9044511765241623, + "epoch": 0.08966922715159116, + "grad_norm": 30.17085075378418, + "learning_rate": 9.98053860283731e-07, + "logits/chosen": -21.207617728808614, + "logits/rejected": -20.905731823651053, + "logps/chosen": -628.0943202972412, + "logps/rejected": -420.4750385284424, + "loss": 0.9711873531341553, + "mean_token_accuracy": 0.7811061069369316, + "num_tokens": 8034884.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.023002621717751026, + "rewards/margins": 0.1186411960516125, + "rewards/rejected": -0.09563857619650662, + "step": 143 + }, + { + "entropy": 1.1677963212132454, + "epoch": 0.09029628468411977, + "grad_norm": 31.993118286132812, + "learning_rate": 9.9796014752445e-07, + "logits/chosen": -18.57265198556839, + "logits/rejected": -17.995440262698004, + "logps/chosen": -658.4226865768433, + "logps/rejected": -438.4370231628418, + "loss": 0.9786888360977173, + "mean_token_accuracy": 0.7285968512296677, + "num_tokens": 8082039.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.02490088331978768, + "rewards/margins": 0.08490909356623888, + "rewards/rejected": -0.10980997630394995, + "step": 144 + }, + { + "entropy": 1.0427600964903831, + "epoch": 0.09092334221664838, + "grad_norm": 31.311111450195312, + "learning_rate": 9.978642358332307e-07, + "logits/chosen": -19.139753277032113, + "logits/rejected": -17.59120642360813, + "logps/chosen": -567.9225654602051, + "logps/rejected": -465.25221252441406, + "loss": 0.9686312675476074, + "mean_token_accuracy": 0.7571733519434929, + "num_tokens": 8123718.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.008705203887075186, + "rewards/margins": 0.12602692190557718, + "rewards/rejected": -0.117321718018502, + "step": 145 + }, + { + "entropy": 0.8850021287798882, + "epoch": 0.09155039974917699, + "grad_norm": 37.63220977783203, + "learning_rate": 9.97766125633608e-07, + "logits/chosen": -19.910044793415654, + "logits/rejected": -22.54540787489115, + "logps/chosen": -699.3141222000122, + "logps/rejected": -466.68331146240234, + "loss": 0.9604589939117432, + "mean_token_accuracy": 0.7936916127800941, + "num_tokens": 8187632.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.003929961123503745, + "rewards/margins": 0.168057446135208, + "rewards/rejected": -0.17198741296306252, + "step": 146 + }, + { + "entropy": 1.0326995179057121, + "epoch": 0.0921774572817056, + "grad_norm": 37.081241607666016, + "learning_rate": 9.976658173588243e-07, + "logits/chosen": -21.190682950448576, + "logits/rejected": -17.69587061048208, + "logps/chosen": -751.942907333374, + "logps/rejected": -442.78441619873047, + "loss": 0.9758281707763672, + "mean_token_accuracy": 0.7523355558514595, + "num_tokens": 8243290.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.02370329963741824, + "rewards/margins": 0.09733096603304148, + "rewards/rejected": -0.12103426642715931, + "step": 147 + }, + { + "entropy": 0.9734203144907951, + "epoch": 0.09280451481423421, + "grad_norm": 36.707550048828125, + "learning_rate": 9.97563311451829e-07, + "logits/chosen": -23.493140481868807, + "logits/rejected": -19.432473725597365, + "logps/chosen": -1202.7809944152832, + "logps/rejected": -374.36139488220215, + "loss": 0.9666430950164795, + "mean_token_accuracy": 0.7533522471785545, + "num_tokens": 8312496.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.040649566042702645, + "rewards/margins": 0.13564491388387978, + "rewards/rejected": -0.0949953489471227, + "step": 148 + }, + { + "entropy": 1.3850471526384354, + "epoch": 0.09343157234676282, + "grad_norm": 31.51274871826172, + "learning_rate": 9.974586083652757e-07, + "logits/chosen": -18.153076228160014, + "logits/rejected": -16.684204876050245, + "logps/chosen": -845.3816833496094, + "logps/rejected": -408.8254728317261, + "loss": 0.9770760536193848, + "mean_token_accuracy": 0.6601794064044952, + "num_tokens": 8362689.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.018910931539721787, + "rewards/margins": 0.09186943899840117, + "rewards/rejected": -0.11078036949038506, + "step": 149 + }, + { + "entropy": 1.0344312191009521, + "epoch": 0.09405862987929142, + "grad_norm": 37.89040756225586, + "learning_rate": 9.97351708561521e-07, + "logits/chosen": -23.15418936171772, + "logits/rejected": -19.21865452626319, + "logps/chosen": -769.2827854156494, + "logps/rejected": -666.4356002807617, + "loss": 0.9749352335929871, + "mean_token_accuracy": 0.7564720064401627, + "num_tokens": 8420985.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.009593813680112362, + "rewards/margins": 0.10142657672986388, + "rewards/rejected": -0.1110203885473311, + "step": 150 + }, + { + "entropy": 1.175799421966076, + "epoch": 0.09468568741182004, + "grad_norm": 40.132625579833984, + "learning_rate": 9.972426125126207e-07, + "logits/chosen": -21.38852635915963, + "logits/rejected": -18.672145327724344, + "logps/chosen": -1471.715404510498, + "logps/rejected": -486.47255516052246, + "loss": 0.980172336101532, + "mean_token_accuracy": 0.721579059958458, + "num_tokens": 8489644.0, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.03523680008947849, + "rewards/margins": 0.0802477840334177, + "rewards/rejected": -0.11548458551988006, + "step": 151 + }, + { + "entropy": 1.4164262264966965, + "epoch": 0.09531274494434865, + "grad_norm": 27.696420669555664, + "learning_rate": 9.971313207003307e-07, + "logits/chosen": -13.93982345493672, + "logits/rejected": -14.11517608202279, + "logps/chosen": -740.9725232124329, + "logps/rejected": -564.2864780426025, + "loss": 0.9797288179397583, + "mean_token_accuracy": 0.6809670105576515, + "num_tokens": 8537512.0, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.0021716501796618104, + "rewards/margins": 0.08156625635456294, + "rewards/rejected": -0.08373790187761188, + "step": 152 + }, + { + "entropy": 1.263432890176773, + "epoch": 0.09593980247687725, + "grad_norm": 31.5176944732666, + "learning_rate": 9.970178336161016e-07, + "logits/chosen": -16.973323202222108, + "logits/rejected": -19.614638086614725, + "logps/chosen": -711.4418649673462, + "logps/rejected": -372.44454193115234, + "loss": 0.9762611389160156, + "mean_token_accuracy": 0.7092464938759804, + "num_tokens": 8591662.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.0145274052338209, + "rewards/margins": 0.09459308302029967, + "rewards/rejected": -0.10912049002945423, + "step": 153 + }, + { + "entropy": 1.1620648950338364, + "epoch": 0.09656686000940587, + "grad_norm": 30.509321212768555, + "learning_rate": 9.969021517610792e-07, + "logits/chosen": -13.79756665559363, + "logits/rejected": -15.715619490328542, + "logps/chosen": -604.1418495178223, + "logps/rejected": -382.564510345459, + "loss": 0.9641399383544922, + "mean_token_accuracy": 0.726889930665493, + "num_tokens": 8636893.0, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0023322205524891615, + "rewards/margins": 0.14497205498628318, + "rewards/rejected": -0.1473042806610465, + "step": 154 + }, + { + "entropy": 1.1641441136598587, + "epoch": 0.09719391754193447, + "grad_norm": 41.579978942871094, + "learning_rate": 9.967842756461002e-07, + "logits/chosen": -20.334094398749123, + "logits/rejected": -22.011842279254914, + "logps/chosen": -1403.928207397461, + "logps/rejected": -554.649658203125, + "loss": 0.9708878397941589, + "mean_token_accuracy": 0.72523083537817, + "num_tokens": 8717868.0, + "rewards/accuracies": 0.71875, + "rewards/chosen": 0.00970022683031857, + "rewards/margins": 0.11776156397536397, + "rewards/rejected": -0.10806133644655347, + "step": 155 + }, + { + "entropy": 1.2410652860999107, + "epoch": 0.09782097507446308, + "grad_norm": 125.10015106201172, + "learning_rate": 9.966642057916914e-07, + "logits/chosen": -18.70830338783564, + "logits/rejected": -19.97097714965005, + "logps/chosen": -1191.791633605957, + "logps/rejected": -752.3146705627441, + "loss": 0.9524157643318176, + "mean_token_accuracy": 0.7021138742566109, + "num_tokens": 8807483.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.0013844413042534143, + "rewards/margins": 0.19747850112617016, + "rewards/rejected": -0.19886294193565845, + "step": 156 + }, + { + "entropy": 1.029944323003292, + "epoch": 0.0984480326069917, + "grad_norm": 35.78042221069336, + "learning_rate": 9.965419427280668e-07, + "logits/chosen": -23.38625175610134, + "logits/rejected": -20.01934285422626, + "logps/chosen": -1151.3675441741943, + "logps/rejected": -498.4595642089844, + "loss": 0.9675576090812683, + "mean_token_accuracy": 0.7536375895142555, + "num_tokens": 8887962.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.03968535171588883, + "rewards/margins": 0.13185083586722612, + "rewards/rejected": -0.09216548292897642, + "step": 157 + }, + { + "entropy": 1.1238584071397781, + "epoch": 0.0990750901395203, + "grad_norm": 35.993568420410156, + "learning_rate": 9.964174869951254e-07, + "logits/chosen": -13.67164112165123, + "logits/rejected": -19.3182056629662, + "logps/chosen": -457.62388134002686, + "logps/rejected": -487.38970375061035, + "loss": 0.9535990953445435, + "mean_token_accuracy": 0.7514778524637222, + "num_tokens": 8930150.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.004519644775427878, + "rewards/margins": 0.18916182266548276, + "rewards/rejected": -0.19368146173655987, + "step": 158 + }, + { + "entropy": 0.9919542148709297, + "epoch": 0.09970214767204891, + "grad_norm": 35.438880920410156, + "learning_rate": 9.962908391424487e-07, + "logits/chosen": -24.406944580639333, + "logits/rejected": -21.38588868915605, + "logps/chosen": -899.7554340362549, + "logps/rejected": -343.76156425476074, + "loss": 0.9654215574264526, + "mean_token_accuracy": 0.7644909620285034, + "num_tokens": 8991665.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.006943491520360112, + "rewards/margins": 0.14082542480900884, + "rewards/rejected": -0.1338819395750761, + "step": 159 + }, + { + "entropy": 1.0232055224478245, + "epoch": 0.10032920520457753, + "grad_norm": 31.05710220336914, + "learning_rate": 9.961619997292983e-07, + "logits/chosen": -17.982335887836616, + "logits/rejected": -20.611772617446324, + "logps/chosen": -540.6755952835083, + "logps/rejected": -586.5883369445801, + "loss": 0.9546467065811157, + "mean_token_accuracy": 0.7635128647089005, + "num_tokens": 9042126.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0020738002785947174, + "rewards/margins": 0.19118777592666447, + "rewards/rejected": -0.19326158007606864, + "step": 160 + }, + { + "entropy": 1.0774603635072708, + "epoch": 0.10095626273710613, + "grad_norm": 30.445066452026367, + "learning_rate": 9.960309693246134e-07, + "logits/chosen": -16.290460512960674, + "logits/rejected": -18.012763543697705, + "logps/chosen": -576.4469928741455, + "logps/rejected": -511.25352478027344, + "loss": 0.9666774868965149, + "mean_token_accuracy": 0.7399578765034676, + "num_tokens": 9088125.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.01442447875160724, + "rewards/margins": 0.1348671750165522, + "rewards/rejected": -0.14929165970534086, + "step": 161 + }, + { + "entropy": 1.0612441897392273, + "epoch": 0.10158332026963474, + "grad_norm": 37.424896240234375, + "learning_rate": 9.958977485070087e-07, + "logits/chosen": -19.54973934978431, + "logits/rejected": -21.830906008805396, + "logps/chosen": -839.8476104736328, + "logps/rejected": -619.4777336120605, + "loss": 0.9631219506263733, + "mean_token_accuracy": 0.7454415857791901, + "num_tokens": 9147591.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.027470878849271685, + "rewards/margins": 0.14951888378709555, + "rewards/rejected": -0.17698976676911116, + "step": 162 + }, + { + "entropy": 0.8433318734169006, + "epoch": 0.10221037780216335, + "grad_norm": 36.499000549316406, + "learning_rate": 9.957623378647708e-07, + "logits/chosen": -22.97117871622693, + "logits/rejected": -23.077180808340206, + "logps/chosen": -762.8622913360596, + "logps/rejected": -496.52293968200684, + "loss": 0.9598798155784607, + "mean_token_accuracy": 0.7907682359218597, + "num_tokens": 9204397.0, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.008882903959602118, + "rewards/margins": 0.16420582123100758, + "rewards/rejected": -0.15532291657291353, + "step": 163 + }, + { + "entropy": 1.1806890442967415, + "epoch": 0.10283743533469196, + "grad_norm": 45.862754821777344, + "learning_rate": 9.956247379958574e-07, + "logits/chosen": -19.059974763772217, + "logits/rejected": -18.85946041293872, + "logps/chosen": -1369.5862731933594, + "logps/rejected": -567.7572498321533, + "loss": 0.9552696943283081, + "mean_token_accuracy": 0.7177683860063553, + "num_tokens": 9271429.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.03492974303662777, + "rewards/margins": 0.18170535750687122, + "rewards/rejected": -0.14677561353892088, + "step": 164 + }, + { + "entropy": 1.0775589682161808, + "epoch": 0.10346449286722056, + "grad_norm": 29.601449966430664, + "learning_rate": 9.954849495078926e-07, + "logits/chosen": -19.275060518402277, + "logits/rejected": -19.166819179421008, + "logps/chosen": -660.1427116394043, + "logps/rejected": -383.6678161621094, + "loss": 0.958690881729126, + "mean_token_accuracy": 0.7361882030963898, + "num_tokens": 9312257.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.03519108361797407, + "rewards/margins": 0.1664116713218391, + "rewards/rejected": -0.13122059125453234, + "step": 165 + }, + { + "entropy": 1.1816764548420906, + "epoch": 0.10409155039974918, + "grad_norm": 30.252086639404297, + "learning_rate": 9.953429730181652e-07, + "logits/chosen": -16.37164589704853, + "logits/rejected": -15.937881459823364, + "logps/chosen": -442.10915756225586, + "logps/rejected": -464.68394470214844, + "loss": 0.9728706479072571, + "mean_token_accuracy": 0.7251901105046272, + "num_tokens": 9350579.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.025675334269180894, + "rewards/margins": 0.10957675683312118, + "rewards/rejected": -0.13525208923965693, + "step": 166 + }, + { + "entropy": 1.0316157564520836, + "epoch": 0.10471860793227779, + "grad_norm": 39.15115737915039, + "learning_rate": 9.95198809153627e-07, + "logits/chosen": -17.97388225518009, + "logits/rejected": -17.878072330124738, + "logps/chosen": -963.8222351074219, + "logps/rejected": -576.2222747802734, + "loss": 0.9726594686508179, + "mean_token_accuracy": 0.7661974504590034, + "num_tokens": 9420912.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": -0.039964041672647, + "rewards/margins": 0.10996103240177035, + "rewards/rejected": -0.14992507733404636, + "step": 167 + }, + { + "entropy": 0.9888377711176872, + "epoch": 0.10534566546480639, + "grad_norm": 33.86361312866211, + "learning_rate": 9.950524585508875e-07, + "logits/chosen": -21.71417541647197, + "logits/rejected": -21.1007052103952, + "logps/chosen": -570.1340103149414, + "logps/rejected": -569.658317565918, + "loss": 0.965457558631897, + "mean_token_accuracy": 0.7609636634588242, + "num_tokens": 9472501.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.004693154362030327, + "rewards/margins": 0.13876159954816103, + "rewards/rejected": -0.13406844343990088, + "step": 168 + }, + { + "entropy": 1.1498691216111183, + "epoch": 0.10597272299733501, + "grad_norm": 35.466957092285156, + "learning_rate": 9.949039218562137e-07, + "logits/chosen": -23.389534862650184, + "logits/rejected": -20.17964972359958, + "logps/chosen": -982.7829933166504, + "logps/rejected": -726.9613132476807, + "loss": 0.9566149711608887, + "mean_token_accuracy": 0.7251748219132423, + "num_tokens": 9543720.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.003413940197788179, + "rewards/margins": 0.1751425163820386, + "rewards/rejected": -0.17855645716190338, + "step": 169 + }, + { + "entropy": 0.8980679214000702, + "epoch": 0.10659978052986362, + "grad_norm": 40.1265754699707, + "learning_rate": 9.947531997255256e-07, + "logits/chosen": -24.576026786001893, + "logits/rejected": -21.24743446770542, + "logps/chosen": -646.5030174255371, + "logps/rejected": -609.3380126953125, + "loss": 0.946999728679657, + "mean_token_accuracy": 0.8074663206934929, + "num_tokens": 9599025.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.024876392912119627, + "rewards/margins": 0.21698989067226648, + "rewards/rejected": -0.19211349356919527, + "step": 170 + }, + { + "entropy": 1.041010521352291, + "epoch": 0.10722683806239222, + "grad_norm": 27.908248901367188, + "learning_rate": 9.946002928243938e-07, + "logits/chosen": -17.12985163970272, + "logits/rejected": -18.323306945407023, + "logps/chosen": -343.0621109008789, + "logps/rejected": -436.75281524658203, + "loss": 0.9608134031295776, + "mean_token_accuracy": 0.7628256380558014, + "num_tokens": 9635752.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.014670340926386416, + "rewards/margins": 0.15794761665165424, + "rewards/rejected": -0.1726179551333189, + "step": 171 + }, + { + "entropy": 0.9685352593660355, + "epoch": 0.10785389559492084, + "grad_norm": 44.593017578125, + "learning_rate": 9.94445201828037e-07, + "logits/chosen": -25.693284187291074, + "logits/rejected": -20.638393007437365, + "logps/chosen": -1019.63330078125, + "logps/rejected": -641.4812908172607, + "loss": 0.9540131092071533, + "mean_token_accuracy": 0.7625846937298775, + "num_tokens": 9707928.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.051992423948831856, + "rewards/margins": 0.18558591045439243, + "rewards/rejected": -0.23757833428680897, + "step": 172 + }, + { + "entropy": 1.035721518099308, + "epoch": 0.10848095312744944, + "grad_norm": 36.23087692260742, + "learning_rate": 9.942879274213183e-07, + "logits/chosen": -25.004740833477207, + "logits/rejected": -21.763571870178154, + "logps/chosen": -1005.9208698272705, + "logps/rejected": -417.8051815032959, + "loss": 0.9568801522254944, + "mean_token_accuracy": 0.7389874830842018, + "num_tokens": 9762747.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.041769477422349155, + "rewards/margins": 0.17441253177821636, + "rewards/rejected": -0.13264305610209703, + "step": 173 + }, + { + "entropy": 1.2502099946141243, + "epoch": 0.10910801065997805, + "grad_norm": 26.709156036376953, + "learning_rate": 9.941284702987425e-07, + "logits/chosen": -15.603563936993165, + "logits/rejected": -17.525335746655358, + "logps/chosen": -483.58291244506836, + "logps/rejected": -384.8349323272705, + "loss": 0.9618538618087769, + "mean_token_accuracy": 0.706192672252655, + "num_tokens": 9801642.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.01173106487840414, + "rewards/margins": 0.15353717282414436, + "rewards/rejected": -0.1418061126023531, + "step": 174 + }, + { + "entropy": 1.0089079961180687, + "epoch": 0.10973506819250667, + "grad_norm": 34.94546890258789, + "learning_rate": 9.939668311644527e-07, + "logits/chosen": -21.48884231436644, + "logits/rejected": -20.73044133310641, + "logps/chosen": -809.1402287483215, + "logps/rejected": -620.0923671722412, + "loss": 0.9594979286193848, + "mean_token_accuracy": 0.764327310025692, + "num_tokens": 9861136.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": 0.0033170885872095823, + "rewards/margins": 0.1640703366138041, + "rewards/rejected": -0.1607532473281026, + "step": 175 + }, + { + "entropy": 1.134946659207344, + "epoch": 0.11036212572503527, + "grad_norm": 40.56920623779297, + "learning_rate": 9.938030107322283e-07, + "logits/chosen": -20.60884003968547, + "logits/rejected": -21.315807587757533, + "logps/chosen": -1011.4535865783691, + "logps/rejected": -677.3356189727783, + "loss": 0.9518853425979614, + "mean_token_accuracy": 0.7143147438764572, + "num_tokens": 9940296.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.020753700169734657, + "rewards/margins": 0.2017311709932983, + "rewards/rejected": -0.2224848661571741, + "step": 176 + }, + { + "entropy": 1.1413284689188004, + "epoch": 0.11098918325756388, + "grad_norm": 34.17391586303711, + "learning_rate": 9.936370097254803e-07, + "logits/chosen": -18.22259319683248, + "logits/rejected": -16.596884329014106, + "logps/chosen": -748.4427871704102, + "logps/rejected": -550.8146286010742, + "loss": 0.9616140127182007, + "mean_token_accuracy": 0.7435401454567909, + "num_tokens": 9992025.0, + "rewards/accuracies": 0.78125, + "rewards/chosen": 0.020526117412373424, + "rewards/margins": 0.15512995416065678, + "rewards/rejected": -0.1346038356423378, + "step": 177 + }, + { + "entropy": 1.1366123706102371, + "epoch": 0.1116162407900925, + "grad_norm": 42.503440856933594, + "learning_rate": 9.93468828877249e-07, + "logits/chosen": -22.75767775859101, + "logits/rejected": -21.918987652590392, + "logps/chosen": -1206.1099195480347, + "logps/rejected": -610.3419342041016, + "loss": 0.9570979475975037, + "mean_token_accuracy": 0.7211766019463539, + "num_tokens": 10059360.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.0131734365131706, + "rewards/margins": 0.18104426749050617, + "rewards/rejected": -0.1942176972515881, + "step": 178 + }, + { + "entropy": 1.107018142938614, + "epoch": 0.1122432983226211, + "grad_norm": 39.66263198852539, + "learning_rate": 9.93298468930201e-07, + "logits/chosen": -23.260314784767324, + "logits/rejected": -21.11647381786282, + "logps/chosen": -1436.7120723724365, + "logps/rejected": -508.69267654418945, + "loss": 0.9501864910125732, + "mean_token_accuracy": 0.7304199710488319, + "num_tokens": 10132130.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.028971436579013243, + "rewards/margins": 0.20222028624266386, + "rewards/rejected": -0.17324885539710522, + "step": 179 + }, + { + "entropy": 1.021133229136467, + "epoch": 0.1128703558551497, + "grad_norm": 29.389991760253906, + "learning_rate": 9.93125930636625e-07, + "logits/chosen": -18.06104104884531, + "logits/rejected": -19.473802853816192, + "logps/chosen": -539.9062232971191, + "logps/rejected": -461.96774101257324, + "loss": 0.9612171053886414, + "mean_token_accuracy": 0.7571789473295212, + "num_tokens": 10182254.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.02405679877847433, + "rewards/margins": 0.1570815034210682, + "rewards/rejected": -0.1330247106961906, + "step": 180 + }, + { + "entropy": 1.1228575333952904, + "epoch": 0.11349741338767833, + "grad_norm": 31.7424373626709, + "learning_rate": 9.929512147584296e-07, + "logits/chosen": -19.25161771955138, + "logits/rejected": -19.300870767626026, + "logps/chosen": -471.5840435028076, + "logps/rejected": -377.43126487731934, + "loss": 0.9580326080322266, + "mean_token_accuracy": 0.7341411262750626, + "num_tokens": 10231221.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.00470810174010694, + "rewards/margins": 0.1703320173546672, + "rewards/rejected": -0.16562391445040703, + "step": 181 + }, + { + "entropy": 1.2480486258864403, + "epoch": 0.11412447092020693, + "grad_norm": 33.407691955566406, + "learning_rate": 9.927743220671389e-07, + "logits/chosen": -15.707493700178858, + "logits/rejected": -18.14101813743978, + "logps/chosen": -644.9055366516113, + "logps/rejected": -403.51587295532227, + "loss": 0.9577570557594299, + "mean_token_accuracy": 0.695245198905468, + "num_tokens": 10278764.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.00990894460119307, + "rewards/margins": 0.1714292955584824, + "rewards/rejected": -0.18133824318647385, + "step": 182 + }, + { + "entropy": 0.9243633225560188, + "epoch": 0.11475152845273554, + "grad_norm": 34.56248474121094, + "learning_rate": 9.925952533438897e-07, + "logits/chosen": -17.46200834389154, + "logits/rejected": -16.492948873374484, + "logps/chosen": -617.6435585021973, + "logps/rejected": -372.62390518188477, + "loss": 0.948718786239624, + "mean_token_accuracy": 0.7732345685362816, + "num_tokens": 10341069.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": -0.010069805663079023, + "rewards/margins": 0.2102643116377294, + "rewards/rejected": -0.22033411543816328, + "step": 183 + }, + { + "entropy": 0.9938921704888344, + "epoch": 0.11537858598526415, + "grad_norm": 30.834369659423828, + "learning_rate": 9.924140093794277e-07, + "logits/chosen": -19.204634086255844, + "logits/rejected": -19.86160843964533, + "logps/chosen": -502.1897392272949, + "logps/rejected": -514.1468658447266, + "loss": 0.9647988080978394, + "mean_token_accuracy": 0.7703093513846397, + "num_tokens": 10385519.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": 0.006156077841296792, + "rewards/margins": 0.14176303520798683, + "rewards/rejected": -0.13560695759952068, + "step": 184 + }, + { + "entropy": 0.9982673227787018, + "epoch": 0.11600564351779276, + "grad_norm": 33.376678466796875, + "learning_rate": 9.922305909741046e-07, + "logits/chosen": -20.081957609627672, + "logits/rejected": -20.08288046634073, + "logps/chosen": -563.1763916015625, + "logps/rejected": -455.9494934082031, + "loss": 0.940411388874054, + "mean_token_accuracy": 0.7691720277070999, + "num_tokens": 10451619.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.03211024316260591, + "rewards/margins": 0.24388167914003134, + "rewards/rejected": -0.21177144069224596, + "step": 185 + }, + { + "entropy": 1.2043246403336525, + "epoch": 0.11663270105032136, + "grad_norm": 35.30168151855469, + "learning_rate": 9.92044998937874e-07, + "logits/chosen": -15.935400744656912, + "logits/rejected": -17.80423133241425, + "logps/chosen": -507.5350112915039, + "logps/rejected": -506.1169013977051, + "loss": 0.9446260929107666, + "mean_token_accuracy": 0.7261253371834755, + "num_tokens": 10504967.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.011272144154645503, + "rewards/margins": 0.2257434045895934, + "rewards/rejected": -0.21447127033025026, + "step": 186 + }, + { + "entropy": 0.8656372576951981, + "epoch": 0.11725975858284998, + "grad_norm": 36.30891799926758, + "learning_rate": 9.918572340902878e-07, + "logits/chosen": -20.357606409612657, + "logits/rejected": -20.43430765009451, + "logps/chosen": -806.364107131958, + "logps/rejected": -409.4738140106201, + "loss": 0.9610611796379089, + "mean_token_accuracy": 0.7912320047616959, + "num_tokens": 10559810.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.01855588040780276, + "rewards/margins": 0.16032809671014547, + "rewards/rejected": -0.14177220640704036, + "step": 187 + }, + { + "entropy": 1.2360781207680702, + "epoch": 0.11788681611537859, + "grad_norm": 36.44474411010742, + "learning_rate": 9.916672972604927e-07, + "logits/chosen": -21.232208354402246, + "logits/rejected": -20.78721834595899, + "logps/chosen": -872.3558025360107, + "logps/rejected": -562.6851234436035, + "loss": 0.956852912902832, + "mean_token_accuracy": 0.720037579536438, + "num_tokens": 10620148.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": 0.012437668745405972, + "rewards/margins": 0.17596057945047505, + "rewards/rejected": -0.16352291277144104, + "step": 188 + }, + { + "entropy": 1.016458660364151, + "epoch": 0.11851387364790719, + "grad_norm": 29.050859451293945, + "learning_rate": 9.914751892872274e-07, + "logits/chosen": -19.08313772226747, + "logits/rejected": -18.82169734271658, + "logps/chosen": -434.9751796722412, + "logps/rejected": -346.7390670776367, + "loss": 0.9528908133506775, + "mean_token_accuracy": 0.7646205425262451, + "num_tokens": 10670772.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.008685210370458663, + "rewards/margins": 0.19308780319988728, + "rewards/rejected": -0.20177301485091448, + "step": 189 + }, + { + "entropy": 1.1021007224917412, + "epoch": 0.11914093118043581, + "grad_norm": 44.599769592285156, + "learning_rate": 9.91280911018817e-07, + "logits/chosen": -21.5532820720735, + "logits/rejected": -20.40933043838709, + "logps/chosen": -1575.5354919433594, + "logps/rejected": -422.48606395721436, + "loss": 0.941632866859436, + "mean_token_accuracy": 0.7219045013189316, + "num_tokens": 10758488.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.06574870133772492, + "rewards/margins": 0.2363713993690908, + "rewards/rejected": -0.1706227008253336, + "step": 190 + }, + { + "entropy": 1.1249773353338242, + "epoch": 0.11976798871296442, + "grad_norm": 37.6546745300293, + "learning_rate": 9.910844633131712e-07, + "logits/chosen": -17.602501168387953, + "logits/rejected": -18.726467814742886, + "logps/chosen": -641.0849552154541, + "logps/rejected": -560.3315982818604, + "loss": 0.935768723487854, + "mean_token_accuracy": 0.7447556182742119, + "num_tokens": 10819612.0, + "rewards/accuracies": 0.90625, + "rewards/chosen": -0.0099241798161529, + "rewards/margins": 0.26406861934810877, + "rewards/rejected": -0.27399280294775963, + "step": 191 + }, + { + "entropy": 1.0111653581261635, + "epoch": 0.12039504624549302, + "grad_norm": 33.29735565185547, + "learning_rate": 9.908858470377793e-07, + "logits/chosen": -17.99586764005896, + "logits/rejected": -16.641471216812302, + "logps/chosen": -754.2721481323242, + "logps/rejected": -414.00395679473877, + "loss": 0.9521582126617432, + "mean_token_accuracy": 0.7499677836894989, + "num_tokens": 10882013.0, + "rewards/accuracies": 0.96875, + "rewards/chosen": 0.00994637084659189, + "rewards/margins": 0.19447646429762244, + "rewards/rejected": -0.1845300872810185, + "step": 192 + }, + { + "entropy": 1.0526621490716934, + "epoch": 0.12102210377802164, + "grad_norm": 35.13934326171875, + "learning_rate": 9.906850630697066e-07, + "logits/chosen": -17.88888213857138, + "logits/rejected": -17.117701311736873, + "logps/chosen": -626.3125686645508, + "logps/rejected": -390.99532318115234, + "loss": 0.9473018646240234, + "mean_token_accuracy": 0.7478787750005722, + "num_tokens": 10932194.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.013837772188708186, + "rewards/margins": 0.21603740751743317, + "rewards/rejected": -0.22987518273293972, + "step": 193 + }, + { + "entropy": 1.2843475490808487, + "epoch": 0.12164916131055024, + "grad_norm": 33.156471252441406, + "learning_rate": 9.904821122955914e-07, + "logits/chosen": -16.072002517290578, + "logits/rejected": -17.32703283993392, + "logps/chosen": -858.3284645080566, + "logps/rejected": -468.63574600219727, + "loss": 0.957085132598877, + "mean_token_accuracy": 0.6953159794211388, + "num_tokens": 10989560.0, + "rewards/accuracies": 0.84375, + "rewards/chosen": -0.0002469784813001752, + "rewards/margins": 0.174753800034523, + "rewards/rejected": -0.1750007774680853, + "step": 194 + }, + { + "entropy": 0.9444542303681374, + "epoch": 0.12227621884307885, + "grad_norm": 33.4073371887207, + "learning_rate": 9.90276995611639e-07, + "logits/chosen": -19.142496042660422, + "logits/rejected": -21.297342713346943, + "logps/chosen": -639.1740913391113, + "logps/rejected": -354.38869857788086, + "loss": 0.9653919339179993, + "mean_token_accuracy": 0.7545767053961754, + "num_tokens": 11034532.0, + "rewards/accuracies": 0.875, + "rewards/chosen": 0.0012790057808160782, + "rewards/margins": 0.1389640721026808, + "rewards/rejected": -0.13768507237546146, + "step": 195 + }, + { + "entropy": 1.106497973203659, + "epoch": 0.12290327637560747, + "grad_norm": 30.54036521911621, + "learning_rate": 9.900697139236208e-07, + "logits/chosen": -17.157277160405563, + "logits/rejected": -18.095409943495174, + "logps/chosen": -551.8531856536865, + "logps/rejected": -366.1446056365967, + "loss": 0.9476866722106934, + "mean_token_accuracy": 0.7291347607970238, + "num_tokens": 11079071.0, + "rewards/accuracies": 0.9375, + "rewards/chosen": 0.026466174283996224, + "rewards/margins": 0.21335551515221596, + "rewards/rejected": -0.18688933923840523, + "step": 196 + }, + { + "entropy": 1.0959510654211044, + "epoch": 0.12353033390813607, + "grad_norm": 36.96934127807617, + "learning_rate": 9.898602681468674e-07, + "logits/chosen": -19.027734293968553, + "logits/rejected": -19.864234419673636, + "logps/chosen": -954.4189186096191, + "logps/rejected": -566.7688503265381, + "loss": 0.9575784206390381, + "mean_token_accuracy": 0.7271228209137917, + "num_tokens": 11136634.0, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.016784888110123575, + "rewards/margins": 0.17304366454482079, + "rewards/rejected": -0.18982854578644037, + "step": 197 + }, + { + "entropy": 0.8774717822670937, + "epoch": 0.12415739144066468, + "grad_norm": 36.82571792602539, + "learning_rate": 9.896486592062659e-07, + "logits/chosen": -28.647222960438814, + "logits/rejected": -22.060415168768344, + "logps/chosen": -836.5278701782227, + "logps/rejected": -480.254337310791, + "loss": 0.9511983394622803, + "mean_token_accuracy": 0.7737741991877556, + "num_tokens": 11197525.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.008797007380053401, + "rewards/margins": 0.20592515252064914, + "rewards/rejected": -0.21472215885296464, + "step": 198 + }, + { + "entropy": 1.1285011321306229, + "epoch": 0.1247844489731933, + "grad_norm": 42.2977409362793, + "learning_rate": 9.89434888036256e-07, + "logits/chosen": -18.49698845016767, + "logits/rejected": -17.254445681297014, + "logps/chosen": -1127.2061748504639, + "logps/rejected": -956.0397682189941, + "loss": 0.9597939848899841, + "mean_token_accuracy": 0.731256939470768, + "num_tokens": 11292505.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.003246246255002916, + "rewards/margins": 0.16256380267441273, + "rewards/rejected": -0.1658100476488471, + "step": 199 + }, + { + "entropy": 0.9618688002228737, + "epoch": 0.1254115065057219, + "grad_norm": 35.041629791259766, + "learning_rate": 9.892189555808251e-07, + "logits/chosen": -23.16837855908106, + "logits/rejected": -22.01573163400953, + "logps/chosen": -595.0505418777466, + "logps/rejected": -505.8145923614502, + "loss": 0.9585968852043152, + "mean_token_accuracy": 0.7569021657109261, + "num_tokens": 11345306.0, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.027557444758713245, + "rewards/margins": 0.16784314159303904, + "rewards/rejected": -0.19540058448910713, + "step": 200 + } + ], + "logging_steps": 1, + "max_steps": 1595, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.084673057646182e+16, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}