| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.1254115065057219, |
| "eval_steps": 500, |
| "global_step": 200, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.171857863664627, |
| "epoch": 0.0006270575325286095, |
| "grad_norm": 36.12955856323242, |
| "learning_rate": 0.0, |
| "logits/chosen": -16.618545689498976, |
| "logits/rejected": -19.895854701940685, |
| "logps/chosen": -618.4926280975342, |
| "logps/rejected": -544.1511726379395, |
| "loss": 1.0, |
| "mean_token_accuracy": 0.7207779660820961, |
| "num_tokens": 44742.0, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "entropy": 1.3859521597623825, |
| "epoch": 0.001254115065057219, |
| "grad_norm": 35.26237487792969, |
| "learning_rate": 1e-08, |
| "logits/chosen": -16.148485680429634, |
| "logits/rejected": -15.444903538915032, |
| "logps/chosen": -789.9899635314941, |
| "logps/rejected": -629.6869964599609, |
| "loss": 1.0, |
| "mean_token_accuracy": 0.6869945079088211, |
| "num_tokens": 96877.0, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 2 |
| }, |
| { |
| "entropy": 1.1979895681142807, |
| "epoch": 0.0018811725975858284, |
| "grad_norm": 41.65127944946289, |
| "learning_rate": 2e-08, |
| "logits/chosen": -18.29156918013904, |
| "logits/rejected": -19.65692519211939, |
| "logps/chosen": -1245.2266998291016, |
| "logps/rejected": -570.8414497375488, |
| "loss": 1.0063536167144775, |
| "mean_token_accuracy": 0.6914148852229118, |
| "num_tokens": 168920.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.017706198908854276, |
| "rewards/margins": -0.025545110227540135, |
| "rewards/rejected": 0.007838911202270538, |
| "step": 3 |
| }, |
| { |
| "entropy": 1.1795232072472572, |
| "epoch": 0.002508230130114438, |
| "grad_norm": 44.819602966308594, |
| "learning_rate": 3e-08, |
| "logits/chosen": -20.724100083229605, |
| "logits/rejected": -19.704480662994772, |
| "logps/chosen": -1142.84916305542, |
| "logps/rejected": -649.1872062683105, |
| "loss": 1.0045382976531982, |
| "mean_token_accuracy": 0.7210699692368507, |
| "num_tokens": 243342.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.00683369068428874, |
| "rewards/margins": -0.01820890378439799, |
| "rewards/rejected": 0.025042592780664563, |
| "step": 4 |
| }, |
| { |
| "entropy": 1.2270928248763084, |
| "epoch": 0.0031352876626430477, |
| "grad_norm": 32.66579055786133, |
| "learning_rate": 4e-08, |
| "logits/chosen": -16.499294980475682, |
| "logits/rejected": -17.05921506779658, |
| "logps/chosen": -709.8800754547119, |
| "logps/rejected": -586.3846015930176, |
| "loss": 1.0000782012939453, |
| "mean_token_accuracy": 0.7035035043954849, |
| "num_tokens": 295956.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.004917382728308439, |
| "rewards/margins": -0.00028197653591632843, |
| "rewards/rejected": -0.004635405610315502, |
| "step": 5 |
| }, |
| { |
| "entropy": 0.8437139950692654, |
| "epoch": 0.003762345195171657, |
| "grad_norm": 35.4331169128418, |
| "learning_rate": 5e-08, |
| "logits/chosen": -21.37335490682356, |
| "logits/rejected": -21.437480680261856, |
| "logps/chosen": -284.04754638671875, |
| "logps/rejected": -445.4689540863037, |
| "loss": 0.9983432292938232, |
| "mean_token_accuracy": 0.8082250654697418, |
| "num_tokens": 342091.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.01554340252187103, |
| "rewards/margins": 0.0067013868829235435, |
| "rewards/rejected": 0.008842015580739826, |
| "step": 6 |
| }, |
| { |
| "entropy": 1.0479743406176567, |
| "epoch": 0.004389402727700266, |
| "grad_norm": 37.445518493652344, |
| "learning_rate": 6e-08, |
| "logits/chosen": -18.649493652023672, |
| "logits/rejected": -18.4066487416478, |
| "logps/chosen": -805.4484777450562, |
| "logps/rejected": -680.746976852417, |
| "loss": 0.9999387264251709, |
| "mean_token_accuracy": 0.7536595985293388, |
| "num_tokens": 402856.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.002710643340833485, |
| "rewards/margins": 0.00014570658095180988, |
| "rewards/rejected": 0.0025649368471931666, |
| "step": 7 |
| }, |
| { |
| "entropy": 0.9191301837563515, |
| "epoch": 0.005016460260228876, |
| "grad_norm": 38.56687927246094, |
| "learning_rate": 7e-08, |
| "logits/chosen": -23.125927299845493, |
| "logits/rejected": -18.183520901420447, |
| "logps/chosen": -924.5588388442993, |
| "logps/rejected": -373.86534118652344, |
| "loss": 0.9900327324867249, |
| "mean_token_accuracy": 0.7644752189517021, |
| "num_tokens": 466828.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.01720759950694628, |
| "rewards/margins": 0.04020594718167558, |
| "rewards/rejected": -0.02299834630684927, |
| "step": 8 |
| }, |
| { |
| "entropy": 0.9575163498520851, |
| "epoch": 0.005643517792757485, |
| "grad_norm": 40.65217590332031, |
| "learning_rate": 8e-08, |
| "logits/chosen": -21.412901522784708, |
| "logits/rejected": -21.05138834665733, |
| "logps/chosen": -816.7314529418945, |
| "logps/rejected": -663.6353340148926, |
| "loss": 1.0026217699050903, |
| "mean_token_accuracy": 0.771336242556572, |
| "num_tokens": 527777.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.005182235909160227, |
| "rewards/margins": -0.010497396113350987, |
| "rewards/rejected": 0.005315160844475031, |
| "step": 9 |
| }, |
| { |
| "entropy": 1.193418636918068, |
| "epoch": 0.006270575325286095, |
| "grad_norm": 42.18134689331055, |
| "learning_rate": 9e-08, |
| "logits/chosen": -20.675864147312936, |
| "logits/rejected": -20.81446933923923, |
| "logps/chosen": -905.0363216400146, |
| "logps/rejected": -529.3571662902832, |
| "loss": 1.0143247842788696, |
| "mean_token_accuracy": 0.7211106270551682, |
| "num_tokens": 582379.0, |
| "rewards/accuracies": 0.34375, |
| "rewards/chosen": -0.04024720628513023, |
| "rewards/margins": -0.05793563392944634, |
| "rewards/rejected": 0.01768842909950763, |
| "step": 10 |
| }, |
| { |
| "entropy": 1.0013729706406593, |
| "epoch": 0.006897632857814705, |
| "grad_norm": 26.49334144592285, |
| "learning_rate": 1e-07, |
| "logits/chosen": -18.441954576827186, |
| "logits/rejected": -20.784630543895993, |
| "logps/chosen": -532.7153358459473, |
| "logps/rejected": -424.26265144348145, |
| "loss": 0.9981816411018372, |
| "mean_token_accuracy": 0.7596741318702698, |
| "num_tokens": 629715.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.00034768966725096107, |
| "rewards/margins": 0.007268631597980857, |
| "rewards/rejected": -0.006920939544215798, |
| "step": 11 |
| }, |
| { |
| "entropy": 1.0685330666601658, |
| "epoch": 0.007524690390343314, |
| "grad_norm": 34.10297393798828, |
| "learning_rate": 1.0999999999999999e-07, |
| "logits/chosen": -20.029515190554566, |
| "logits/rejected": -21.2224080516646, |
| "logps/chosen": -728.1428518295288, |
| "logps/rejected": -420.13951873779297, |
| "loss": 0.9948853850364685, |
| "mean_token_accuracy": 0.7521882280707359, |
| "num_tokens": 675406.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.010362100991187617, |
| "rewards/margins": 0.020601681084372103, |
| "rewards/rejected": -0.030963782221078873, |
| "step": 12 |
| }, |
| { |
| "entropy": 1.0431296452879906, |
| "epoch": 0.008151747922871924, |
| "grad_norm": 26.776187896728516, |
| "learning_rate": 1.2e-07, |
| "logits/chosen": -19.538335123973816, |
| "logits/rejected": -20.38955248873754, |
| "logps/chosen": -445.5426654815674, |
| "logps/rejected": -416.24525260925293, |
| "loss": 0.997660756111145, |
| "mean_token_accuracy": 0.7694718763232231, |
| "num_tokens": 722623.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.010003616625908762, |
| "rewards/margins": 0.009433707047719508, |
| "rewards/rejected": 0.0005699098983313888, |
| "step": 13 |
| }, |
| { |
| "entropy": 1.328450158238411, |
| "epoch": 0.008778805455400532, |
| "grad_norm": 40.44959259033203, |
| "learning_rate": 1.3e-07, |
| "logits/chosen": -16.50173976340228, |
| "logits/rejected": -18.966777530094504, |
| "logps/chosen": -1086.8657112121582, |
| "logps/rejected": -677.222972869873, |
| "loss": 0.9905858039855957, |
| "mean_token_accuracy": 0.6911511719226837, |
| "num_tokens": 786297.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.018621409428305924, |
| "rewards/margins": 0.03770919982343912, |
| "rewards/rejected": -0.01908779090445023, |
| "step": 14 |
| }, |
| { |
| "entropy": 1.0233074873685837, |
| "epoch": 0.009405862987929142, |
| "grad_norm": 28.252763748168945, |
| "learning_rate": 1.4e-07, |
| "logits/chosen": -17.86138401016042, |
| "logits/rejected": -19.831209346381634, |
| "logps/chosen": -449.9010829925537, |
| "logps/rejected": -492.7903289794922, |
| "loss": 1.0015244483947754, |
| "mean_token_accuracy": 0.7536996155977249, |
| "num_tokens": 837006.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.0008059862302616239, |
| "rewards/margins": -0.006120585021562874, |
| "rewards/rejected": 0.006926572299562395, |
| "step": 15 |
| }, |
| { |
| "entropy": 1.1438564285635948, |
| "epoch": 0.010032920520457752, |
| "grad_norm": 39.57615280151367, |
| "learning_rate": 1.5e-07, |
| "logits/chosen": -17.861874499690074, |
| "logits/rejected": -19.590980097470435, |
| "logps/chosen": -791.6395130157471, |
| "logps/rejected": -567.6202964782715, |
| "loss": 0.9942368268966675, |
| "mean_token_accuracy": 0.7436006516218185, |
| "num_tokens": 896266.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.008153757720720023, |
| "rewards/margins": 0.023181513650342822, |
| "rewards/rejected": -0.015027755754999816, |
| "step": 16 |
| }, |
| { |
| "entropy": 1.1797396391630173, |
| "epoch": 0.010659978052986362, |
| "grad_norm": 31.603225708007812, |
| "learning_rate": 1.6e-07, |
| "logits/chosen": -16.27227861739119, |
| "logits/rejected": -20.263959422316383, |
| "logps/chosen": -687.8001708984375, |
| "logps/rejected": -614.8188095092773, |
| "loss": 1.00355064868927, |
| "mean_token_accuracy": 0.7303128838539124, |
| "num_tokens": 962374.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.0015672160079702735, |
| "rewards/margins": -0.014266991434851661, |
| "rewards/rejected": 0.012699775630608201, |
| "step": 17 |
| }, |
| { |
| "entropy": 0.9821090400218964, |
| "epoch": 0.01128703558551497, |
| "grad_norm": 30.358339309692383, |
| "learning_rate": 1.7000000000000001e-07, |
| "logits/chosen": -19.800772516773982, |
| "logits/rejected": -17.959288041851217, |
| "logps/chosen": -607.3202495574951, |
| "logps/rejected": -534.6925010681152, |
| "loss": 1.002077579498291, |
| "mean_token_accuracy": 0.7688523679971695, |
| "num_tokens": 1016007.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.0275348040740937, |
| "rewards/margins": -0.008413461735472083, |
| "rewards/rejected": -0.019121342920698225, |
| "step": 18 |
| }, |
| { |
| "entropy": 1.2196400687098503, |
| "epoch": 0.01191409311804358, |
| "grad_norm": 37.30662536621094, |
| "learning_rate": 1.8e-07, |
| "logits/chosen": -17.72504789763709, |
| "logits/rejected": -16.12664834863506, |
| "logps/chosen": -813.8953742980957, |
| "logps/rejected": -505.71807956695557, |
| "loss": 1.004115343093872, |
| "mean_token_accuracy": 0.7162602841854095, |
| "num_tokens": 1080373.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.006188086466863751, |
| "rewards/margins": -0.016581419855356216, |
| "rewards/rejected": 0.010393334203399718, |
| "step": 19 |
| }, |
| { |
| "entropy": 1.1595325469970703, |
| "epoch": 0.01254115065057219, |
| "grad_norm": 44.13706970214844, |
| "learning_rate": 1.8999999999999998e-07, |
| "logits/chosen": -19.441370010137216, |
| "logits/rejected": -19.42652252828183, |
| "logps/chosen": -1272.5718240737915, |
| "logps/rejected": -622.7399673461914, |
| "loss": 1.0018830299377441, |
| "mean_token_accuracy": 0.7498048022389412, |
| "num_tokens": 1154517.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.02443491853773594, |
| "rewards/margins": -0.007660747622139752, |
| "rewards/rejected": -0.016774169402197003, |
| "step": 20 |
| }, |
| { |
| "entropy": 1.1010279133915901, |
| "epoch": 0.013168208183100799, |
| "grad_norm": 41.25276184082031, |
| "learning_rate": 2e-07, |
| "logits/chosen": -21.419456943034934, |
| "logits/rejected": -18.181673739978688, |
| "logps/chosen": -728.3854866027832, |
| "logps/rejected": -525.0471820831299, |
| "loss": 0.9977896213531494, |
| "mean_token_accuracy": 0.7623501494526863, |
| "num_tokens": 1212935.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.010096669750055298, |
| "rewards/margins": 0.008904347196221352, |
| "rewards/rejected": 0.0011923184501938522, |
| "step": 21 |
| }, |
| { |
| "entropy": 1.028150089085102, |
| "epoch": 0.01379526571562941, |
| "grad_norm": 36.785099029541016, |
| "learning_rate": 2.0999999999999997e-07, |
| "logits/chosen": -21.92485367743462, |
| "logits/rejected": -18.499413813157435, |
| "logps/chosen": -951.3307018280029, |
| "logps/rejected": -426.83284759521484, |
| "loss": 0.9958165884017944, |
| "mean_token_accuracy": 0.7534473612904549, |
| "num_tokens": 1281702.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.011519648949615657, |
| "rewards/margins": 0.01635956938844174, |
| "rewards/rejected": -0.027879221714101732, |
| "step": 22 |
| }, |
| { |
| "entropy": 0.9120204672217369, |
| "epoch": 0.01442232324815802, |
| "grad_norm": 36.060447692871094, |
| "learning_rate": 2.1999999999999998e-07, |
| "logits/chosen": -20.064261738727723, |
| "logits/rejected": -21.198644490481797, |
| "logps/chosen": -380.3253688812256, |
| "logps/rejected": -429.5263252258301, |
| "loss": 0.9947471022605896, |
| "mean_token_accuracy": 0.7963218614459038, |
| "num_tokens": 1327486.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.007161529851146042, |
| "rewards/margins": 0.02124298777198419, |
| "rewards/rejected": -0.014081457338761538, |
| "step": 23 |
| }, |
| { |
| "entropy": 1.282885067164898, |
| "epoch": 0.015049380780686628, |
| "grad_norm": 29.555295944213867, |
| "learning_rate": 2.3e-07, |
| "logits/chosen": -14.928810725754646, |
| "logits/rejected": -15.783012175933775, |
| "logps/chosen": -594.7181587219238, |
| "logps/rejected": -425.2836685180664, |
| "loss": 1.0085357427597046, |
| "mean_token_accuracy": 0.7083301916718483, |
| "num_tokens": 1377436.0, |
| "rewards/accuracies": 0.28125, |
| "rewards/chosen": -0.018809219647664577, |
| "rewards/margins": -0.03420144016854465, |
| "rewards/rejected": 0.015392220113426447, |
| "step": 24 |
| }, |
| { |
| "entropy": 1.170431300997734, |
| "epoch": 0.015676438313215236, |
| "grad_norm": 30.94621467590332, |
| "learning_rate": 2.4e-07, |
| "logits/chosen": -15.975754306334057, |
| "logits/rejected": -18.080566240707103, |
| "logps/chosen": -616.5587253570557, |
| "logps/rejected": -574.7409896850586, |
| "loss": 0.995916485786438, |
| "mean_token_accuracy": 0.7270784974098206, |
| "num_tokens": 1424961.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": 0.010408571077277884, |
| "rewards/margins": 0.01637996477074921, |
| "rewards/rejected": -0.00597139319870621, |
| "step": 25 |
| }, |
| { |
| "entropy": 0.8303776457905769, |
| "epoch": 0.016303495845743848, |
| "grad_norm": 33.599998474121094, |
| "learning_rate": 2.5e-07, |
| "logits/chosen": -25.26339476179235, |
| "logits/rejected": -21.119193724011815, |
| "logps/chosen": -830.6931796073914, |
| "logps/rejected": -321.0673656463623, |
| "loss": 1.0016303062438965, |
| "mean_token_accuracy": 0.7916704788804054, |
| "num_tokens": 1479087.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.01703651063144207, |
| "rewards/margins": -0.007413207786157727, |
| "rewards/rejected": -0.009623300982639194, |
| "step": 26 |
| }, |
| { |
| "entropy": 0.991417720913887, |
| "epoch": 0.016930553378272456, |
| "grad_norm": 36.997718811035156, |
| "learning_rate": 2.6e-07, |
| "logits/chosen": -21.212152458499872, |
| "logits/rejected": -18.952785907097258, |
| "logps/chosen": -704.5350914001465, |
| "logps/rejected": -370.6160888671875, |
| "loss": 0.9924548864364624, |
| "mean_token_accuracy": 0.7522578835487366, |
| "num_tokens": 1545047.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": 0.017328572226688266, |
| "rewards/margins": 0.03090921661350876, |
| "rewards/rejected": -0.013580642873421311, |
| "step": 27 |
| }, |
| { |
| "entropy": 0.9894383996725082, |
| "epoch": 0.017557610910801064, |
| "grad_norm": 32.38890838623047, |
| "learning_rate": 2.7e-07, |
| "logits/chosen": -15.813927428167577, |
| "logits/rejected": -17.403900338000618, |
| "logps/chosen": -491.92512130737305, |
| "logps/rejected": -495.05503511428833, |
| "loss": 0.9995163083076477, |
| "mean_token_accuracy": 0.7497244253754616, |
| "num_tokens": 1594476.0, |
| "rewards/accuracies": 0.5, |
| "rewards/chosen": -0.008761245291680098, |
| "rewards/margins": 0.001895940862596035, |
| "rewards/rejected": -0.010657186910975724, |
| "step": 28 |
| }, |
| { |
| "entropy": 0.9598497748374939, |
| "epoch": 0.018184668443329676, |
| "grad_norm": 37.41246795654297, |
| "learning_rate": 2.8e-07, |
| "logits/chosen": -19.56551149166531, |
| "logits/rejected": -19.280089414645445, |
| "logps/chosen": -857.0342273712158, |
| "logps/rejected": -602.1430015563965, |
| "loss": 0.9875982403755188, |
| "mean_token_accuracy": 0.7813946753740311, |
| "num_tokens": 1660065.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.013675417518243194, |
| "rewards/margins": 0.05004646920133382, |
| "rewards/rejected": -0.03637105316738598, |
| "step": 29 |
| }, |
| { |
| "entropy": 1.324970230460167, |
| "epoch": 0.018811725975858284, |
| "grad_norm": 36.4524040222168, |
| "learning_rate": 2.9e-07, |
| "logits/chosen": -21.331314482840543, |
| "logits/rejected": -18.526820149021674, |
| "logps/chosen": -1117.674301147461, |
| "logps/rejected": -583.0690956115723, |
| "loss": 1.0027996301651, |
| "mean_token_accuracy": 0.687925897538662, |
| "num_tokens": 1720616.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.020995987113565207, |
| "rewards/margins": -0.01269948878325522, |
| "rewards/rejected": -0.008296501007862389, |
| "step": 30 |
| }, |
| { |
| "entropy": 1.0734140500426292, |
| "epoch": 0.019438783508386896, |
| "grad_norm": 46.66982650756836, |
| "learning_rate": 3e-07, |
| "logits/chosen": -16.946001394868276, |
| "logits/rejected": -16.085758178871664, |
| "logps/chosen": -1352.3280754089355, |
| "logps/rejected": -446.2639093399048, |
| "loss": 0.9967788457870483, |
| "mean_token_accuracy": 0.7191508114337921, |
| "num_tokens": 1811101.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.0069078231463208795, |
| "rewards/margins": 0.012905016890726984, |
| "rewards/rejected": -0.005997193278744817, |
| "step": 31 |
| }, |
| { |
| "entropy": 1.323067456483841, |
| "epoch": 0.020065841040915505, |
| "grad_norm": 43.25249099731445, |
| "learning_rate": 3.1e-07, |
| "logits/chosen": -17.15731645945775, |
| "logits/rejected": -16.6517620046484, |
| "logps/chosen": -1334.5013446807861, |
| "logps/rejected": -645.0475730895996, |
| "loss": 0.9926663637161255, |
| "mean_token_accuracy": 0.7025224380195141, |
| "num_tokens": 1874585.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.0028306127060204744, |
| "rewards/margins": 0.0292252313811332, |
| "rewards/rejected": -0.032055844203568995, |
| "step": 32 |
| }, |
| { |
| "entropy": 1.0061679631471634, |
| "epoch": 0.020692898573444113, |
| "grad_norm": 31.95186996459961, |
| "learning_rate": 3.2e-07, |
| "logits/chosen": -19.57589939389031, |
| "logits/rejected": -19.90894640965601, |
| "logps/chosen": -616.9296951293945, |
| "logps/rejected": -426.7467842102051, |
| "loss": 1.0076781511306763, |
| "mean_token_accuracy": 0.7674555331468582, |
| "num_tokens": 1923915.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": -0.03830282250419259, |
| "rewards/margins": -0.031945616006851196, |
| "rewards/rejected": -0.006357205333188176, |
| "step": 33 |
| }, |
| { |
| "entropy": 0.9171701222658157, |
| "epoch": 0.021319956105972725, |
| "grad_norm": 40.395572662353516, |
| "learning_rate": 3.3e-07, |
| "logits/chosen": -23.769768694096186, |
| "logits/rejected": -21.32001457302996, |
| "logps/chosen": -1274.3026218414307, |
| "logps/rejected": -430.8449001312256, |
| "loss": 0.991915225982666, |
| "mean_token_accuracy": 0.7600029706954956, |
| "num_tokens": 1992060.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.0150325192080345, |
| "rewards/margins": 0.03390131541527808, |
| "rewards/rejected": -0.01886879827361554, |
| "step": 34 |
| }, |
| { |
| "entropy": 1.1749595329165459, |
| "epoch": 0.021947013638501333, |
| "grad_norm": 44.95557403564453, |
| "learning_rate": 3.4000000000000003e-07, |
| "logits/chosen": -20.226573322321272, |
| "logits/rejected": -19.058830108404514, |
| "logps/chosen": -1253.8970890045166, |
| "logps/rejected": -554.8555345535278, |
| "loss": 1.0085124969482422, |
| "mean_token_accuracy": 0.7034121379256248, |
| "num_tokens": 2055149.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.031581409042701125, |
| "rewards/margins": -0.03442110400646925, |
| "rewards/rejected": 0.002839697408489883, |
| "step": 35 |
| }, |
| { |
| "entropy": 1.1094688102602959, |
| "epoch": 0.02257407117102994, |
| "grad_norm": 40.33211135864258, |
| "learning_rate": 3.5e-07, |
| "logits/chosen": -20.24939867823679, |
| "logits/rejected": -20.34034861023941, |
| "logps/chosen": -832.8481521606445, |
| "logps/rejected": -694.2218322753906, |
| "loss": 0.9992198944091797, |
| "mean_token_accuracy": 0.7405928075313568, |
| "num_tokens": 2115443.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.009844484797213227, |
| "rewards/margins": 0.0029849251732230186, |
| "rewards/rejected": -0.01282941095996648, |
| "step": 36 |
| }, |
| { |
| "entropy": 0.9377781078219414, |
| "epoch": 0.023201128703558553, |
| "grad_norm": 34.241668701171875, |
| "learning_rate": 3.6e-07, |
| "logits/chosen": -23.224412647259843, |
| "logits/rejected": -21.117019989134523, |
| "logps/chosen": -721.9840965270996, |
| "logps/rejected": -388.344877243042, |
| "loss": 0.99440598487854, |
| "mean_token_accuracy": 0.7658621892333031, |
| "num_tokens": 2172629.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.016015969216823578, |
| "rewards/margins": 0.022238188714254647, |
| "rewards/rejected": -0.0062222188571467996, |
| "step": 37 |
| }, |
| { |
| "entropy": 1.0614213570952415, |
| "epoch": 0.02382818623608716, |
| "grad_norm": 34.88137435913086, |
| "learning_rate": 3.7e-07, |
| "logits/chosen": -21.41548795553992, |
| "logits/rejected": -18.69351990419123, |
| "logps/chosen": -1103.8941230773926, |
| "logps/rejected": -619.8993282318115, |
| "loss": 1.011991024017334, |
| "mean_token_accuracy": 0.7375566810369492, |
| "num_tokens": 2243025.0, |
| "rewards/accuracies": 0.25, |
| "rewards/chosen": -0.0650398419238627, |
| "rewards/margins": -0.04807211959268898, |
| "rewards/rejected": -0.016967719770036638, |
| "step": 38 |
| }, |
| { |
| "entropy": 1.2264064773917198, |
| "epoch": 0.02445524376861577, |
| "grad_norm": 32.48948669433594, |
| "learning_rate": 3.7999999999999996e-07, |
| "logits/chosen": -18.42769778338538, |
| "logits/rejected": -16.890675790644988, |
| "logps/chosen": -775.1542854309082, |
| "logps/rejected": -521.0977792739868, |
| "loss": 1.0028043985366821, |
| "mean_token_accuracy": 0.7075130566954613, |
| "num_tokens": 2294828.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.009019852615892887, |
| "rewards/margins": -0.011407412588596344, |
| "rewards/rejected": 0.002387559274211526, |
| "step": 39 |
| }, |
| { |
| "entropy": 1.2194309085607529, |
| "epoch": 0.02508230130114438, |
| "grad_norm": 35.85207748413086, |
| "learning_rate": 3.8999999999999997e-07, |
| "logits/chosen": -18.869462390156798, |
| "logits/rejected": -17.76926630663928, |
| "logps/chosen": -866.6771221160889, |
| "logps/rejected": -515.5025405883789, |
| "loss": 1.0078489780426025, |
| "mean_token_accuracy": 0.7282191216945648, |
| "num_tokens": 2359584.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.019920013728551567, |
| "rewards/margins": -0.03206599899567664, |
| "rewards/rejected": 0.012145984219387174, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.9393897280097008, |
| "epoch": 0.02570935883367299, |
| "grad_norm": 29.508663177490234, |
| "learning_rate": 4e-07, |
| "logits/chosen": -19.542580737616923, |
| "logits/rejected": -17.785255602711995, |
| "logps/chosen": -440.89880752563477, |
| "logps/rejected": -531.4820680618286, |
| "loss": 0.9985625147819519, |
| "mean_token_accuracy": 0.7809053212404251, |
| "num_tokens": 2407189.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.0036070493515580893, |
| "rewards/margins": 0.005761435255408287, |
| "rewards/rejected": -0.002154385729227215, |
| "step": 41 |
| }, |
| { |
| "entropy": 1.1858881711959839, |
| "epoch": 0.026336416366201598, |
| "grad_norm": 34.17562484741211, |
| "learning_rate": 4.0999999999999994e-07, |
| "logits/chosen": -20.121897634772164, |
| "logits/rejected": -19.622514667746742, |
| "logps/chosen": -804.0400094985962, |
| "logps/rejected": -586.3046684265137, |
| "loss": 0.9875714778900146, |
| "mean_token_accuracy": 0.7288667857646942, |
| "num_tokens": 2459432.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.0406644003232941, |
| "rewards/margins": 0.05225609429180622, |
| "rewards/rejected": -0.011591696529649198, |
| "step": 42 |
| }, |
| { |
| "entropy": 0.8192418366670609, |
| "epoch": 0.02696347389873021, |
| "grad_norm": 43.9438591003418, |
| "learning_rate": 4.1999999999999995e-07, |
| "logits/chosen": -24.092886788439017, |
| "logits/rejected": -23.664276651461066, |
| "logps/chosen": -584.1518545150757, |
| "logps/rejected": -503.76978302001953, |
| "loss": 0.9815411567687988, |
| "mean_token_accuracy": 0.8075317144393921, |
| "num_tokens": 2512702.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.04232085426338017, |
| "rewards/margins": 0.07526338030584157, |
| "rewards/rejected": -0.032942528603598475, |
| "step": 43 |
| }, |
| { |
| "entropy": 0.9453827887773514, |
| "epoch": 0.02759053143125882, |
| "grad_norm": 35.92549514770508, |
| "learning_rate": 4.2999999999999996e-07, |
| "logits/chosen": -18.338003960845416, |
| "logits/rejected": -19.673824728737838, |
| "logps/chosen": -925.3223133087158, |
| "logps/rejected": -475.0994644165039, |
| "loss": 0.9924615025520325, |
| "mean_token_accuracy": 0.769691713154316, |
| "num_tokens": 2575249.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.02094321296317503, |
| "rewards/margins": 0.03026525373570621, |
| "rewards/rejected": -0.009322041762061417, |
| "step": 44 |
| }, |
| { |
| "entropy": 0.8736219257116318, |
| "epoch": 0.028217588963787427, |
| "grad_norm": 33.97110366821289, |
| "learning_rate": 4.3999999999999997e-07, |
| "logits/chosen": -22.332301111125155, |
| "logits/rejected": -20.832310241019115, |
| "logps/chosen": -462.5166606903076, |
| "logps/rejected": -383.00551986694336, |
| "loss": 1.0048575401306152, |
| "mean_token_accuracy": 0.7859590947628021, |
| "num_tokens": 2621631.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.009766561212018132, |
| "rewards/margins": -0.019437916460447013, |
| "rewards/rejected": 0.009671354899182916, |
| "step": 45 |
| }, |
| { |
| "entropy": 1.02370435744524, |
| "epoch": 0.02884464649631604, |
| "grad_norm": 31.75177764892578, |
| "learning_rate": 4.5e-07, |
| "logits/chosen": -18.283618627987572, |
| "logits/rejected": -18.815319615630727, |
| "logps/chosen": -466.81629753112793, |
| "logps/rejected": -454.2089099884033, |
| "loss": 0.9951180219650269, |
| "mean_token_accuracy": 0.7507254704833031, |
| "num_tokens": 2674748.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.001771716313669458, |
| "rewards/margins": 0.019629769725725055, |
| "rewards/rejected": -0.01785805242252536, |
| "step": 46 |
| }, |
| { |
| "entropy": 0.9373743236064911, |
| "epoch": 0.029471704028844647, |
| "grad_norm": 42.6140251159668, |
| "learning_rate": 4.6e-07, |
| "logits/chosen": -25.528625869209804, |
| "logits/rejected": -21.64687665278925, |
| "logps/chosen": -1166.7054986953735, |
| "logps/rejected": -438.6943473815918, |
| "loss": 0.9925483465194702, |
| "mean_token_accuracy": 0.759131945669651, |
| "num_tokens": 2744660.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.005512146046385169, |
| "rewards/margins": 0.03011180693283677, |
| "rewards/rejected": -0.024599659722298384, |
| "step": 47 |
| }, |
| { |
| "entropy": 0.9783306047320366, |
| "epoch": 0.030098761561373255, |
| "grad_norm": 35.963809967041016, |
| "learning_rate": 4.6999999999999995e-07, |
| "logits/chosen": -19.222043644762906, |
| "logits/rejected": -18.896907946229895, |
| "logps/chosen": -515.0348672866821, |
| "logps/rejected": -419.7729835510254, |
| "loss": 0.9998578429222107, |
| "mean_token_accuracy": 0.792612262070179, |
| "num_tokens": 2790068.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.02343193959677592, |
| "rewards/margins": 0.0005922880955040455, |
| "rewards/rejected": -0.024024227866902947, |
| "step": 48 |
| }, |
| { |
| "entropy": 1.10813407599926, |
| "epoch": 0.030725819093901867, |
| "grad_norm": 35.080543518066406, |
| "learning_rate": 4.8e-07, |
| "logits/chosen": -22.976597678281987, |
| "logits/rejected": -20.831361335827353, |
| "logps/chosen": -826.8215522766113, |
| "logps/rejected": -406.26672554016113, |
| "loss": 1.008866310119629, |
| "mean_token_accuracy": 0.7337941229343414, |
| "num_tokens": 2845743.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.02741124981548637, |
| "rewards/margins": -0.03900438791606575, |
| "rewards/rejected": 0.011593140079639852, |
| "step": 49 |
| }, |
| { |
| "entropy": 1.3011442199349403, |
| "epoch": 0.03135287662643047, |
| "grad_norm": 32.856292724609375, |
| "learning_rate": 4.9e-07, |
| "logits/chosen": -14.496447176077387, |
| "logits/rejected": -17.84699543352937, |
| "logps/chosen": -565.9537868499756, |
| "logps/rejected": -459.75712490081787, |
| "loss": 0.9952901005744934, |
| "mean_token_accuracy": 0.7119247242808342, |
| "num_tokens": 2899233.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.004313211014959961, |
| "rewards/margins": 0.018873692606575787, |
| "rewards/rejected": -0.014560480834916234, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.2501383051276207, |
| "epoch": 0.031979934158959084, |
| "grad_norm": 36.4530143737793, |
| "learning_rate": 5e-07, |
| "logits/chosen": -21.17299309242602, |
| "logits/rejected": -22.865218106484676, |
| "logps/chosen": -866.4957427978516, |
| "logps/rejected": -508.9821949005127, |
| "loss": 1.0008279085159302, |
| "mean_token_accuracy": 0.7066351845860481, |
| "num_tokens": 2952150.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.0076497383415699005, |
| "rewards/margins": -0.003307956736534834, |
| "rewards/rejected": -0.004341780979302712, |
| "step": 51 |
| }, |
| { |
| "entropy": 1.1913169473409653, |
| "epoch": 0.032606991691487695, |
| "grad_norm": 40.79720687866211, |
| "learning_rate": 5.1e-07, |
| "logits/chosen": -17.331403938671592, |
| "logits/rejected": -16.946636435404315, |
| "logps/chosen": -760.1813850402832, |
| "logps/rejected": -703.0359497070312, |
| "loss": 1.0072510242462158, |
| "mean_token_accuracy": 0.7222515121102333, |
| "num_tokens": 3005503.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": -0.013277458609081805, |
| "rewards/margins": -0.029311579186469316, |
| "rewards/rejected": 0.01603412051917985, |
| "step": 52 |
| }, |
| { |
| "entropy": 1.3226243034005165, |
| "epoch": 0.0332340492240163, |
| "grad_norm": 37.33218002319336, |
| "learning_rate": 5.2e-07, |
| "logits/chosen": -19.339108194716523, |
| "logits/rejected": -17.709144071414954, |
| "logps/chosen": -1036.9900550842285, |
| "logps/rejected": -671.3515205383301, |
| "loss": 1.0005102157592773, |
| "mean_token_accuracy": 0.699407085776329, |
| "num_tokens": 3062231.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.01915261917747557, |
| "rewards/margins": -0.002298903651535511, |
| "rewards/rejected": -0.016853714711032808, |
| "step": 53 |
| }, |
| { |
| "entropy": 1.2621377930045128, |
| "epoch": 0.03386110675654491, |
| "grad_norm": 36.86682891845703, |
| "learning_rate": 5.3e-07, |
| "logits/chosen": -23.098386869669465, |
| "logits/rejected": -18.559492736951306, |
| "logps/chosen": -879.0279788970947, |
| "logps/rejected": -472.2311248779297, |
| "loss": 0.9976851940155029, |
| "mean_token_accuracy": 0.687630370259285, |
| "num_tokens": 3118527.0, |
| "rewards/accuracies": 0.53125, |
| "rewards/chosen": -0.01943317288532853, |
| "rewards/margins": 0.009305761312134564, |
| "rewards/rejected": -0.028738934081047773, |
| "step": 54 |
| }, |
| { |
| "entropy": 1.0990462750196457, |
| "epoch": 0.034488164289073524, |
| "grad_norm": 29.28276824951172, |
| "learning_rate": 5.4e-07, |
| "logits/chosen": -17.590637043087355, |
| "logits/rejected": -16.153945977490892, |
| "logps/chosen": -548.3314371109009, |
| "logps/rejected": -324.43446254730225, |
| "loss": 0.9927998781204224, |
| "mean_token_accuracy": 0.7380417436361313, |
| "num_tokens": 3164828.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.007261359598487616, |
| "rewards/margins": 0.02887870534323156, |
| "rewards/rejected": -0.0216173455119133, |
| "step": 55 |
| }, |
| { |
| "entropy": 1.2540519461035728, |
| "epoch": 0.03511522182160213, |
| "grad_norm": 36.696895599365234, |
| "learning_rate": 5.5e-07, |
| "logits/chosen": -15.279084980657903, |
| "logits/rejected": -15.824559065606763, |
| "logps/chosen": -903.7292518615723, |
| "logps/rejected": -541.010986328125, |
| "loss": 0.9985660314559937, |
| "mean_token_accuracy": 0.6998207569122314, |
| "num_tokens": 3233270.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.016322331794071943, |
| "rewards/margins": 0.0056688676122576, |
| "rewards/rejected": -0.021991199755575508, |
| "step": 56 |
| }, |
| { |
| "entropy": 1.0729844830930233, |
| "epoch": 0.03574227935413074, |
| "grad_norm": 26.47355842590332, |
| "learning_rate": 5.6e-07, |
| "logits/chosen": -19.623318897458958, |
| "logits/rejected": -20.258368385656922, |
| "logps/chosen": -469.5618600845337, |
| "logps/rejected": -361.1621437072754, |
| "loss": 1.0013068914413452, |
| "mean_token_accuracy": 0.7484212070703506, |
| "num_tokens": 3268958.0, |
| "rewards/accuracies": 0.4375, |
| "rewards/chosen": 0.004558162530884147, |
| "rewards/margins": -0.005245101172477007, |
| "rewards/rejected": 0.009803264052607119, |
| "step": 57 |
| }, |
| { |
| "entropy": 0.9888226762413979, |
| "epoch": 0.03636933688665935, |
| "grad_norm": 28.04119873046875, |
| "learning_rate": 5.699999999999999e-07, |
| "logits/chosen": -18.97722177970246, |
| "logits/rejected": -17.818980544574217, |
| "logps/chosen": -490.36932277679443, |
| "logps/rejected": -353.6288299560547, |
| "loss": 0.9960744976997375, |
| "mean_token_accuracy": 0.763416476547718, |
| "num_tokens": 3318736.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.0007263210136443377, |
| "rewards/margins": 0.015722022857517004, |
| "rewards/rejected": -0.016448343638330698, |
| "step": 58 |
| }, |
| { |
| "entropy": 1.2886198982596397, |
| "epoch": 0.03699639441918796, |
| "grad_norm": 43.716182708740234, |
| "learning_rate": 5.8e-07, |
| "logits/chosen": -20.97464568747773, |
| "logits/rejected": -18.98481146897743, |
| "logps/chosen": -1585.0814723968506, |
| "logps/rejected": -528.0391731262207, |
| "loss": 0.9945598840713501, |
| "mean_token_accuracy": 0.7001667320728302, |
| "num_tokens": 3386147.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.019920118735171854, |
| "rewards/margins": 0.021774652879685163, |
| "rewards/rejected": -0.0018545325146988034, |
| "step": 59 |
| }, |
| { |
| "entropy": 1.039273351430893, |
| "epoch": 0.03762345195171657, |
| "grad_norm": 27.81415367126465, |
| "learning_rate": 5.9e-07, |
| "logits/chosen": -18.662482620406607, |
| "logits/rejected": -20.1330546037938, |
| "logps/chosen": -365.54387283325195, |
| "logps/rejected": -373.88531494140625, |
| "loss": 0.996242105960846, |
| "mean_token_accuracy": 0.7572688236832619, |
| "num_tokens": 3424538.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.006609153002500534, |
| "rewards/margins": 0.015012014424428344, |
| "rewards/rejected": -0.02162116818362847, |
| "step": 60 |
| }, |
| { |
| "entropy": 1.2042050436139107, |
| "epoch": 0.03825050948424518, |
| "grad_norm": 36.16862869262695, |
| "learning_rate": 6e-07, |
| "logits/chosen": -20.75980830973855, |
| "logits/rejected": -20.60451823594014, |
| "logps/chosen": -1099.3400793075562, |
| "logps/rejected": -421.7832374572754, |
| "loss": 1.0015130043029785, |
| "mean_token_accuracy": 0.7048044949769974, |
| "num_tokens": 3491026.0, |
| "rewards/accuracies": 0.46875, |
| "rewards/chosen": 0.00036664726212620735, |
| "rewards/margins": -0.005983836483210325, |
| "rewards/rejected": 0.00635048293042928, |
| "step": 61 |
| }, |
| { |
| "entropy": 1.0666373148560524, |
| "epoch": 0.03887756701677379, |
| "grad_norm": 43.32762908935547, |
| "learning_rate": 6.1e-07, |
| "logits/chosen": -23.173134701752, |
| "logits/rejected": -20.672318471013675, |
| "logps/chosen": -1242.8998069763184, |
| "logps/rejected": -464.8951187133789, |
| "loss": 0.9945209622383118, |
| "mean_token_accuracy": 0.7305290177464485, |
| "num_tokens": 3555810.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.005398005829192698, |
| "rewards/margins": 0.022236518329009414, |
| "rewards/rejected": -0.016838514362461865, |
| "step": 62 |
| }, |
| { |
| "entropy": 0.7979357466101646, |
| "epoch": 0.0395046245493024, |
| "grad_norm": 30.137588500976562, |
| "learning_rate": 6.2e-07, |
| "logits/chosen": -22.89783292156173, |
| "logits/rejected": -22.199726863036197, |
| "logps/chosen": -536.3147115707397, |
| "logps/rejected": -364.88674545288086, |
| "loss": 0.9864821434020996, |
| "mean_token_accuracy": 0.8112821727991104, |
| "num_tokens": 3604824.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.03518715803511441, |
| "rewards/margins": 0.056509705260396004, |
| "rewards/rejected": -0.02132254181196913, |
| "step": 63 |
| }, |
| { |
| "entropy": 1.1044994071125984, |
| "epoch": 0.04013168208183101, |
| "grad_norm": 29.391481399536133, |
| "learning_rate": 6.3e-07, |
| "logits/chosen": -24.507481146010182, |
| "logits/rejected": -22.21625134174552, |
| "logps/chosen": -702.180121421814, |
| "logps/rejected": -308.05684661865234, |
| "loss": 0.9998461604118347, |
| "mean_token_accuracy": 0.7285795137286186, |
| "num_tokens": 3651250.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.004402894992381334, |
| "rewards/margins": -0.0003644675016403198, |
| "rewards/rejected": -0.004038428800413385, |
| "step": 64 |
| }, |
| { |
| "entropy": 0.8255317062139511, |
| "epoch": 0.04075873961435962, |
| "grad_norm": 36.75194549560547, |
| "learning_rate": 6.4e-07, |
| "logits/chosen": -23.442134502527015, |
| "logits/rejected": -21.24577292120065, |
| "logps/chosen": -481.1418218612671, |
| "logps/rejected": -390.04198455810547, |
| "loss": 0.9808884859085083, |
| "mean_token_accuracy": 0.8040206730365753, |
| "num_tokens": 3700811.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.03725379565730691, |
| "rewards/margins": 0.0775255209300667, |
| "rewards/rejected": -0.040271724574267864, |
| "step": 65 |
| }, |
| { |
| "entropy": 0.9590631648898125, |
| "epoch": 0.041385797146888226, |
| "grad_norm": 31.754987716674805, |
| "learning_rate": 6.5e-07, |
| "logits/chosen": -23.604474652863118, |
| "logits/rejected": -21.18440918134874, |
| "logps/chosen": -892.4836721420288, |
| "logps/rejected": -423.81532859802246, |
| "loss": 0.9878406524658203, |
| "mean_token_accuracy": 0.7679838165640831, |
| "num_tokens": 3752809.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": 0.035222190992499236, |
| "rewards/margins": 0.05575526849133894, |
| "rewards/rejected": -0.020533079630695283, |
| "step": 66 |
| }, |
| { |
| "entropy": 0.9526357278227806, |
| "epoch": 0.04201285467941684, |
| "grad_norm": 38.09070587158203, |
| "learning_rate": 6.6e-07, |
| "logits/chosen": -19.300392282829993, |
| "logits/rejected": -20.892184024152968, |
| "logps/chosen": -767.138952255249, |
| "logps/rejected": -475.6669750213623, |
| "loss": 0.9936155080795288, |
| "mean_token_accuracy": 0.7684061825275421, |
| "num_tokens": 3817303.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.02726819575764239, |
| "rewards/margins": 0.025456703966483474, |
| "rewards/rejected": 0.0018114912672899663, |
| "step": 67 |
| }, |
| { |
| "entropy": 0.7557753957808018, |
| "epoch": 0.04263991221194545, |
| "grad_norm": 31.207439422607422, |
| "learning_rate": 6.7e-07, |
| "logits/chosen": -22.31144983616734, |
| "logits/rejected": -22.303479200481075, |
| "logps/chosen": -516.0994625091553, |
| "logps/rejected": -432.821720123291, |
| "loss": 0.993211030960083, |
| "mean_token_accuracy": 0.8136529326438904, |
| "num_tokens": 3869975.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.008595484774559736, |
| "rewards/margins": 0.027202091412618756, |
| "rewards/rejected": -0.018606607103720307, |
| "step": 68 |
| }, |
| { |
| "entropy": 1.078584998846054, |
| "epoch": 0.043266969744474054, |
| "grad_norm": 35.849456787109375, |
| "learning_rate": 6.800000000000001e-07, |
| "logits/chosen": -20.188257880122976, |
| "logits/rejected": -19.395488505188574, |
| "logps/chosen": -778.6550903320312, |
| "logps/rejected": -460.7582092285156, |
| "loss": 0.9967914819717407, |
| "mean_token_accuracy": 0.739520289003849, |
| "num_tokens": 3917591.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.0018483520834706724, |
| "rewards/margins": 0.013088171719573438, |
| "rewards/rejected": -0.014936523628421128, |
| "step": 69 |
| }, |
| { |
| "entropy": 1.0627146326005459, |
| "epoch": 0.043894027277002666, |
| "grad_norm": 33.957401275634766, |
| "learning_rate": 6.9e-07, |
| "logits/chosen": -19.09365423111348, |
| "logits/rejected": -19.238957994462496, |
| "logps/chosen": -625.0276880264282, |
| "logps/rejected": -460.5406036376953, |
| "loss": 0.9846621155738831, |
| "mean_token_accuracy": 0.7464545965194702, |
| "num_tokens": 3963982.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.005297277413774282, |
| "rewards/margins": 0.06182992341928184, |
| "rewards/rejected": -0.056532644899562, |
| "step": 70 |
| }, |
| { |
| "entropy": 1.0039971619844437, |
| "epoch": 0.04452108480953128, |
| "grad_norm": 36.00166702270508, |
| "learning_rate": 7e-07, |
| "logits/chosen": -21.96269636469623, |
| "logits/rejected": -21.29580632185736, |
| "logps/chosen": -772.8854808807373, |
| "logps/rejected": -615.0897903442383, |
| "loss": 0.9930751323699951, |
| "mean_token_accuracy": 0.7512786686420441, |
| "num_tokens": 4028293.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": 0.0168970461236313, |
| "rewards/margins": 0.02881601534318179, |
| "rewards/rejected": -0.011918970150873065, |
| "step": 71 |
| }, |
| { |
| "entropy": 1.302395537495613, |
| "epoch": 0.04514814234205988, |
| "grad_norm": 39.504661560058594, |
| "learning_rate": 7.1e-07, |
| "logits/chosen": -19.878782312586424, |
| "logits/rejected": -18.000410834982972, |
| "logps/chosen": -1210.3033666610718, |
| "logps/rejected": -607.6138458251953, |
| "loss": 0.9891315698623657, |
| "mean_token_accuracy": 0.703778937458992, |
| "num_tokens": 4089535.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.0096849100664258, |
| "rewards/margins": 0.043962169205769897, |
| "rewards/rejected": -0.034277260303497314, |
| "step": 72 |
| }, |
| { |
| "entropy": 0.9964236989617348, |
| "epoch": 0.045775199874588494, |
| "grad_norm": 33.938472747802734, |
| "learning_rate": 7.2e-07, |
| "logits/chosen": -20.415109094760414, |
| "logits/rejected": -17.26682495783373, |
| "logps/chosen": -721.8188791275024, |
| "logps/rejected": -408.42410469055176, |
| "loss": 0.995058536529541, |
| "mean_token_accuracy": 0.7626092284917831, |
| "num_tokens": 4140075.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.015723853604868054, |
| "rewards/margins": 0.019197183661162853, |
| "rewards/rejected": -0.03492103505413979, |
| "step": 73 |
| }, |
| { |
| "entropy": 1.017537921667099, |
| "epoch": 0.046402257407117106, |
| "grad_norm": 26.6578311920166, |
| "learning_rate": 7.3e-07, |
| "logits/chosen": -15.80474927414226, |
| "logits/rejected": -16.128822298593438, |
| "logps/chosen": -515.7086997032166, |
| "logps/rejected": -239.00895309448242, |
| "loss": 0.9995874762535095, |
| "mean_token_accuracy": 0.7658038064837456, |
| "num_tokens": 4194003.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.008220234914915636, |
| "rewards/margins": 0.0016142040840350091, |
| "rewards/rejected": -0.009834438504185528, |
| "step": 74 |
| }, |
| { |
| "entropy": 0.9735286235809326, |
| "epoch": 0.04702931493964571, |
| "grad_norm": 33.58881759643555, |
| "learning_rate": 7.4e-07, |
| "logits/chosen": -20.946241053605, |
| "logits/rejected": -19.067083934619777, |
| "logps/chosen": -674.0198707580566, |
| "logps/rejected": -419.6971263885498, |
| "loss": 0.991060733795166, |
| "mean_token_accuracy": 0.7505866810679436, |
| "num_tokens": 4244222.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.015050875255838037, |
| "rewards/margins": 0.03578460752032697, |
| "rewards/rejected": -0.020733732322696596, |
| "step": 75 |
| }, |
| { |
| "entropy": 1.137626238167286, |
| "epoch": 0.04765637247217432, |
| "grad_norm": 38.98091125488281, |
| "learning_rate": 7.5e-07, |
| "logits/chosen": -22.829858763000864, |
| "logits/rejected": -19.879969265793594, |
| "logps/chosen": -1177.3832550048828, |
| "logps/rejected": -511.0323143005371, |
| "loss": 0.9940009713172913, |
| "mean_token_accuracy": 0.7326341941952705, |
| "num_tokens": 4313717.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.021500761155039072, |
| "rewards/margins": 0.023731452063657343, |
| "rewards/rejected": -0.0452322136843577, |
| "step": 76 |
| }, |
| { |
| "entropy": 1.2288251742720604, |
| "epoch": 0.048283430004702935, |
| "grad_norm": 28.220809936523438, |
| "learning_rate": 7.599999999999999e-07, |
| "logits/chosen": -17.088404479801554, |
| "logits/rejected": -18.424581121455233, |
| "logps/chosen": -600.109245300293, |
| "logps/rejected": -335.4466133117676, |
| "loss": 1.0012062788009644, |
| "mean_token_accuracy": 0.7148683369159698, |
| "num_tokens": 4358615.0, |
| "rewards/accuracies": 0.375, |
| "rewards/chosen": -0.014499580138362944, |
| "rewards/margins": -0.004860305110923946, |
| "rewards/rejected": -0.00963927514385432, |
| "step": 77 |
| }, |
| { |
| "entropy": 1.0265920907258987, |
| "epoch": 0.04891048753723154, |
| "grad_norm": 34.03989791870117, |
| "learning_rate": 7.699999999999999e-07, |
| "logits/chosen": -22.126280957342644, |
| "logits/rejected": -21.625379783333514, |
| "logps/chosen": -804.5845255851746, |
| "logps/rejected": -453.7879638671875, |
| "loss": 0.9970452785491943, |
| "mean_token_accuracy": 0.7403309643268585, |
| "num_tokens": 4415873.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.008073777018580586, |
| "rewards/margins": 0.011700771981850266, |
| "rewards/rejected": -0.019774550921283662, |
| "step": 78 |
| }, |
| { |
| "entropy": 1.262473076581955, |
| "epoch": 0.04953754506976015, |
| "grad_norm": 36.033348083496094, |
| "learning_rate": 7.799999999999999e-07, |
| "logits/chosen": -18.124917388062837, |
| "logits/rejected": -19.211221946008383, |
| "logps/chosen": -795.0781173706055, |
| "logps/rejected": -577.8308029174805, |
| "loss": 0.9934069514274597, |
| "mean_token_accuracy": 0.7068465352058411, |
| "num_tokens": 4477334.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.004607248993124813, |
| "rewards/margins": 0.026270719012245536, |
| "rewards/rejected": -0.030877968529239297, |
| "step": 79 |
| }, |
| { |
| "entropy": 0.9185331761837006, |
| "epoch": 0.05016460260228876, |
| "grad_norm": 36.832279205322266, |
| "learning_rate": 7.9e-07, |
| "logits/chosen": -20.603378074573, |
| "logits/rejected": -17.056396975980135, |
| "logps/chosen": -758.8550605773926, |
| "logps/rejected": -412.65998458862305, |
| "loss": 0.9883898496627808, |
| "mean_token_accuracy": 0.7726860344409943, |
| "num_tokens": 4533455.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.0016075177991297096, |
| "rewards/margins": 0.04657475184649229, |
| "rewards/rejected": -0.04818226984934881, |
| "step": 80 |
| }, |
| { |
| "entropy": 1.121558502316475, |
| "epoch": 0.05079166013481737, |
| "grad_norm": 36.56794738769531, |
| "learning_rate": 8e-07, |
| "logits/chosen": -20.37862110155874, |
| "logits/rejected": -18.992690204889037, |
| "logps/chosen": -1184.8006420135498, |
| "logps/rejected": -610.0655975341797, |
| "loss": 0.9853536486625671, |
| "mean_token_accuracy": 0.7357284799218178, |
| "num_tokens": 4596652.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.02295642625540495, |
| "rewards/margins": 0.059380507678724825, |
| "rewards/rejected": -0.03642408235464245, |
| "step": 81 |
| }, |
| { |
| "entropy": 1.0517073720693588, |
| "epoch": 0.05141871766734598, |
| "grad_norm": 39.78794860839844, |
| "learning_rate": 8.1e-07, |
| "logits/chosen": -23.220199460872006, |
| "logits/rejected": -21.76251201594231, |
| "logps/chosen": -901.9024753570557, |
| "logps/rejected": -581.7087097167969, |
| "loss": 0.9863370656967163, |
| "mean_token_accuracy": 0.7483935281634331, |
| "num_tokens": 4660574.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.005605014972388744, |
| "rewards/margins": 0.05513616371899843, |
| "rewards/rejected": -0.04953114781528711, |
| "step": 82 |
| }, |
| { |
| "entropy": 1.031716726720333, |
| "epoch": 0.05204577519987459, |
| "grad_norm": 34.147884368896484, |
| "learning_rate": 8.199999999999999e-07, |
| "logits/chosen": -17.43597000771303, |
| "logits/rejected": -17.040958005172556, |
| "logps/chosen": -611.6086521148682, |
| "logps/rejected": -463.6491508483887, |
| "loss": 0.9925582408905029, |
| "mean_token_accuracy": 0.7591038644313812, |
| "num_tokens": 4726456.0, |
| "rewards/accuracies": 0.5625, |
| "rewards/chosen": -0.009094578330405056, |
| "rewards/margins": 0.029864652664400637, |
| "rewards/rejected": -0.03895922936499119, |
| "step": 83 |
| }, |
| { |
| "entropy": 1.1349261552095413, |
| "epoch": 0.052672832732403196, |
| "grad_norm": 32.638126373291016, |
| "learning_rate": 8.299999999999999e-07, |
| "logits/chosen": -17.404521363601013, |
| "logits/rejected": -19.98172508139376, |
| "logps/chosen": -767.6361827850342, |
| "logps/rejected": -448.48921966552734, |
| "loss": 0.9871995449066162, |
| "mean_token_accuracy": 0.7235324308276176, |
| "num_tokens": 4792314.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.017612980911508203, |
| "rewards/margins": 0.05156451647053473, |
| "rewards/rejected": -0.033951534889638424, |
| "step": 84 |
| }, |
| { |
| "entropy": 1.1050493195652962, |
| "epoch": 0.05329989026493181, |
| "grad_norm": 36.08334732055664, |
| "learning_rate": 8.399999999999999e-07, |
| "logits/chosen": -16.500999504676678, |
| "logits/rejected": -18.25587297431137, |
| "logps/chosen": -675.2915000915527, |
| "logps/rejected": -561.0027389526367, |
| "loss": 0.9919638633728027, |
| "mean_token_accuracy": 0.741675615310669, |
| "num_tokens": 4849066.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.0004984733532182872, |
| "rewards/margins": 0.03216184466145933, |
| "rewards/rejected": -0.03166337008588016, |
| "step": 85 |
| }, |
| { |
| "entropy": 1.220378190279007, |
| "epoch": 0.05392694779746042, |
| "grad_norm": 40.76183319091797, |
| "learning_rate": 8.499999999999999e-07, |
| "logits/chosen": -23.90722501767462, |
| "logits/rejected": -22.38695916119651, |
| "logps/chosen": -1048.8524017333984, |
| "logps/rejected": -473.95777893066406, |
| "loss": 0.9880315661430359, |
| "mean_token_accuracy": 0.7109057381749153, |
| "num_tokens": 4923190.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.0222030496224761, |
| "rewards/margins": 0.04874912742525339, |
| "rewards/rejected": -0.07095217774622142, |
| "step": 86 |
| }, |
| { |
| "entropy": 1.2091117054224014, |
| "epoch": 0.054554005329989025, |
| "grad_norm": 35.72580337524414, |
| "learning_rate": 8.599999999999999e-07, |
| "logits/chosen": -20.30170921476879, |
| "logits/rejected": -21.20954152859497, |
| "logps/chosen": -951.2183666229248, |
| "logps/rejected": -513.4668788909912, |
| "loss": 0.9863914251327515, |
| "mean_token_accuracy": 0.7123682126402855, |
| "num_tokens": 4974868.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.026849399670027196, |
| "rewards/margins": 0.05532143288291991, |
| "rewards/rejected": -0.028472037287428975, |
| "step": 87 |
| }, |
| { |
| "entropy": 1.073004886507988, |
| "epoch": 0.05518106286251764, |
| "grad_norm": 36.27262496948242, |
| "learning_rate": 8.699999999999999e-07, |
| "logits/chosen": -19.685747925382, |
| "logits/rejected": -17.267068163974074, |
| "logps/chosen": -578.3146362304688, |
| "logps/rejected": -475.0130271911621, |
| "loss": 0.9875953197479248, |
| "mean_token_accuracy": 0.7410635352134705, |
| "num_tokens": 5037188.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.014228225103579462, |
| "rewards/margins": 0.04966498585417867, |
| "rewards/rejected": -0.03543675900436938, |
| "step": 88 |
| }, |
| { |
| "entropy": 1.056903399527073, |
| "epoch": 0.05580812039504625, |
| "grad_norm": 35.787532806396484, |
| "learning_rate": 8.799999999999999e-07, |
| "logits/chosen": -20.64660484271999, |
| "logits/rejected": -19.71323913593022, |
| "logps/chosen": -527.9435653686523, |
| "logps/rejected": -577.5134410858154, |
| "loss": 0.9897867441177368, |
| "mean_token_accuracy": 0.7574738562107086, |
| "num_tokens": 5086966.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.00012873177183791995, |
| "rewards/margins": 0.04125080001540482, |
| "rewards/rejected": -0.04112206675927155, |
| "step": 89 |
| }, |
| { |
| "entropy": 0.9986204952001572, |
| "epoch": 0.05643517792757485, |
| "grad_norm": 33.426368713378906, |
| "learning_rate": 8.9e-07, |
| "logits/chosen": -18.57544006926955, |
| "logits/rejected": -19.12848221709334, |
| "logps/chosen": -766.3484830856323, |
| "logps/rejected": -476.17732429504395, |
| "loss": 0.9895883798599243, |
| "mean_token_accuracy": 0.7547919675707817, |
| "num_tokens": 5144107.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.013297693920321763, |
| "rewards/margins": 0.04271816968685016, |
| "rewards/rejected": -0.029420473758364096, |
| "step": 90 |
| }, |
| { |
| "entropy": 1.1071998253464699, |
| "epoch": 0.057062235460103465, |
| "grad_norm": 46.52241516113281, |
| "learning_rate": 9e-07, |
| "logits/chosen": -20.750110720341095, |
| "logits/rejected": -20.73160492144149, |
| "logps/chosen": -1105.2754192352295, |
| "logps/rejected": -582.0479850769043, |
| "loss": 0.9801906943321228, |
| "mean_token_accuracy": 0.7243269085884094, |
| "num_tokens": 5211621.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.019272988894954324, |
| "rewards/margins": 0.0801434232853353, |
| "rewards/rejected": -0.06087043380830437, |
| "step": 91 |
| }, |
| { |
| "entropy": 1.1382925510406494, |
| "epoch": 0.05768929299263208, |
| "grad_norm": 37.77758026123047, |
| "learning_rate": 9.1e-07, |
| "logits/chosen": -20.904564962987713, |
| "logits/rejected": -21.018963758425333, |
| "logps/chosen": -640.5741844177246, |
| "logps/rejected": -708.7987327575684, |
| "loss": 0.9877656102180481, |
| "mean_token_accuracy": 0.7462773993611336, |
| "num_tokens": 5263236.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.032095869741169736, |
| "rewards/margins": 0.04855327168479562, |
| "rewards/rejected": -0.08064913935959339, |
| "step": 92 |
| }, |
| { |
| "entropy": 1.1960504204034805, |
| "epoch": 0.05831635052516068, |
| "grad_norm": 27.9012508392334, |
| "learning_rate": 9.2e-07, |
| "logits/chosen": -17.198728228506177, |
| "logits/rejected": -19.989566953539725, |
| "logps/chosen": -581.2822170257568, |
| "logps/rejected": -365.0048427581787, |
| "loss": 0.9875739812850952, |
| "mean_token_accuracy": 0.7414722666144371, |
| "num_tokens": 5301490.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0007973910542204976, |
| "rewards/margins": 0.04991468833759427, |
| "rewards/rejected": -0.04911729716695845, |
| "step": 93 |
| }, |
| { |
| "entropy": 0.9865989461541176, |
| "epoch": 0.058943408057689294, |
| "grad_norm": 31.25222396850586, |
| "learning_rate": 9.3e-07, |
| "logits/chosen": -18.790141123207366, |
| "logits/rejected": -16.14887539333265, |
| "logps/chosen": -577.1131019592285, |
| "logps/rejected": -340.10778999328613, |
| "loss": 0.9858304262161255, |
| "mean_token_accuracy": 0.7623919099569321, |
| "num_tokens": 5361142.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.0011998027912341058, |
| "rewards/margins": 0.057147986139170825, |
| "rewards/rejected": -0.05594818387180567, |
| "step": 94 |
| }, |
| { |
| "entropy": 1.044435366988182, |
| "epoch": 0.059570465590217905, |
| "grad_norm": 26.22032928466797, |
| "learning_rate": 9.399999999999999e-07, |
| "logits/chosen": -17.09738355288199, |
| "logits/rejected": -17.458014048093904, |
| "logps/chosen": -388.39873027801514, |
| "logps/rejected": -326.0696334838867, |
| "loss": 0.9898619055747986, |
| "mean_token_accuracy": 0.7581661641597748, |
| "num_tokens": 5408748.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.0021333397598937154, |
| "rewards/margins": 0.04059030464850366, |
| "rewards/rejected": -0.04272364475764334, |
| "step": 95 |
| }, |
| { |
| "entropy": 0.8697420880198479, |
| "epoch": 0.06019752312274651, |
| "grad_norm": 42.353424072265625, |
| "learning_rate": 9.499999999999999e-07, |
| "logits/chosen": -23.41937699390698, |
| "logits/rejected": -21.70754765415036, |
| "logps/chosen": -1428.7145009040833, |
| "logps/rejected": -443.71881103515625, |
| "loss": 0.9914920926094055, |
| "mean_token_accuracy": 0.7912928014993668, |
| "num_tokens": 5483101.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.011147335055284202, |
| "rewards/margins": 0.045076546957716346, |
| "rewards/rejected": -0.03392921155318618, |
| "step": 96 |
| }, |
| { |
| "entropy": 1.1013405546545982, |
| "epoch": 0.06082458065527512, |
| "grad_norm": 36.10636520385742, |
| "learning_rate": 9.6e-07, |
| "logits/chosen": -19.821280825841903, |
| "logits/rejected": -19.578973804909, |
| "logps/chosen": -985.1940336227417, |
| "logps/rejected": -473.689453125, |
| "loss": 0.983747124671936, |
| "mean_token_accuracy": 0.7495378330349922, |
| "num_tokens": 5534649.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.0021885630558244884, |
| "rewards/margins": 0.06537919119000435, |
| "rewards/rejected": -0.06756775546818972, |
| "step": 97 |
| }, |
| { |
| "entropy": 1.2237722724676132, |
| "epoch": 0.061451638187803734, |
| "grad_norm": 41.582496643066406, |
| "learning_rate": 9.7e-07, |
| "logits/chosen": -20.153029153130383, |
| "logits/rejected": -17.664550339329576, |
| "logps/chosen": -1082.3712921142578, |
| "logps/rejected": -649.8507308959961, |
| "loss": 0.9809252023696899, |
| "mean_token_accuracy": 0.718035951256752, |
| "num_tokens": 5612387.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.02365162328351289, |
| "rewards/margins": 0.07657872815616429, |
| "rewards/rejected": -0.052927102777175605, |
| "step": 98 |
| }, |
| { |
| "entropy": 1.0099660605192184, |
| "epoch": 0.06207869572033234, |
| "grad_norm": 30.545495986938477, |
| "learning_rate": 9.8e-07, |
| "logits/chosen": -19.286327351085962, |
| "logits/rejected": -19.547563799363445, |
| "logps/chosen": -635.9861097335815, |
| "logps/rejected": -379.7309799194336, |
| "loss": 0.9843310117721558, |
| "mean_token_accuracy": 0.7559159845113754, |
| "num_tokens": 5664396.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.005910164851229638, |
| "rewards/margins": 0.06448549684137106, |
| "rewards/rejected": -0.07039566081948578, |
| "step": 99 |
| }, |
| { |
| "entropy": 1.1891107335686684, |
| "epoch": 0.06270575325286094, |
| "grad_norm": 30.441099166870117, |
| "learning_rate": 9.9e-07, |
| "logits/chosen": -16.462167828377467, |
| "logits/rejected": -17.668335453566645, |
| "logps/chosen": -774.8042068481445, |
| "logps/rejected": -352.1152057647705, |
| "loss": 0.9829254150390625, |
| "mean_token_accuracy": 0.723647378385067, |
| "num_tokens": 5712927.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.0013654606882482767, |
| "rewards/margins": 0.0686984455678612, |
| "rewards/rejected": -0.07006390765309334, |
| "step": 100 |
| }, |
| { |
| "entropy": 1.0059744790196419, |
| "epoch": 0.06333281078538956, |
| "grad_norm": 34.044551849365234, |
| "learning_rate": 1e-06, |
| "logits/chosen": -22.569647432957222, |
| "logits/rejected": -20.571769885244017, |
| "logps/chosen": -737.9557514190674, |
| "logps/rejected": -456.0193290710449, |
| "loss": 0.9836858510971069, |
| "mean_token_accuracy": 0.7708024978637695, |
| "num_tokens": 5762779.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.02014523115940392, |
| "rewards/margins": 0.0654695084085688, |
| "rewards/rejected": -0.045324277016334236, |
| "step": 101 |
| }, |
| { |
| "entropy": 1.1221561804413795, |
| "epoch": 0.06395986831791817, |
| "grad_norm": 35.60519790649414, |
| "learning_rate": 9.999988960301596e-07, |
| "logits/chosen": -17.63612561400455, |
| "logits/rejected": -21.225849866079535, |
| "logps/chosen": -851.1271896362305, |
| "logps/rejected": -731.4087066650391, |
| "loss": 0.9757521152496338, |
| "mean_token_accuracy": 0.7322330251336098, |
| "num_tokens": 5820523.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": 0.011595683405175805, |
| "rewards/margins": 0.09945710864849389, |
| "rewards/rejected": -0.08786143315955997, |
| "step": 102 |
| }, |
| { |
| "entropy": 1.033433958888054, |
| "epoch": 0.06458692585044677, |
| "grad_norm": 33.74344253540039, |
| "learning_rate": 9.999955841255138e-07, |
| "logits/chosen": -21.6586000014676, |
| "logits/rejected": -20.241547622800056, |
| "logps/chosen": -748.6037483215332, |
| "logps/rejected": -495.99595642089844, |
| "loss": 0.9870611429214478, |
| "mean_token_accuracy": 0.7368155121803284, |
| "num_tokens": 5870733.0, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.03833825106266886, |
| "rewards/margins": 0.050038286950439215, |
| "rewards/rejected": -0.08837653254158795, |
| "step": 103 |
| }, |
| { |
| "entropy": 1.118708185851574, |
| "epoch": 0.06521398338297539, |
| "grad_norm": 43.72235107421875, |
| "learning_rate": 9.999900643006873e-07, |
| "logits/chosen": -22.952759755082543, |
| "logits/rejected": -18.112035021982514, |
| "logps/chosen": -1293.3799285888672, |
| "logps/rejected": -614.5923004150391, |
| "loss": 0.9773090481758118, |
| "mean_token_accuracy": 0.7410418093204498, |
| "num_tokens": 5941283.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.02265817968873307, |
| "rewards/margins": 0.09182942099869251, |
| "rewards/rejected": -0.06917124305618927, |
| "step": 104 |
| }, |
| { |
| "entropy": 1.3742387741804123, |
| "epoch": 0.065841040915504, |
| "grad_norm": 39.11056137084961, |
| "learning_rate": 9.99982336580055e-07, |
| "logits/chosen": -16.713279247073885, |
| "logits/rejected": -16.98860342760087, |
| "logps/chosen": -950.8361873626709, |
| "logps/rejected": -464.50708770751953, |
| "loss": 0.9773339629173279, |
| "mean_token_accuracy": 0.6922967359423637, |
| "num_tokens": 6000343.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.006588876945897937, |
| "rewards/margins": 0.09067038784269243, |
| "rewards/rejected": -0.08408151054754853, |
| "step": 105 |
| }, |
| { |
| "entropy": 1.1545687764883041, |
| "epoch": 0.0664680984480326, |
| "grad_norm": 34.153663635253906, |
| "learning_rate": 9.99972400997742e-07, |
| "logits/chosen": -20.552279910445396, |
| "logits/rejected": -18.52949584159018, |
| "logps/chosen": -710.0431318283081, |
| "logps/rejected": -350.6925792694092, |
| "loss": 0.982926070690155, |
| "mean_token_accuracy": 0.7238080576062202, |
| "num_tokens": 6047719.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.01743771624751389, |
| "rewards/margins": 0.06860713846981525, |
| "rewards/rejected": -0.05116942140739411, |
| "step": 106 |
| }, |
| { |
| "entropy": 0.8841631710529327, |
| "epoch": 0.06709515598056122, |
| "grad_norm": 30.25786018371582, |
| "learning_rate": 9.999602575976219e-07, |
| "logits/chosen": -16.81424879838689, |
| "logits/rejected": -16.14855911409956, |
| "logps/chosen": -426.7861785888672, |
| "logps/rejected": -396.69312858581543, |
| "loss": 0.979150652885437, |
| "mean_token_accuracy": 0.8006457611918449, |
| "num_tokens": 6096830.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.007483278808649629, |
| "rewards/margins": 0.0836036205291748, |
| "rewards/rejected": -0.07612034154590219, |
| "step": 107 |
| }, |
| { |
| "entropy": 1.0404388830065727, |
| "epoch": 0.06772221351308982, |
| "grad_norm": 33.5556640625, |
| "learning_rate": 9.999459064333188e-07, |
| "logits/chosen": -15.91254721701439, |
| "logits/rejected": -16.29134417675286, |
| "logps/chosen": -601.9646797180176, |
| "logps/rejected": -457.3084487915039, |
| "loss": 0.9777726531028748, |
| "mean_token_accuracy": 0.7538063228130341, |
| "num_tokens": 6145266.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.019957647193223238, |
| "rewards/margins": 0.08928463887423277, |
| "rewards/rejected": -0.0693269899347797, |
| "step": 108 |
| }, |
| { |
| "entropy": 1.2834724336862564, |
| "epoch": 0.06834927104561843, |
| "grad_norm": 34.99026870727539, |
| "learning_rate": 9.999293475682062e-07, |
| "logits/chosen": -17.259286736211884, |
| "logits/rejected": -17.444496267757785, |
| "logps/chosen": -667.7793045043945, |
| "logps/rejected": -577.6125526428223, |
| "loss": 0.9799240231513977, |
| "mean_token_accuracy": 0.6992196813225746, |
| "num_tokens": 6196039.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.005005566752515733, |
| "rewards/margins": 0.08093663223553449, |
| "rewards/rejected": -0.08594219759106636, |
| "step": 109 |
| }, |
| { |
| "entropy": 1.055838204920292, |
| "epoch": 0.06897632857814705, |
| "grad_norm": 33.32623291015625, |
| "learning_rate": 9.999105810754053e-07, |
| "logits/chosen": -18.407450521012187, |
| "logits/rejected": -18.873402077141666, |
| "logps/chosen": -799.7204494476318, |
| "logps/rejected": -477.783242225647, |
| "loss": 0.9677953720092773, |
| "mean_token_accuracy": 0.7510818913578987, |
| "num_tokens": 6263445.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.052083960617892444, |
| "rewards/margins": 0.1307980790734291, |
| "rewards/rejected": -0.07871411880478263, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.9602865129709244, |
| "epoch": 0.06960338611067565, |
| "grad_norm": 28.93109130859375, |
| "learning_rate": 9.99889607037787e-07, |
| "logits/chosen": -16.415104591720436, |
| "logits/rejected": -18.58728532903226, |
| "logps/chosen": -381.7690181732178, |
| "logps/rejected": -416.77470779418945, |
| "loss": 0.9900650382041931, |
| "mean_token_accuracy": 0.7899875342845917, |
| "num_tokens": 6311526.0, |
| "rewards/accuracies": 0.59375, |
| "rewards/chosen": -0.004977155942469835, |
| "rewards/margins": 0.039845253340899944, |
| "rewards/rejected": -0.044822409050539136, |
| "step": 111 |
| }, |
| { |
| "entropy": 0.8192609176039696, |
| "epoch": 0.07023044364320426, |
| "grad_norm": 39.31092071533203, |
| "learning_rate": 9.998664255479704e-07, |
| "logits/chosen": -25.072994460556252, |
| "logits/rejected": -22.42456335487019, |
| "logps/chosen": -1064.658107995987, |
| "logps/rejected": -414.81433868408203, |
| "loss": 0.9966785311698914, |
| "mean_token_accuracy": 0.8144989609718323, |
| "num_tokens": 6373673.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": -0.06160880299285054, |
| "rewards/margins": 0.012114129436668009, |
| "rewards/rejected": -0.0737229329533875, |
| "step": 112 |
| }, |
| { |
| "entropy": 0.9448361806571484, |
| "epoch": 0.07085750117573288, |
| "grad_norm": 39.90821075439453, |
| "learning_rate": 9.99841036708322e-07, |
| "logits/chosen": -23.766030290287702, |
| "logits/rejected": -25.34154914444923, |
| "logps/chosen": -1000.321138381958, |
| "logps/rejected": -514.6866340637207, |
| "loss": 0.9712129831314087, |
| "mean_token_accuracy": 0.7757420614361763, |
| "num_tokens": 6428282.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.034964483114890754, |
| "rewards/margins": 0.11884536314755678, |
| "rewards/rejected": -0.08388087863568217, |
| "step": 113 |
| }, |
| { |
| "entropy": 0.8173889517784119, |
| "epoch": 0.07148455870826148, |
| "grad_norm": 30.850637435913086, |
| "learning_rate": 9.998134406309553e-07, |
| "logits/chosen": -25.008434498378293, |
| "logits/rejected": -22.165182027384386, |
| "logps/chosen": -702.1321802139282, |
| "logps/rejected": -378.8375816345215, |
| "loss": 0.9969829320907593, |
| "mean_token_accuracy": 0.8100090399384499, |
| "num_tokens": 6478216.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.0617737959837541, |
| "rewards/margins": 0.0021084430627524853, |
| "rewards/rejected": -0.06388223776593804, |
| "step": 114 |
| }, |
| { |
| "entropy": 1.0738236010074615, |
| "epoch": 0.07211161624079009, |
| "grad_norm": 37.23641586303711, |
| "learning_rate": 9.997836374377318e-07, |
| "logits/chosen": -21.598642905098632, |
| "logits/rejected": -23.624653024193194, |
| "logps/chosen": -677.4854145050049, |
| "logps/rejected": -513.8848571777344, |
| "loss": 0.9803842306137085, |
| "mean_token_accuracy": 0.7514287456870079, |
| "num_tokens": 6522645.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.0021543916955124587, |
| "rewards/margins": 0.0788510333513841, |
| "rewards/rejected": -0.07669664057902992, |
| "step": 115 |
| }, |
| { |
| "entropy": 1.0687246397137642, |
| "epoch": 0.0727386737733187, |
| "grad_norm": 28.08934783935547, |
| "learning_rate": 9.997516272602588e-07, |
| "logits/chosen": -18.0843256562564, |
| "logits/rejected": -17.27698922154224, |
| "logps/chosen": -591.8647804260254, |
| "logps/rejected": -329.94952392578125, |
| "loss": 0.9886791706085205, |
| "mean_token_accuracy": 0.7329046651721001, |
| "num_tokens": 6565380.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.014423841843381524, |
| "rewards/margins": 0.045475234859623015, |
| "rewards/rejected": -0.05989907821640372, |
| "step": 116 |
| }, |
| { |
| "entropy": 1.2525769509375095, |
| "epoch": 0.07336573130584731, |
| "grad_norm": 32.46056365966797, |
| "learning_rate": 9.997174102398892e-07, |
| "logits/chosen": -19.069984483617883, |
| "logits/rejected": -18.31701463203426, |
| "logps/chosen": -857.891770362854, |
| "logps/rejected": -576.6237850189209, |
| "loss": 0.9878202676773071, |
| "mean_token_accuracy": 0.7090706676244736, |
| "num_tokens": 6621385.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.012965936097316444, |
| "rewards/margins": 0.04882583348080516, |
| "rewards/rejected": -0.06179177016019821, |
| "step": 117 |
| }, |
| { |
| "entropy": 1.0708193145692348, |
| "epoch": 0.07399278883837591, |
| "grad_norm": 33.78789138793945, |
| "learning_rate": 9.996809865277214e-07, |
| "logits/chosen": -17.575822411940983, |
| "logits/rejected": -15.71341222924891, |
| "logps/chosen": -628.3108444213867, |
| "logps/rejected": -381.9364585876465, |
| "loss": 0.984896183013916, |
| "mean_token_accuracy": 0.7500499114394188, |
| "num_tokens": 6671308.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.02526119421236217, |
| "rewards/margins": 0.060638573253527284, |
| "rewards/rejected": -0.0858997656032443, |
| "step": 118 |
| }, |
| { |
| "entropy": 1.1464358419179916, |
| "epoch": 0.07461984637090453, |
| "grad_norm": 29.262676239013672, |
| "learning_rate": 9.996423562845978e-07, |
| "logits/chosen": -18.950197207830822, |
| "logits/rejected": -17.89534831083926, |
| "logps/chosen": -499.1498966217041, |
| "logps/rejected": -397.74127197265625, |
| "loss": 0.9772639274597168, |
| "mean_token_accuracy": 0.7390127554535866, |
| "num_tokens": 6707362.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.014790326356887817, |
| "rewards/margins": 0.09125107247382402, |
| "rewards/rejected": -0.07646074634976685, |
| "step": 119 |
| }, |
| { |
| "entropy": 1.1956902518868446, |
| "epoch": 0.07524690390343314, |
| "grad_norm": 31.247859954833984, |
| "learning_rate": 9.996015196811054e-07, |
| "logits/chosen": -16.430566598689513, |
| "logits/rejected": -16.301428702284802, |
| "logps/chosen": -712.8943672180176, |
| "logps/rejected": -467.7664604187012, |
| "loss": 0.9875868558883667, |
| "mean_token_accuracy": 0.7251445576548576, |
| "num_tokens": 6761518.0, |
| "rewards/accuracies": 0.625, |
| "rewards/chosen": 0.00015118438750505447, |
| "rewards/margins": 0.05001709656789899, |
| "rewards/rejected": -0.04986591334454715, |
| "step": 120 |
| }, |
| { |
| "entropy": 1.04892847687006, |
| "epoch": 0.07587396143596174, |
| "grad_norm": 36.938690185546875, |
| "learning_rate": 9.995584768975734e-07, |
| "logits/chosen": -20.353520038495564, |
| "logits/rejected": -18.411114375084413, |
| "logps/chosen": -787.6422328948975, |
| "logps/rejected": -501.9888114929199, |
| "loss": 0.9638845920562744, |
| "mean_token_accuracy": 0.7510411590337753, |
| "num_tokens": 6814204.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.037587814789731055, |
| "rewards/margins": 0.1457526022568345, |
| "rewards/rejected": -0.10816478868946433, |
| "step": 121 |
| }, |
| { |
| "entropy": 0.911387711763382, |
| "epoch": 0.07650101896849036, |
| "grad_norm": 30.251392364501953, |
| "learning_rate": 9.995132281240734e-07, |
| "logits/chosen": -19.869032639332318, |
| "logits/rejected": -18.91703122960338, |
| "logps/chosen": -582.7686996459961, |
| "logps/rejected": -366.2984085083008, |
| "loss": 0.9788750410079956, |
| "mean_token_accuracy": 0.7888387069106102, |
| "num_tokens": 6861531.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.003442022774834186, |
| "rewards/margins": 0.0853227588813752, |
| "rewards/rejected": -0.08188073709607124, |
| "step": 122 |
| }, |
| { |
| "entropy": 1.0427976697683334, |
| "epoch": 0.07712807650101897, |
| "grad_norm": 36.89279556274414, |
| "learning_rate": 9.994657735604188e-07, |
| "logits/chosen": -24.044872176023844, |
| "logits/rejected": -22.597699453222447, |
| "logps/chosen": -966.8881568908691, |
| "logps/rejected": -505.8887948989868, |
| "loss": 0.9708135724067688, |
| "mean_token_accuracy": 0.7332872226834297, |
| "num_tokens": 6921458.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.012047640164382756, |
| "rewards/margins": 0.11771579552441835, |
| "rewards/rejected": -0.10566815291531384, |
| "step": 123 |
| }, |
| { |
| "entropy": 1.0259113758802414, |
| "epoch": 0.07775513403354758, |
| "grad_norm": 37.096275329589844, |
| "learning_rate": 9.994161134161632e-07, |
| "logits/chosen": -19.591871441281974, |
| "logits/rejected": -20.936406703396575, |
| "logps/chosen": -818.1269016265869, |
| "logps/rejected": -639.8456077575684, |
| "loss": 0.9665595293045044, |
| "mean_token_accuracy": 0.7770890146493912, |
| "num_tokens": 6982937.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.020557800424285233, |
| "rewards/margins": 0.13559715449810028, |
| "rewards/rejected": -0.11503935337532312, |
| "step": 124 |
| }, |
| { |
| "entropy": 1.1710731238126755, |
| "epoch": 0.07838219156607619, |
| "grad_norm": 42.78230667114258, |
| "learning_rate": 9.993642479105997e-07, |
| "logits/chosen": -24.393052790442358, |
| "logits/rejected": -18.192713037735032, |
| "logps/chosen": -1177.7320861816406, |
| "logps/rejected": -522.7479839324951, |
| "loss": 0.9867607355117798, |
| "mean_token_accuracy": 0.712364636361599, |
| "num_tokens": 7046947.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.02724925708025694, |
| "rewards/margins": 0.052655004314146936, |
| "rewards/rejected": -0.07990426244214177, |
| "step": 125 |
| }, |
| { |
| "entropy": 1.1666254326701164, |
| "epoch": 0.0790092490986048, |
| "grad_norm": 32.3548469543457, |
| "learning_rate": 9.993101772727601e-07, |
| "logits/chosen": -17.63798263385176, |
| "logits/rejected": -17.47313740458377, |
| "logps/chosen": -606.1291179656982, |
| "logps/rejected": -430.56521797180176, |
| "loss": 0.9682327508926392, |
| "mean_token_accuracy": 0.7458649724721909, |
| "num_tokens": 7102060.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.0330769574502483, |
| "rewards/margins": 0.13182413554750383, |
| "rewards/rejected": -0.09874717495404184, |
| "step": 126 |
| }, |
| { |
| "entropy": 1.1291334107518196, |
| "epoch": 0.07963630663113341, |
| "grad_norm": 31.756563186645508, |
| "learning_rate": 9.99253901741414e-07, |
| "logits/chosen": -21.307392332824012, |
| "logits/rejected": -20.84085959531398, |
| "logps/chosen": -732.371265411377, |
| "logps/rejected": -503.3115940093994, |
| "loss": 0.9792959690093994, |
| "mean_token_accuracy": 0.7251338735222816, |
| "num_tokens": 7157801.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.018919202295364812, |
| "rewards/margins": 0.08486578240990639, |
| "rewards/rejected": -0.06594657967798412, |
| "step": 127 |
| }, |
| { |
| "entropy": 1.1786841079592705, |
| "epoch": 0.08026336416366202, |
| "grad_norm": 40.29298782348633, |
| "learning_rate": 9.99195421565067e-07, |
| "logits/chosen": -17.529350354706732, |
| "logits/rejected": -20.833092058953017, |
| "logps/chosen": -1033.621503829956, |
| "logps/rejected": -691.7955369949341, |
| "loss": 0.9725984334945679, |
| "mean_token_accuracy": 0.7169393450021744, |
| "num_tokens": 7224323.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.006326087401248515, |
| "rewards/margins": 0.11025936878286302, |
| "rewards/rejected": -0.11658545769751072, |
| "step": 128 |
| }, |
| { |
| "entropy": 1.1065169423818588, |
| "epoch": 0.08089042169619062, |
| "grad_norm": 30.228065490722656, |
| "learning_rate": 9.991347370019609e-07, |
| "logits/chosen": -18.675428265484687, |
| "logits/rejected": -18.02665901699452, |
| "logps/chosen": -738.0616884231567, |
| "logps/rejected": -413.77689933776855, |
| "loss": 0.9663163423538208, |
| "mean_token_accuracy": 0.7391728013753891, |
| "num_tokens": 7273940.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.04217634559608996, |
| "rewards/margins": 0.1416997853666544, |
| "rewards/rejected": -0.09952343860641122, |
| "step": 129 |
| }, |
| { |
| "entropy": 1.0223164483904839, |
| "epoch": 0.08151747922871924, |
| "grad_norm": 36.71884536743164, |
| "learning_rate": 9.990718483200711e-07, |
| "logits/chosen": -24.017658505382084, |
| "logits/rejected": -19.894383210249206, |
| "logps/chosen": -853.0483455657959, |
| "logps/rejected": -410.2704782485962, |
| "loss": 0.9696524739265442, |
| "mean_token_accuracy": 0.7612569332122803, |
| "num_tokens": 7330670.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.022751914570108056, |
| "rewards/margins": 0.12564924580510706, |
| "rewards/rejected": -0.10289733344689012, |
| "step": 130 |
| }, |
| { |
| "entropy": 1.188691645860672, |
| "epoch": 0.08214453676124785, |
| "grad_norm": 45.50562286376953, |
| "learning_rate": 9.990067557971066e-07, |
| "logits/chosen": -20.96738395725814, |
| "logits/rejected": -18.577314710392834, |
| "logps/chosen": -1276.1438827514648, |
| "logps/rejected": -671.3540458679199, |
| "loss": 0.9635441303253174, |
| "mean_token_accuracy": 0.7195305675268173, |
| "num_tokens": 7400528.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.027424347819760442, |
| "rewards/margins": 0.14744434040039778, |
| "rewards/rejected": -0.1200199886225164, |
| "step": 131 |
| }, |
| { |
| "entropy": 0.9636916220188141, |
| "epoch": 0.08277159429377645, |
| "grad_norm": 28.21497344970703, |
| "learning_rate": 9.989394597205082e-07, |
| "logits/chosen": -17.83059240747361, |
| "logits/rejected": -17.95696408960033, |
| "logps/chosen": -485.29137802124023, |
| "logps/rejected": -400.46766471862793, |
| "loss": 0.976447343826294, |
| "mean_token_accuracy": 0.7697746828198433, |
| "num_tokens": 7462922.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.010323267662897706, |
| "rewards/margins": 0.09466969268396497, |
| "rewards/rejected": -0.08434642176143825, |
| "step": 132 |
| }, |
| { |
| "entropy": 0.9702051728963852, |
| "epoch": 0.08339865182630507, |
| "grad_norm": 35.21070861816406, |
| "learning_rate": 9.98869960387447e-07, |
| "logits/chosen": -24.15228929902058, |
| "logits/rejected": -17.62097141810482, |
| "logps/chosen": -707.7485370635986, |
| "logps/rejected": -328.29246520996094, |
| "loss": 0.9750052690505981, |
| "mean_token_accuracy": 0.7517875507473946, |
| "num_tokens": 7516053.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.020313383429311216, |
| "rewards/margins": 0.10041370801627636, |
| "rewards/rejected": -0.08010032417951152, |
| "step": 133 |
| }, |
| { |
| "entropy": 1.080765277147293, |
| "epoch": 0.08402570935883368, |
| "grad_norm": 33.306419372558594, |
| "learning_rate": 9.98798258104824e-07, |
| "logits/chosen": -16.50647893746725, |
| "logits/rejected": -18.638188273587172, |
| "logps/chosen": -394.3768768310547, |
| "logps/rejected": -530.8435726165771, |
| "loss": 0.9684998393058777, |
| "mean_token_accuracy": 0.7539431154727936, |
| "num_tokens": 7560146.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.008416369266342372, |
| "rewards/margins": 0.12770695355720818, |
| "rewards/rejected": -0.13612332614138722, |
| "step": 134 |
| }, |
| { |
| "entropy": 1.0699984654784203, |
| "epoch": 0.08465276689136228, |
| "grad_norm": 29.02651596069336, |
| "learning_rate": 9.987243531892676e-07, |
| "logits/chosen": -17.323968607265364, |
| "logits/rejected": -17.327844017915886, |
| "logps/chosen": -423.5040798187256, |
| "logps/rejected": -401.37097549438477, |
| "loss": 0.9706511497497559, |
| "mean_token_accuracy": 0.7508202418684959, |
| "num_tokens": 7604430.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.010786252591060475, |
| "rewards/margins": 0.11822709790430963, |
| "rewards/rejected": -0.10744084196630865, |
| "step": 135 |
| }, |
| { |
| "entropy": 1.1741472780704498, |
| "epoch": 0.0852798244238909, |
| "grad_norm": 29.518667221069336, |
| "learning_rate": 9.98648245967133e-07, |
| "logits/chosen": -17.057511505199, |
| "logits/rejected": -15.816799192033697, |
| "logps/chosen": -578.4385671615601, |
| "logps/rejected": -470.2986297607422, |
| "loss": 0.9768874645233154, |
| "mean_token_accuracy": 0.7322021424770355, |
| "num_tokens": 7661280.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.018278606235980988, |
| "rewards/margins": 0.09282787260599434, |
| "rewards/rejected": -0.07454926625359803, |
| "step": 136 |
| }, |
| { |
| "entropy": 1.1715349406003952, |
| "epoch": 0.0859068819564195, |
| "grad_norm": 35.8231086730957, |
| "learning_rate": 9.985699367745007e-07, |
| "logits/chosen": -20.207632968359622, |
| "logits/rejected": -18.918033251926897, |
| "logps/chosen": -1063.0584182739258, |
| "logps/rejected": -546.3697662353516, |
| "loss": 0.9809889793395996, |
| "mean_token_accuracy": 0.7140867561101913, |
| "num_tokens": 7723139.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.016168745700269938, |
| "rewards/margins": 0.07651804899796844, |
| "rewards/rejected": -0.09268679865635931, |
| "step": 137 |
| }, |
| { |
| "entropy": 1.0441111102700233, |
| "epoch": 0.08653393948894811, |
| "grad_norm": 32.78045654296875, |
| "learning_rate": 9.984894259571743e-07, |
| "logits/chosen": -19.221359907309978, |
| "logits/rejected": -18.10666979652929, |
| "logps/chosen": -489.88498306274414, |
| "logps/rejected": -398.86695098876953, |
| "loss": 0.9633641242980957, |
| "mean_token_accuracy": 0.7486372217535973, |
| "num_tokens": 7766818.0, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.011141044669784606, |
| "rewards/margins": 0.14769388129934669, |
| "rewards/rejected": -0.1365528372116387, |
| "step": 138 |
| }, |
| { |
| "entropy": 1.1274549514055252, |
| "epoch": 0.08716099702147673, |
| "grad_norm": 28.61403465270996, |
| "learning_rate": 9.984067138706801e-07, |
| "logits/chosen": -20.09536989108541, |
| "logits/rejected": -19.089721130331522, |
| "logps/chosen": -668.453311920166, |
| "logps/rejected": -400.32433700561523, |
| "loss": 0.9768990278244019, |
| "mean_token_accuracy": 0.7357787117362022, |
| "num_tokens": 7805514.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.008357231738045812, |
| "rewards/margins": 0.09283104329369962, |
| "rewards/rejected": -0.08447381528094411, |
| "step": 139 |
| }, |
| { |
| "entropy": 1.072766751050949, |
| "epoch": 0.08778805455400533, |
| "grad_norm": 36.328330993652344, |
| "learning_rate": 9.983218008802647e-07, |
| "logits/chosen": -16.597205351543526, |
| "logits/rejected": -17.38877890508226, |
| "logps/chosen": -845.426420211792, |
| "logps/rejected": -446.5038776397705, |
| "loss": 0.969933271408081, |
| "mean_token_accuracy": 0.7438737154006958, |
| "num_tokens": 7867552.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.004711989313364029, |
| "rewards/margins": 0.12162661645561457, |
| "rewards/rejected": -0.12633860250934958, |
| "step": 140 |
| }, |
| { |
| "entropy": 1.1710950955748558, |
| "epoch": 0.08841511208653394, |
| "grad_norm": 35.29788589477539, |
| "learning_rate": 9.982346873608936e-07, |
| "logits/chosen": -17.457327570012882, |
| "logits/rejected": -17.907635026265908, |
| "logps/chosen": -594.5470962524414, |
| "logps/rejected": -608.6735458374023, |
| "loss": 0.9617944359779358, |
| "mean_token_accuracy": 0.7367723286151886, |
| "num_tokens": 7913056.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.0014770900015719235, |
| "rewards/margins": 0.15395231172442436, |
| "rewards/rejected": -0.15247522108256817, |
| "step": 141 |
| }, |
| { |
| "entropy": 1.0754098296165466, |
| "epoch": 0.08904216961906256, |
| "grad_norm": 35.45695877075195, |
| "learning_rate": 9.981453736972495e-07, |
| "logits/chosen": -19.118562752807563, |
| "logits/rejected": -16.91145482295967, |
| "logps/chosen": -698.3301868438721, |
| "logps/rejected": -497.7921562194824, |
| "loss": 0.9720945954322815, |
| "mean_token_accuracy": 0.7595574334263802, |
| "num_tokens": 7980591.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.01288341250619851, |
| "rewards/margins": 0.11340800416655838, |
| "rewards/rejected": -0.10052459384314716, |
| "step": 142 |
| }, |
| { |
| "entropy": 0.9044511765241623, |
| "epoch": 0.08966922715159116, |
| "grad_norm": 30.17085075378418, |
| "learning_rate": 9.98053860283731e-07, |
| "logits/chosen": -21.207617728808614, |
| "logits/rejected": -20.905731823651053, |
| "logps/chosen": -628.0943202972412, |
| "logps/rejected": -420.4750385284424, |
| "loss": 0.9711873531341553, |
| "mean_token_accuracy": 0.7811061069369316, |
| "num_tokens": 8034884.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.023002621717751026, |
| "rewards/margins": 0.1186411960516125, |
| "rewards/rejected": -0.09563857619650662, |
| "step": 143 |
| }, |
| { |
| "entropy": 1.1677963212132454, |
| "epoch": 0.09029628468411977, |
| "grad_norm": 31.993118286132812, |
| "learning_rate": 9.9796014752445e-07, |
| "logits/chosen": -18.57265198556839, |
| "logits/rejected": -17.995440262698004, |
| "logps/chosen": -658.4226865768433, |
| "logps/rejected": -438.4370231628418, |
| "loss": 0.9786888360977173, |
| "mean_token_accuracy": 0.7285968512296677, |
| "num_tokens": 8082039.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.02490088331978768, |
| "rewards/margins": 0.08490909356623888, |
| "rewards/rejected": -0.10980997630394995, |
| "step": 144 |
| }, |
| { |
| "entropy": 1.0427600964903831, |
| "epoch": 0.09092334221664838, |
| "grad_norm": 31.311111450195312, |
| "learning_rate": 9.978642358332307e-07, |
| "logits/chosen": -19.139753277032113, |
| "logits/rejected": -17.59120642360813, |
| "logps/chosen": -567.9225654602051, |
| "logps/rejected": -465.25221252441406, |
| "loss": 0.9686312675476074, |
| "mean_token_accuracy": 0.7571733519434929, |
| "num_tokens": 8123718.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.008705203887075186, |
| "rewards/margins": 0.12602692190557718, |
| "rewards/rejected": -0.117321718018502, |
| "step": 145 |
| }, |
| { |
| "entropy": 0.8850021287798882, |
| "epoch": 0.09155039974917699, |
| "grad_norm": 37.63220977783203, |
| "learning_rate": 9.97766125633608e-07, |
| "logits/chosen": -19.910044793415654, |
| "logits/rejected": -22.54540787489115, |
| "logps/chosen": -699.3141222000122, |
| "logps/rejected": -466.68331146240234, |
| "loss": 0.9604589939117432, |
| "mean_token_accuracy": 0.7936916127800941, |
| "num_tokens": 8187632.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.003929961123503745, |
| "rewards/margins": 0.168057446135208, |
| "rewards/rejected": -0.17198741296306252, |
| "step": 146 |
| }, |
| { |
| "entropy": 1.0326995179057121, |
| "epoch": 0.0921774572817056, |
| "grad_norm": 37.081241607666016, |
| "learning_rate": 9.976658173588243e-07, |
| "logits/chosen": -21.190682950448576, |
| "logits/rejected": -17.69587061048208, |
| "logps/chosen": -751.942907333374, |
| "logps/rejected": -442.78441619873047, |
| "loss": 0.9758281707763672, |
| "mean_token_accuracy": 0.7523355558514595, |
| "num_tokens": 8243290.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.02370329963741824, |
| "rewards/margins": 0.09733096603304148, |
| "rewards/rejected": -0.12103426642715931, |
| "step": 147 |
| }, |
| { |
| "entropy": 0.9734203144907951, |
| "epoch": 0.09280451481423421, |
| "grad_norm": 36.707550048828125, |
| "learning_rate": 9.97563311451829e-07, |
| "logits/chosen": -23.493140481868807, |
| "logits/rejected": -19.432473725597365, |
| "logps/chosen": -1202.7809944152832, |
| "logps/rejected": -374.36139488220215, |
| "loss": 0.9666430950164795, |
| "mean_token_accuracy": 0.7533522471785545, |
| "num_tokens": 8312496.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.040649566042702645, |
| "rewards/margins": 0.13564491388387978, |
| "rewards/rejected": -0.0949953489471227, |
| "step": 148 |
| }, |
| { |
| "entropy": 1.3850471526384354, |
| "epoch": 0.09343157234676282, |
| "grad_norm": 31.51274871826172, |
| "learning_rate": 9.974586083652757e-07, |
| "logits/chosen": -18.153076228160014, |
| "logits/rejected": -16.684204876050245, |
| "logps/chosen": -845.3816833496094, |
| "logps/rejected": -408.8254728317261, |
| "loss": 0.9770760536193848, |
| "mean_token_accuracy": 0.6601794064044952, |
| "num_tokens": 8362689.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.018910931539721787, |
| "rewards/margins": 0.09186943899840117, |
| "rewards/rejected": -0.11078036949038506, |
| "step": 149 |
| }, |
| { |
| "entropy": 1.0344312191009521, |
| "epoch": 0.09405862987929142, |
| "grad_norm": 37.89040756225586, |
| "learning_rate": 9.97351708561521e-07, |
| "logits/chosen": -23.15418936171772, |
| "logits/rejected": -19.21865452626319, |
| "logps/chosen": -769.2827854156494, |
| "logps/rejected": -666.4356002807617, |
| "loss": 0.9749352335929871, |
| "mean_token_accuracy": 0.7564720064401627, |
| "num_tokens": 8420985.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.009593813680112362, |
| "rewards/margins": 0.10142657672986388, |
| "rewards/rejected": -0.1110203885473311, |
| "step": 150 |
| }, |
| { |
| "entropy": 1.175799421966076, |
| "epoch": 0.09468568741182004, |
| "grad_norm": 40.132625579833984, |
| "learning_rate": 9.972426125126207e-07, |
| "logits/chosen": -21.38852635915963, |
| "logits/rejected": -18.672145327724344, |
| "logps/chosen": -1471.715404510498, |
| "logps/rejected": -486.47255516052246, |
| "loss": 0.980172336101532, |
| "mean_token_accuracy": 0.721579059958458, |
| "num_tokens": 8489644.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.03523680008947849, |
| "rewards/margins": 0.0802477840334177, |
| "rewards/rejected": -0.11548458551988006, |
| "step": 151 |
| }, |
| { |
| "entropy": 1.4164262264966965, |
| "epoch": 0.09531274494434865, |
| "grad_norm": 27.696420669555664, |
| "learning_rate": 9.971313207003307e-07, |
| "logits/chosen": -13.93982345493672, |
| "logits/rejected": -14.11517608202279, |
| "logps/chosen": -740.9725232124329, |
| "logps/rejected": -564.2864780426025, |
| "loss": 0.9797288179397583, |
| "mean_token_accuracy": 0.6809670105576515, |
| "num_tokens": 8537512.0, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.0021716501796618104, |
| "rewards/margins": 0.08156625635456294, |
| "rewards/rejected": -0.08373790187761188, |
| "step": 152 |
| }, |
| { |
| "entropy": 1.263432890176773, |
| "epoch": 0.09593980247687725, |
| "grad_norm": 31.5176944732666, |
| "learning_rate": 9.970178336161016e-07, |
| "logits/chosen": -16.973323202222108, |
| "logits/rejected": -19.614638086614725, |
| "logps/chosen": -711.4418649673462, |
| "logps/rejected": -372.44454193115234, |
| "loss": 0.9762611389160156, |
| "mean_token_accuracy": 0.7092464938759804, |
| "num_tokens": 8591662.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.0145274052338209, |
| "rewards/margins": 0.09459308302029967, |
| "rewards/rejected": -0.10912049002945423, |
| "step": 153 |
| }, |
| { |
| "entropy": 1.1620648950338364, |
| "epoch": 0.09656686000940587, |
| "grad_norm": 30.509321212768555, |
| "learning_rate": 9.969021517610792e-07, |
| "logits/chosen": -13.79756665559363, |
| "logits/rejected": -15.715619490328542, |
| "logps/chosen": -604.1418495178223, |
| "logps/rejected": -382.564510345459, |
| "loss": 0.9641399383544922, |
| "mean_token_accuracy": 0.726889930665493, |
| "num_tokens": 8636893.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.0023322205524891615, |
| "rewards/margins": 0.14497205498628318, |
| "rewards/rejected": -0.1473042806610465, |
| "step": 154 |
| }, |
| { |
| "entropy": 1.1641441136598587, |
| "epoch": 0.09719391754193447, |
| "grad_norm": 41.579978942871094, |
| "learning_rate": 9.967842756461002e-07, |
| "logits/chosen": -20.334094398749123, |
| "logits/rejected": -22.011842279254914, |
| "logps/chosen": -1403.928207397461, |
| "logps/rejected": -554.649658203125, |
| "loss": 0.9708878397941589, |
| "mean_token_accuracy": 0.72523083537817, |
| "num_tokens": 8717868.0, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.00970022683031857, |
| "rewards/margins": 0.11776156397536397, |
| "rewards/rejected": -0.10806133644655347, |
| "step": 155 |
| }, |
| { |
| "entropy": 1.2410652860999107, |
| "epoch": 0.09782097507446308, |
| "grad_norm": 125.10015106201172, |
| "learning_rate": 9.966642057916914e-07, |
| "logits/chosen": -18.70830338783564, |
| "logits/rejected": -19.97097714965005, |
| "logps/chosen": -1191.791633605957, |
| "logps/rejected": -752.3146705627441, |
| "loss": 0.9524157643318176, |
| "mean_token_accuracy": 0.7021138742566109, |
| "num_tokens": 8807483.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.0013844413042534143, |
| "rewards/margins": 0.19747850112617016, |
| "rewards/rejected": -0.19886294193565845, |
| "step": 156 |
| }, |
| { |
| "entropy": 1.029944323003292, |
| "epoch": 0.0984480326069917, |
| "grad_norm": 35.78042221069336, |
| "learning_rate": 9.965419427280668e-07, |
| "logits/chosen": -23.38625175610134, |
| "logits/rejected": -20.01934285422626, |
| "logps/chosen": -1151.3675441741943, |
| "logps/rejected": -498.4595642089844, |
| "loss": 0.9675576090812683, |
| "mean_token_accuracy": 0.7536375895142555, |
| "num_tokens": 8887962.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.03968535171588883, |
| "rewards/margins": 0.13185083586722612, |
| "rewards/rejected": -0.09216548292897642, |
| "step": 157 |
| }, |
| { |
| "entropy": 1.1238584071397781, |
| "epoch": 0.0990750901395203, |
| "grad_norm": 35.993568420410156, |
| "learning_rate": 9.964174869951254e-07, |
| "logits/chosen": -13.67164112165123, |
| "logits/rejected": -19.3182056629662, |
| "logps/chosen": -457.62388134002686, |
| "logps/rejected": -487.38970375061035, |
| "loss": 0.9535990953445435, |
| "mean_token_accuracy": 0.7514778524637222, |
| "num_tokens": 8930150.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.004519644775427878, |
| "rewards/margins": 0.18916182266548276, |
| "rewards/rejected": -0.19368146173655987, |
| "step": 158 |
| }, |
| { |
| "entropy": 0.9919542148709297, |
| "epoch": 0.09970214767204891, |
| "grad_norm": 35.438880920410156, |
| "learning_rate": 9.962908391424487e-07, |
| "logits/chosen": -24.406944580639333, |
| "logits/rejected": -21.38588868915605, |
| "logps/chosen": -899.7554340362549, |
| "logps/rejected": -343.76156425476074, |
| "loss": 0.9654215574264526, |
| "mean_token_accuracy": 0.7644909620285034, |
| "num_tokens": 8991665.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.006943491520360112, |
| "rewards/margins": 0.14082542480900884, |
| "rewards/rejected": -0.1338819395750761, |
| "step": 159 |
| }, |
| { |
| "entropy": 1.0232055224478245, |
| "epoch": 0.10032920520457753, |
| "grad_norm": 31.05710220336914, |
| "learning_rate": 9.961619997292983e-07, |
| "logits/chosen": -17.982335887836616, |
| "logits/rejected": -20.611772617446324, |
| "logps/chosen": -540.6755952835083, |
| "logps/rejected": -586.5883369445801, |
| "loss": 0.9546467065811157, |
| "mean_token_accuracy": 0.7635128647089005, |
| "num_tokens": 9042126.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.0020738002785947174, |
| "rewards/margins": 0.19118777592666447, |
| "rewards/rejected": -0.19326158007606864, |
| "step": 160 |
| }, |
| { |
| "entropy": 1.0774603635072708, |
| "epoch": 0.10095626273710613, |
| "grad_norm": 30.445066452026367, |
| "learning_rate": 9.960309693246134e-07, |
| "logits/chosen": -16.290460512960674, |
| "logits/rejected": -18.012763543697705, |
| "logps/chosen": -576.4469928741455, |
| "logps/rejected": -511.25352478027344, |
| "loss": 0.9666774868965149, |
| "mean_token_accuracy": 0.7399578765034676, |
| "num_tokens": 9088125.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.01442447875160724, |
| "rewards/margins": 0.1348671750165522, |
| "rewards/rejected": -0.14929165970534086, |
| "step": 161 |
| }, |
| { |
| "entropy": 1.0612441897392273, |
| "epoch": 0.10158332026963474, |
| "grad_norm": 37.424896240234375, |
| "learning_rate": 9.958977485070087e-07, |
| "logits/chosen": -19.54973934978431, |
| "logits/rejected": -21.830906008805396, |
| "logps/chosen": -839.8476104736328, |
| "logps/rejected": -619.4777336120605, |
| "loss": 0.9631219506263733, |
| "mean_token_accuracy": 0.7454415857791901, |
| "num_tokens": 9147591.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.027470878849271685, |
| "rewards/margins": 0.14951888378709555, |
| "rewards/rejected": -0.17698976676911116, |
| "step": 162 |
| }, |
| { |
| "entropy": 0.8433318734169006, |
| "epoch": 0.10221037780216335, |
| "grad_norm": 36.499000549316406, |
| "learning_rate": 9.957623378647708e-07, |
| "logits/chosen": -22.97117871622693, |
| "logits/rejected": -23.077180808340206, |
| "logps/chosen": -762.8622913360596, |
| "logps/rejected": -496.52293968200684, |
| "loss": 0.9598798155784607, |
| "mean_token_accuracy": 0.7907682359218597, |
| "num_tokens": 9204397.0, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": 0.008882903959602118, |
| "rewards/margins": 0.16420582123100758, |
| "rewards/rejected": -0.15532291657291353, |
| "step": 163 |
| }, |
| { |
| "entropy": 1.1806890442967415, |
| "epoch": 0.10283743533469196, |
| "grad_norm": 45.862754821777344, |
| "learning_rate": 9.956247379958574e-07, |
| "logits/chosen": -19.059974763772217, |
| "logits/rejected": -18.85946041293872, |
| "logps/chosen": -1369.5862731933594, |
| "logps/rejected": -567.7572498321533, |
| "loss": 0.9552696943283081, |
| "mean_token_accuracy": 0.7177683860063553, |
| "num_tokens": 9271429.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.03492974303662777, |
| "rewards/margins": 0.18170535750687122, |
| "rewards/rejected": -0.14677561353892088, |
| "step": 164 |
| }, |
| { |
| "entropy": 1.0775589682161808, |
| "epoch": 0.10346449286722056, |
| "grad_norm": 29.601449966430664, |
| "learning_rate": 9.954849495078926e-07, |
| "logits/chosen": -19.275060518402277, |
| "logits/rejected": -19.166819179421008, |
| "logps/chosen": -660.1427116394043, |
| "logps/rejected": -383.6678161621094, |
| "loss": 0.958690881729126, |
| "mean_token_accuracy": 0.7361882030963898, |
| "num_tokens": 9312257.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.03519108361797407, |
| "rewards/margins": 0.1664116713218391, |
| "rewards/rejected": -0.13122059125453234, |
| "step": 165 |
| }, |
| { |
| "entropy": 1.1816764548420906, |
| "epoch": 0.10409155039974918, |
| "grad_norm": 30.252086639404297, |
| "learning_rate": 9.953429730181652e-07, |
| "logits/chosen": -16.37164589704853, |
| "logits/rejected": -15.937881459823364, |
| "logps/chosen": -442.10915756225586, |
| "logps/rejected": -464.68394470214844, |
| "loss": 0.9728706479072571, |
| "mean_token_accuracy": 0.7251901105046272, |
| "num_tokens": 9350579.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.025675334269180894, |
| "rewards/margins": 0.10957675683312118, |
| "rewards/rejected": -0.13525208923965693, |
| "step": 166 |
| }, |
| { |
| "entropy": 1.0316157564520836, |
| "epoch": 0.10471860793227779, |
| "grad_norm": 39.15115737915039, |
| "learning_rate": 9.95198809153627e-07, |
| "logits/chosen": -17.97388225518009, |
| "logits/rejected": -17.878072330124738, |
| "logps/chosen": -963.8222351074219, |
| "logps/rejected": -576.2222747802734, |
| "loss": 0.9726594686508179, |
| "mean_token_accuracy": 0.7661974504590034, |
| "num_tokens": 9420912.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.039964041672647, |
| "rewards/margins": 0.10996103240177035, |
| "rewards/rejected": -0.14992507733404636, |
| "step": 167 |
| }, |
| { |
| "entropy": 0.9888377711176872, |
| "epoch": 0.10534566546480639, |
| "grad_norm": 33.86361312866211, |
| "learning_rate": 9.950524585508875e-07, |
| "logits/chosen": -21.71417541647197, |
| "logits/rejected": -21.1007052103952, |
| "logps/chosen": -570.1340103149414, |
| "logps/rejected": -569.658317565918, |
| "loss": 0.965457558631897, |
| "mean_token_accuracy": 0.7609636634588242, |
| "num_tokens": 9472501.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.004693154362030327, |
| "rewards/margins": 0.13876159954816103, |
| "rewards/rejected": -0.13406844343990088, |
| "step": 168 |
| }, |
| { |
| "entropy": 1.1498691216111183, |
| "epoch": 0.10597272299733501, |
| "grad_norm": 35.466957092285156, |
| "learning_rate": 9.949039218562137e-07, |
| "logits/chosen": -23.389534862650184, |
| "logits/rejected": -20.17964972359958, |
| "logps/chosen": -982.7829933166504, |
| "logps/rejected": -726.9613132476807, |
| "loss": 0.9566149711608887, |
| "mean_token_accuracy": 0.7251748219132423, |
| "num_tokens": 9543720.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.003413940197788179, |
| "rewards/margins": 0.1751425163820386, |
| "rewards/rejected": -0.17855645716190338, |
| "step": 169 |
| }, |
| { |
| "entropy": 0.8980679214000702, |
| "epoch": 0.10659978052986362, |
| "grad_norm": 40.1265754699707, |
| "learning_rate": 9.947531997255256e-07, |
| "logits/chosen": -24.576026786001893, |
| "logits/rejected": -21.24743446770542, |
| "logps/chosen": -646.5030174255371, |
| "logps/rejected": -609.3380126953125, |
| "loss": 0.946999728679657, |
| "mean_token_accuracy": 0.8074663206934929, |
| "num_tokens": 9599025.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.024876392912119627, |
| "rewards/margins": 0.21698989067226648, |
| "rewards/rejected": -0.19211349356919527, |
| "step": 170 |
| }, |
| { |
| "entropy": 1.041010521352291, |
| "epoch": 0.10722683806239222, |
| "grad_norm": 27.908248901367188, |
| "learning_rate": 9.946002928243938e-07, |
| "logits/chosen": -17.12985163970272, |
| "logits/rejected": -18.323306945407023, |
| "logps/chosen": -343.0621109008789, |
| "logps/rejected": -436.75281524658203, |
| "loss": 0.9608134031295776, |
| "mean_token_accuracy": 0.7628256380558014, |
| "num_tokens": 9635752.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.014670340926386416, |
| "rewards/margins": 0.15794761665165424, |
| "rewards/rejected": -0.1726179551333189, |
| "step": 171 |
| }, |
| { |
| "entropy": 0.9685352593660355, |
| "epoch": 0.10785389559492084, |
| "grad_norm": 44.593017578125, |
| "learning_rate": 9.94445201828037e-07, |
| "logits/chosen": -25.693284187291074, |
| "logits/rejected": -20.638393007437365, |
| "logps/chosen": -1019.63330078125, |
| "logps/rejected": -641.4812908172607, |
| "loss": 0.9540131092071533, |
| "mean_token_accuracy": 0.7625846937298775, |
| "num_tokens": 9707928.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.051992423948831856, |
| "rewards/margins": 0.18558591045439243, |
| "rewards/rejected": -0.23757833428680897, |
| "step": 172 |
| }, |
| { |
| "entropy": 1.035721518099308, |
| "epoch": 0.10848095312744944, |
| "grad_norm": 36.23087692260742, |
| "learning_rate": 9.942879274213183e-07, |
| "logits/chosen": -25.004740833477207, |
| "logits/rejected": -21.763571870178154, |
| "logps/chosen": -1005.9208698272705, |
| "logps/rejected": -417.8051815032959, |
| "loss": 0.9568801522254944, |
| "mean_token_accuracy": 0.7389874830842018, |
| "num_tokens": 9762747.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.041769477422349155, |
| "rewards/margins": 0.17441253177821636, |
| "rewards/rejected": -0.13264305610209703, |
| "step": 173 |
| }, |
| { |
| "entropy": 1.2502099946141243, |
| "epoch": 0.10910801065997805, |
| "grad_norm": 26.709156036376953, |
| "learning_rate": 9.941284702987425e-07, |
| "logits/chosen": -15.603563936993165, |
| "logits/rejected": -17.525335746655358, |
| "logps/chosen": -483.58291244506836, |
| "logps/rejected": -384.8349323272705, |
| "loss": 0.9618538618087769, |
| "mean_token_accuracy": 0.706192672252655, |
| "num_tokens": 9801642.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.01173106487840414, |
| "rewards/margins": 0.15353717282414436, |
| "rewards/rejected": -0.1418061126023531, |
| "step": 174 |
| }, |
| { |
| "entropy": 1.0089079961180687, |
| "epoch": 0.10973506819250667, |
| "grad_norm": 34.94546890258789, |
| "learning_rate": 9.939668311644527e-07, |
| "logits/chosen": -21.48884231436644, |
| "logits/rejected": -20.73044133310641, |
| "logps/chosen": -809.1402287483215, |
| "logps/rejected": -620.0923671722412, |
| "loss": 0.9594979286193848, |
| "mean_token_accuracy": 0.764327310025692, |
| "num_tokens": 9861136.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": 0.0033170885872095823, |
| "rewards/margins": 0.1640703366138041, |
| "rewards/rejected": -0.1607532473281026, |
| "step": 175 |
| }, |
| { |
| "entropy": 1.134946659207344, |
| "epoch": 0.11036212572503527, |
| "grad_norm": 40.56920623779297, |
| "learning_rate": 9.938030107322283e-07, |
| "logits/chosen": -20.60884003968547, |
| "logits/rejected": -21.315807587757533, |
| "logps/chosen": -1011.4535865783691, |
| "logps/rejected": -677.3356189727783, |
| "loss": 0.9518853425979614, |
| "mean_token_accuracy": 0.7143147438764572, |
| "num_tokens": 9940296.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.020753700169734657, |
| "rewards/margins": 0.2017311709932983, |
| "rewards/rejected": -0.2224848661571741, |
| "step": 176 |
| }, |
| { |
| "entropy": 1.1413284689188004, |
| "epoch": 0.11098918325756388, |
| "grad_norm": 34.17391586303711, |
| "learning_rate": 9.936370097254803e-07, |
| "logits/chosen": -18.22259319683248, |
| "logits/rejected": -16.596884329014106, |
| "logps/chosen": -748.4427871704102, |
| "logps/rejected": -550.8146286010742, |
| "loss": 0.9616140127182007, |
| "mean_token_accuracy": 0.7435401454567909, |
| "num_tokens": 9992025.0, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.020526117412373424, |
| "rewards/margins": 0.15512995416065678, |
| "rewards/rejected": -0.1346038356423378, |
| "step": 177 |
| }, |
| { |
| "entropy": 1.1366123706102371, |
| "epoch": 0.1116162407900925, |
| "grad_norm": 42.503440856933594, |
| "learning_rate": 9.93468828877249e-07, |
| "logits/chosen": -22.75767775859101, |
| "logits/rejected": -21.918987652590392, |
| "logps/chosen": -1206.1099195480347, |
| "logps/rejected": -610.3419342041016, |
| "loss": 0.9570979475975037, |
| "mean_token_accuracy": 0.7211766019463539, |
| "num_tokens": 10059360.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.0131734365131706, |
| "rewards/margins": 0.18104426749050617, |
| "rewards/rejected": -0.1942176972515881, |
| "step": 178 |
| }, |
| { |
| "entropy": 1.107018142938614, |
| "epoch": 0.1122432983226211, |
| "grad_norm": 39.66263198852539, |
| "learning_rate": 9.93298468930201e-07, |
| "logits/chosen": -23.260314784767324, |
| "logits/rejected": -21.11647381786282, |
| "logps/chosen": -1436.7120723724365, |
| "logps/rejected": -508.69267654418945, |
| "loss": 0.9501864910125732, |
| "mean_token_accuracy": 0.7304199710488319, |
| "num_tokens": 10132130.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.028971436579013243, |
| "rewards/margins": 0.20222028624266386, |
| "rewards/rejected": -0.17324885539710522, |
| "step": 179 |
| }, |
| { |
| "entropy": 1.021133229136467, |
| "epoch": 0.1128703558551497, |
| "grad_norm": 29.389991760253906, |
| "learning_rate": 9.93125930636625e-07, |
| "logits/chosen": -18.06104104884531, |
| "logits/rejected": -19.473802853816192, |
| "logps/chosen": -539.9062232971191, |
| "logps/rejected": -461.96774101257324, |
| "loss": 0.9612171053886414, |
| "mean_token_accuracy": 0.7571789473295212, |
| "num_tokens": 10182254.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.02405679877847433, |
| "rewards/margins": 0.1570815034210682, |
| "rewards/rejected": -0.1330247106961906, |
| "step": 180 |
| }, |
| { |
| "entropy": 1.1228575333952904, |
| "epoch": 0.11349741338767833, |
| "grad_norm": 31.7424373626709, |
| "learning_rate": 9.929512147584296e-07, |
| "logits/chosen": -19.25161771955138, |
| "logits/rejected": -19.300870767626026, |
| "logps/chosen": -471.5840435028076, |
| "logps/rejected": -377.43126487731934, |
| "loss": 0.9580326080322266, |
| "mean_token_accuracy": 0.7341411262750626, |
| "num_tokens": 10231221.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.00470810174010694, |
| "rewards/margins": 0.1703320173546672, |
| "rewards/rejected": -0.16562391445040703, |
| "step": 181 |
| }, |
| { |
| "entropy": 1.2480486258864403, |
| "epoch": 0.11412447092020693, |
| "grad_norm": 33.407691955566406, |
| "learning_rate": 9.927743220671389e-07, |
| "logits/chosen": -15.707493700178858, |
| "logits/rejected": -18.14101813743978, |
| "logps/chosen": -644.9055366516113, |
| "logps/rejected": -403.51587295532227, |
| "loss": 0.9577570557594299, |
| "mean_token_accuracy": 0.695245198905468, |
| "num_tokens": 10278764.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.00990894460119307, |
| "rewards/margins": 0.1714292955584824, |
| "rewards/rejected": -0.18133824318647385, |
| "step": 182 |
| }, |
| { |
| "entropy": 0.9243633225560188, |
| "epoch": 0.11475152845273554, |
| "grad_norm": 34.56248474121094, |
| "learning_rate": 9.925952533438897e-07, |
| "logits/chosen": -17.46200834389154, |
| "logits/rejected": -16.492948873374484, |
| "logps/chosen": -617.6435585021973, |
| "logps/rejected": -372.62390518188477, |
| "loss": 0.948718786239624, |
| "mean_token_accuracy": 0.7732345685362816, |
| "num_tokens": 10341069.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.010069805663079023, |
| "rewards/margins": 0.2102643116377294, |
| "rewards/rejected": -0.22033411543816328, |
| "step": 183 |
| }, |
| { |
| "entropy": 0.9938921704888344, |
| "epoch": 0.11537858598526415, |
| "grad_norm": 30.834369659423828, |
| "learning_rate": 9.924140093794277e-07, |
| "logits/chosen": -19.204634086255844, |
| "logits/rejected": -19.86160843964533, |
| "logps/chosen": -502.1897392272949, |
| "logps/rejected": -514.1468658447266, |
| "loss": 0.9647988080978394, |
| "mean_token_accuracy": 0.7703093513846397, |
| "num_tokens": 10385519.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": 0.006156077841296792, |
| "rewards/margins": 0.14176303520798683, |
| "rewards/rejected": -0.13560695759952068, |
| "step": 184 |
| }, |
| { |
| "entropy": 0.9982673227787018, |
| "epoch": 0.11600564351779276, |
| "grad_norm": 33.376678466796875, |
| "learning_rate": 9.922305909741046e-07, |
| "logits/chosen": -20.081957609627672, |
| "logits/rejected": -20.08288046634073, |
| "logps/chosen": -563.1763916015625, |
| "logps/rejected": -455.9494934082031, |
| "loss": 0.940411388874054, |
| "mean_token_accuracy": 0.7691720277070999, |
| "num_tokens": 10451619.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.03211024316260591, |
| "rewards/margins": 0.24388167914003134, |
| "rewards/rejected": -0.21177144069224596, |
| "step": 185 |
| }, |
| { |
| "entropy": 1.2043246403336525, |
| "epoch": 0.11663270105032136, |
| "grad_norm": 35.30168151855469, |
| "learning_rate": 9.92044998937874e-07, |
| "logits/chosen": -15.935400744656912, |
| "logits/rejected": -17.80423133241425, |
| "logps/chosen": -507.5350112915039, |
| "logps/rejected": -506.1169013977051, |
| "loss": 0.9446260929107666, |
| "mean_token_accuracy": 0.7261253371834755, |
| "num_tokens": 10504967.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.011272144154645503, |
| "rewards/margins": 0.2257434045895934, |
| "rewards/rejected": -0.21447127033025026, |
| "step": 186 |
| }, |
| { |
| "entropy": 0.8656372576951981, |
| "epoch": 0.11725975858284998, |
| "grad_norm": 36.30891799926758, |
| "learning_rate": 9.918572340902878e-07, |
| "logits/chosen": -20.357606409612657, |
| "logits/rejected": -20.43430765009451, |
| "logps/chosen": -806.364107131958, |
| "logps/rejected": -409.4738140106201, |
| "loss": 0.9610611796379089, |
| "mean_token_accuracy": 0.7912320047616959, |
| "num_tokens": 10559810.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.01855588040780276, |
| "rewards/margins": 0.16032809671014547, |
| "rewards/rejected": -0.14177220640704036, |
| "step": 187 |
| }, |
| { |
| "entropy": 1.2360781207680702, |
| "epoch": 0.11788681611537859, |
| "grad_norm": 36.44474411010742, |
| "learning_rate": 9.916672972604927e-07, |
| "logits/chosen": -21.232208354402246, |
| "logits/rejected": -20.78721834595899, |
| "logps/chosen": -872.3558025360107, |
| "logps/rejected": -562.6851234436035, |
| "loss": 0.956852912902832, |
| "mean_token_accuracy": 0.720037579536438, |
| "num_tokens": 10620148.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": 0.012437668745405972, |
| "rewards/margins": 0.17596057945047505, |
| "rewards/rejected": -0.16352291277144104, |
| "step": 188 |
| }, |
| { |
| "entropy": 1.016458660364151, |
| "epoch": 0.11851387364790719, |
| "grad_norm": 29.050859451293945, |
| "learning_rate": 9.914751892872274e-07, |
| "logits/chosen": -19.08313772226747, |
| "logits/rejected": -18.82169734271658, |
| "logps/chosen": -434.9751796722412, |
| "logps/rejected": -346.7390670776367, |
| "loss": 0.9528908133506775, |
| "mean_token_accuracy": 0.7646205425262451, |
| "num_tokens": 10670772.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.008685210370458663, |
| "rewards/margins": 0.19308780319988728, |
| "rewards/rejected": -0.20177301485091448, |
| "step": 189 |
| }, |
| { |
| "entropy": 1.1021007224917412, |
| "epoch": 0.11914093118043581, |
| "grad_norm": 44.599769592285156, |
| "learning_rate": 9.91280911018817e-07, |
| "logits/chosen": -21.5532820720735, |
| "logits/rejected": -20.40933043838709, |
| "logps/chosen": -1575.5354919433594, |
| "logps/rejected": -422.48606395721436, |
| "loss": 0.941632866859436, |
| "mean_token_accuracy": 0.7219045013189316, |
| "num_tokens": 10758488.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.06574870133772492, |
| "rewards/margins": 0.2363713993690908, |
| "rewards/rejected": -0.1706227008253336, |
| "step": 190 |
| }, |
| { |
| "entropy": 1.1249773353338242, |
| "epoch": 0.11976798871296442, |
| "grad_norm": 37.6546745300293, |
| "learning_rate": 9.910844633131712e-07, |
| "logits/chosen": -17.602501168387953, |
| "logits/rejected": -18.726467814742886, |
| "logps/chosen": -641.0849552154541, |
| "logps/rejected": -560.3315982818604, |
| "loss": 0.935768723487854, |
| "mean_token_accuracy": 0.7447556182742119, |
| "num_tokens": 10819612.0, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.0099241798161529, |
| "rewards/margins": 0.26406861934810877, |
| "rewards/rejected": -0.27399280294775963, |
| "step": 191 |
| }, |
| { |
| "entropy": 1.0111653581261635, |
| "epoch": 0.12039504624549302, |
| "grad_norm": 33.29735565185547, |
| "learning_rate": 9.908858470377793e-07, |
| "logits/chosen": -17.99586764005896, |
| "logits/rejected": -16.641471216812302, |
| "logps/chosen": -754.2721481323242, |
| "logps/rejected": -414.00395679473877, |
| "loss": 0.9521582126617432, |
| "mean_token_accuracy": 0.7499677836894989, |
| "num_tokens": 10882013.0, |
| "rewards/accuracies": 0.96875, |
| "rewards/chosen": 0.00994637084659189, |
| "rewards/margins": 0.19447646429762244, |
| "rewards/rejected": -0.1845300872810185, |
| "step": 192 |
| }, |
| { |
| "entropy": 1.0526621490716934, |
| "epoch": 0.12102210377802164, |
| "grad_norm": 35.13934326171875, |
| "learning_rate": 9.906850630697066e-07, |
| "logits/chosen": -17.88888213857138, |
| "logits/rejected": -17.117701311736873, |
| "logps/chosen": -626.3125686645508, |
| "logps/rejected": -390.99532318115234, |
| "loss": 0.9473018646240234, |
| "mean_token_accuracy": 0.7478787750005722, |
| "num_tokens": 10932194.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.013837772188708186, |
| "rewards/margins": 0.21603740751743317, |
| "rewards/rejected": -0.22987518273293972, |
| "step": 193 |
| }, |
| { |
| "entropy": 1.2843475490808487, |
| "epoch": 0.12164916131055024, |
| "grad_norm": 33.156471252441406, |
| "learning_rate": 9.904821122955914e-07, |
| "logits/chosen": -16.072002517290578, |
| "logits/rejected": -17.32703283993392, |
| "logps/chosen": -858.3284645080566, |
| "logps/rejected": -468.63574600219727, |
| "loss": 0.957085132598877, |
| "mean_token_accuracy": 0.6953159794211388, |
| "num_tokens": 10989560.0, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -0.0002469784813001752, |
| "rewards/margins": 0.174753800034523, |
| "rewards/rejected": -0.1750007774680853, |
| "step": 194 |
| }, |
| { |
| "entropy": 0.9444542303681374, |
| "epoch": 0.12227621884307885, |
| "grad_norm": 33.4073371887207, |
| "learning_rate": 9.90276995611639e-07, |
| "logits/chosen": -19.142496042660422, |
| "logits/rejected": -21.297342713346943, |
| "logps/chosen": -639.1740913391113, |
| "logps/rejected": -354.38869857788086, |
| "loss": 0.9653919339179993, |
| "mean_token_accuracy": 0.7545767053961754, |
| "num_tokens": 11034532.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": 0.0012790057808160782, |
| "rewards/margins": 0.1389640721026808, |
| "rewards/rejected": -0.13768507237546146, |
| "step": 195 |
| }, |
| { |
| "entropy": 1.106497973203659, |
| "epoch": 0.12290327637560747, |
| "grad_norm": 30.54036521911621, |
| "learning_rate": 9.900697139236208e-07, |
| "logits/chosen": -17.157277160405563, |
| "logits/rejected": -18.095409943495174, |
| "logps/chosen": -551.8531856536865, |
| "logps/rejected": -366.1446056365967, |
| "loss": 0.9476866722106934, |
| "mean_token_accuracy": 0.7291347607970238, |
| "num_tokens": 11079071.0, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": 0.026466174283996224, |
| "rewards/margins": 0.21335551515221596, |
| "rewards/rejected": -0.18688933923840523, |
| "step": 196 |
| }, |
| { |
| "entropy": 1.0959510654211044, |
| "epoch": 0.12353033390813607, |
| "grad_norm": 36.96934127807617, |
| "learning_rate": 9.898602681468674e-07, |
| "logits/chosen": -19.027734293968553, |
| "logits/rejected": -19.864234419673636, |
| "logps/chosen": -954.4189186096191, |
| "logps/rejected": -566.7688503265381, |
| "loss": 0.9575784206390381, |
| "mean_token_accuracy": 0.7271228209137917, |
| "num_tokens": 11136634.0, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -0.016784888110123575, |
| "rewards/margins": 0.17304366454482079, |
| "rewards/rejected": -0.18982854578644037, |
| "step": 197 |
| }, |
| { |
| "entropy": 0.8774717822670937, |
| "epoch": 0.12415739144066468, |
| "grad_norm": 36.82571792602539, |
| "learning_rate": 9.896486592062659e-07, |
| "logits/chosen": -28.647222960438814, |
| "logits/rejected": -22.060415168768344, |
| "logps/chosen": -836.5278701782227, |
| "logps/rejected": -480.254337310791, |
| "loss": 0.9511983394622803, |
| "mean_token_accuracy": 0.7737741991877556, |
| "num_tokens": 11197525.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.008797007380053401, |
| "rewards/margins": 0.20592515252064914, |
| "rewards/rejected": -0.21472215885296464, |
| "step": 198 |
| }, |
| { |
| "entropy": 1.1285011321306229, |
| "epoch": 0.1247844489731933, |
| "grad_norm": 42.2977409362793, |
| "learning_rate": 9.89434888036256e-07, |
| "logits/chosen": -18.49698845016767, |
| "logits/rejected": -17.254445681297014, |
| "logps/chosen": -1127.2061748504639, |
| "logps/rejected": -956.0397682189941, |
| "loss": 0.9597939848899841, |
| "mean_token_accuracy": 0.731256939470768, |
| "num_tokens": 11292505.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.003246246255002916, |
| "rewards/margins": 0.16256380267441273, |
| "rewards/rejected": -0.1658100476488471, |
| "step": 199 |
| }, |
| { |
| "entropy": 0.9618688002228737, |
| "epoch": 0.1254115065057219, |
| "grad_norm": 35.041629791259766, |
| "learning_rate": 9.892189555808251e-07, |
| "logits/chosen": -23.16837855908106, |
| "logits/rejected": -22.01573163400953, |
| "logps/chosen": -595.0505418777466, |
| "logps/rejected": -505.8145923614502, |
| "loss": 0.9585968852043152, |
| "mean_token_accuracy": 0.7569021657109261, |
| "num_tokens": 11345306.0, |
| "rewards/accuracies": 0.875, |
| "rewards/chosen": -0.027557444758713245, |
| "rewards/margins": 0.16784314159303904, |
| "rewards/rejected": -0.19540058448910713, |
| "step": 200 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 1595, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 5.084673057646182e+16, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
