| { |
| "best_global_step": 4600, |
| "best_metric": 0.7991446852684021, |
| "best_model_checkpoint": "lfm2.5-350m-multilang-code-math/checkpoint-4600", |
| "epoch": 2.0, |
| "eval_steps": 200, |
| "global_step": 5778, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.2043220043182372, |
| "epoch": 0.008655011251514627, |
| "grad_norm": 23.5, |
| "learning_rate": 2.7586206896551725e-06, |
| "loss": 3.0389138793945314, |
| "mean_token_accuracy": 0.5589780753850937, |
| "num_tokens": 151155.0, |
| "step": 25 |
| }, |
| { |
| "entropy": 1.11516796708107, |
| "epoch": 0.017310022503029255, |
| "grad_norm": 7.34375, |
| "learning_rate": 5.6321839080459775e-06, |
| "loss": 1.8928765869140625, |
| "mean_token_accuracy": 0.6762128543853759, |
| "num_tokens": 310957.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 1.2031074380874633, |
| "epoch": 0.02596503375454388, |
| "grad_norm": 3.78125, |
| "learning_rate": 8.505747126436782e-06, |
| "loss": 1.2897039794921874, |
| "mean_token_accuracy": 0.7282541489601135, |
| "num_tokens": 461068.0, |
| "step": 75 |
| }, |
| { |
| "entropy": 1.0323714792728425, |
| "epoch": 0.03462004500605851, |
| "grad_norm": 3.71875, |
| "learning_rate": 1.1379310344827587e-05, |
| "loss": 1.0183975982666016, |
| "mean_token_accuracy": 0.757768098115921, |
| "num_tokens": 615118.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.9247142732143402, |
| "epoch": 0.043275056257573136, |
| "grad_norm": 2.578125, |
| "learning_rate": 1.4252873563218392e-05, |
| "loss": 0.8757442474365235, |
| "mean_token_accuracy": 0.7792941844463348, |
| "num_tokens": 762906.0, |
| "step": 125 |
| }, |
| { |
| "entropy": 0.9190187740325928, |
| "epoch": 0.05193006750908776, |
| "grad_norm": 2.296875, |
| "learning_rate": 1.7126436781609197e-05, |
| "loss": 0.891803970336914, |
| "mean_token_accuracy": 0.7753661525249481, |
| "num_tokens": 921356.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.886636551618576, |
| "epoch": 0.06058507876060239, |
| "grad_norm": 2.296875, |
| "learning_rate": 2e-05, |
| "loss": 0.861456298828125, |
| "mean_token_accuracy": 0.7826532816886902, |
| "num_tokens": 1076183.0, |
| "step": 175 |
| }, |
| { |
| "entropy": 0.9015009069442749, |
| "epoch": 0.06924009001211702, |
| "grad_norm": 2.5625, |
| "learning_rate": 1.9999017921043748e-05, |
| "loss": 0.8748048400878906, |
| "mean_token_accuracy": 0.7792416322231293, |
| "num_tokens": 1229170.0, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.06924009001211702, |
| "eval_entropy": 0.8920288547140653, |
| "eval_loss": 0.8666635155677795, |
| "eval_mean_token_accuracy": 0.7797926357535065, |
| "eval_num_tokens": 1229170.0, |
| "eval_runtime": 16.3134, |
| "eval_samples_per_second": 298.222, |
| "eval_steps_per_second": 18.696, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.9062747704982758, |
| "epoch": 0.07789510126363164, |
| "grad_norm": 2.40625, |
| "learning_rate": 1.999607187707079e-05, |
| "loss": 0.8789938354492187, |
| "mean_token_accuracy": 0.7778702569007874, |
| "num_tokens": 1381380.0, |
| "step": 225 |
| }, |
| { |
| "entropy": 0.8776320171356201, |
| "epoch": 0.08655011251514627, |
| "grad_norm": 2.46875, |
| "learning_rate": 1.9991162446730694e-05, |
| "loss": 0.8517158508300782, |
| "mean_token_accuracy": 0.7819743084907532, |
| "num_tokens": 1538186.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.8657179689407348, |
| "epoch": 0.09520512376666089, |
| "grad_norm": 2.25, |
| "learning_rate": 1.9984290594313102e-05, |
| "loss": 0.8408647155761719, |
| "mean_token_accuracy": 0.7840179252624512, |
| "num_tokens": 1690067.0, |
| "step": 275 |
| }, |
| { |
| "entropy": 0.8955290937423706, |
| "epoch": 0.10386013501817552, |
| "grad_norm": 2.328125, |
| "learning_rate": 1.9975457669558346e-05, |
| "loss": 0.8683735656738282, |
| "mean_token_accuracy": 0.7763688516616821, |
| "num_tokens": 1845254.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.8744900381565094, |
| "epoch": 0.11251514626969016, |
| "grad_norm": 2.34375, |
| "learning_rate": 1.996466540739233e-05, |
| "loss": 0.8470248413085938, |
| "mean_token_accuracy": 0.7828968441486359, |
| "num_tokens": 2000855.0, |
| "step": 325 |
| }, |
| { |
| "entropy": 0.8875207018852234, |
| "epoch": 0.12117015752120477, |
| "grad_norm": 2.34375, |
| "learning_rate": 1.9951915927585762e-05, |
| "loss": 0.8633122253417969, |
| "mean_token_accuracy": 0.7807920384407043, |
| "num_tokens": 2159255.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.8727393555641174, |
| "epoch": 0.1298251687727194, |
| "grad_norm": 2.40625, |
| "learning_rate": 1.9937211734337812e-05, |
| "loss": 0.8484575653076172, |
| "mean_token_accuracy": 0.7856597316265106, |
| "num_tokens": 2313980.0, |
| "step": 375 |
| }, |
| { |
| "entropy": 0.8450514304637909, |
| "epoch": 0.13848018002423404, |
| "grad_norm": 2.390625, |
| "learning_rate": 1.992055571578423e-05, |
| "loss": 0.8231178283691406, |
| "mean_token_accuracy": 0.7884540009498596, |
| "num_tokens": 2468392.0, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.13848018002423404, |
| "eval_entropy": 0.8820633643963298, |
| "eval_loss": 0.8462674021720886, |
| "eval_mean_token_accuracy": 0.7830698802822926, |
| "eval_num_tokens": 2468392.0, |
| "eval_runtime": 16.2902, |
| "eval_samples_per_second": 298.645, |
| "eval_steps_per_second": 18.723, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.8852161896228791, |
| "epoch": 0.14713519127574864, |
| "grad_norm": 2.25, |
| "learning_rate": 1.990195114343008e-05, |
| "loss": 0.8604666137695313, |
| "mean_token_accuracy": 0.7796429479122162, |
| "num_tokens": 2626504.0, |
| "step": 425 |
| }, |
| { |
| "entropy": 0.8595891773700715, |
| "epoch": 0.15579020252726328, |
| "grad_norm": 2.109375, |
| "learning_rate": 1.988140167150716e-05, |
| "loss": 0.8375725555419922, |
| "mean_token_accuracy": 0.7837563192844391, |
| "num_tokens": 2785992.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.8787437427043915, |
| "epoch": 0.1644452137787779, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.9858911336256258e-05, |
| "loss": 0.8602984619140625, |
| "mean_token_accuracy": 0.7801311421394348, |
| "num_tokens": 2943389.0, |
| "step": 475 |
| }, |
| { |
| "entropy": 0.8887673962116241, |
| "epoch": 0.17310022503029254, |
| "grad_norm": 2.34375, |
| "learning_rate": 1.983448455513437e-05, |
| "loss": 0.8559275054931641, |
| "mean_token_accuracy": 0.7799410057067871, |
| "num_tokens": 3098910.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.8632562351226807, |
| "epoch": 0.18175523628180718, |
| "grad_norm": 2.359375, |
| "learning_rate": 1.9808126125947035e-05, |
| "loss": 0.8414566040039062, |
| "mean_token_accuracy": 0.7838216722011566, |
| "num_tokens": 3259790.0, |
| "step": 525 |
| }, |
| { |
| "entropy": 0.8468562078475952, |
| "epoch": 0.19041024753332178, |
| "grad_norm": 2.921875, |
| "learning_rate": 1.977984122590598e-05, |
| "loss": 0.8214312744140625, |
| "mean_token_accuracy": 0.7904518485069275, |
| "num_tokens": 3408921.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.862267234325409, |
| "epoch": 0.1990652587848364, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.9749635410612222e-05, |
| "loss": 0.8381607818603516, |
| "mean_token_accuracy": 0.7850737714767456, |
| "num_tokens": 3564021.0, |
| "step": 575 |
| }, |
| { |
| "entropy": 0.8673117184638977, |
| "epoch": 0.20772027003635105, |
| "grad_norm": 2.28125, |
| "learning_rate": 1.971751461296488e-05, |
| "loss": 0.8422865295410156, |
| "mean_token_accuracy": 0.7839160299301148, |
| "num_tokens": 3720222.0, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.20772027003635105, |
| "eval_entropy": 0.8581681211463741, |
| "eval_loss": 0.8380036950111389, |
| "eval_mean_token_accuracy": 0.7849426593936858, |
| "eval_num_tokens": 3720222.0, |
| "eval_runtime": 16.2659, |
| "eval_samples_per_second": 299.091, |
| "eval_steps_per_second": 18.751, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.896509393453598, |
| "epoch": 0.21637528128786568, |
| "grad_norm": 2.25, |
| "learning_rate": 1.9683485141995832e-05, |
| "loss": 0.8751831817626953, |
| "mean_token_accuracy": 0.778223477602005, |
| "num_tokens": 3868288.0, |
| "step": 625 |
| }, |
| { |
| "entropy": 0.8858666527271271, |
| "epoch": 0.2250302925393803, |
| "grad_norm": 2.4375, |
| "learning_rate": 1.964755368163055e-05, |
| "loss": 0.8597821807861328, |
| "mean_token_accuracy": 0.7796040976047516, |
| "num_tokens": 4020387.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.8457015335559845, |
| "epoch": 0.23368530379089492, |
| "grad_norm": 2.0625, |
| "learning_rate": 1.9609727289375254e-05, |
| "loss": 0.8197212982177734, |
| "mean_token_accuracy": 0.7878750622272491, |
| "num_tokens": 4175769.0, |
| "step": 675 |
| }, |
| { |
| "entropy": 0.8535351765155792, |
| "epoch": 0.24234031504240955, |
| "grad_norm": 2.328125, |
| "learning_rate": 1.9570013394930704e-05, |
| "loss": 0.8300620269775391, |
| "mean_token_accuracy": 0.786250706911087, |
| "num_tokens": 4331114.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.8614680600166321, |
| "epoch": 0.2509953262939242, |
| "grad_norm": 2.0625, |
| "learning_rate": 1.9528419798732902e-05, |
| "loss": 0.8407713317871094, |
| "mean_token_accuracy": 0.7833027255535125, |
| "num_tokens": 4488237.0, |
| "step": 725 |
| }, |
| { |
| "entropy": 0.8501963889598847, |
| "epoch": 0.2596503375454388, |
| "grad_norm": 2.328125, |
| "learning_rate": 1.948495467042095e-05, |
| "loss": 0.8218775177001953, |
| "mean_token_accuracy": 0.788744968175888, |
| "num_tokens": 4642068.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.8383999621868133, |
| "epoch": 0.26830534879695345, |
| "grad_norm": 2.296875, |
| "learning_rate": 1.9439626547232433e-05, |
| "loss": 0.80978515625, |
| "mean_token_accuracy": 0.7903480684757233, |
| "num_tokens": 4788314.0, |
| "step": 775 |
| }, |
| { |
| "entropy": 0.8568655729293824, |
| "epoch": 0.2769603600484681, |
| "grad_norm": 2.53125, |
| "learning_rate": 1.939244433232652e-05, |
| "loss": 0.8343601226806641, |
| "mean_token_accuracy": 0.7855056190490722, |
| "num_tokens": 4941131.0, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.2769603600484681, |
| "eval_entropy": 0.859573404906226, |
| "eval_loss": 0.8307726383209229, |
| "eval_mean_token_accuracy": 0.7864908138259512, |
| "eval_num_tokens": 4941131.0, |
| "eval_runtime": 16.2516, |
| "eval_samples_per_second": 299.355, |
| "eval_steps_per_second": 18.767, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.8670335674285888, |
| "epoch": 0.2856153712999827, |
| "grad_norm": 1.9765625, |
| "learning_rate": 1.934341729303529e-05, |
| "loss": 0.8394245910644531, |
| "mean_token_accuracy": 0.7841019070148468, |
| "num_tokens": 5096940.0, |
| "step": 825 |
| }, |
| { |
| "entropy": 0.867160736322403, |
| "epoch": 0.2942703825514973, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.9292555059043457e-05, |
| "loss": 0.8409418487548828, |
| "mean_token_accuracy": 0.7830280113220215, |
| "num_tokens": 5250616.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.8582142984867096, |
| "epoch": 0.3029253938030119, |
| "grad_norm": 2.15625, |
| "learning_rate": 1.9239867620496952e-05, |
| "loss": 0.8346880340576172, |
| "mean_token_accuracy": 0.783586630821228, |
| "num_tokens": 5408253.0, |
| "step": 875 |
| }, |
| { |
| "entropy": 0.84959406375885, |
| "epoch": 0.31158040505452655, |
| "grad_norm": 2.1875, |
| "learning_rate": 1.9185365326040713e-05, |
| "loss": 0.826163330078125, |
| "mean_token_accuracy": 0.7881129395961761, |
| "num_tokens": 5564921.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.8895779585838318, |
| "epoch": 0.3202354163060412, |
| "grad_norm": 2.1875, |
| "learning_rate": 1.9129058880786026e-05, |
| "loss": 0.866309585571289, |
| "mean_token_accuracy": 0.7782393181324005, |
| "num_tokens": 5722704.0, |
| "step": 925 |
| }, |
| { |
| "entropy": 0.8372803354263305, |
| "epoch": 0.3288904275575558, |
| "grad_norm": 1.890625, |
| "learning_rate": 1.907095934420789e-05, |
| "loss": 0.8083054351806641, |
| "mean_token_accuracy": 0.7903852713108063, |
| "num_tokens": 5881153.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.8762626206874847, |
| "epoch": 0.33754543880907045, |
| "grad_norm": 1.96875, |
| "learning_rate": 1.9011078127972748e-05, |
| "loss": 0.8495908355712891, |
| "mean_token_accuracy": 0.7841223049163818, |
| "num_tokens": 6039323.0, |
| "step": 975 |
| }, |
| { |
| "entropy": 0.8670505321025849, |
| "epoch": 0.3462004500605851, |
| "grad_norm": 2.25, |
| "learning_rate": 1.8949426993697074e-05, |
| "loss": 0.8448488616943359, |
| "mean_token_accuracy": 0.785959769487381, |
| "num_tokens": 6197683.0, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.3462004500605851, |
| "eval_entropy": 0.8536441159052927, |
| "eval_loss": 0.8251546025276184, |
| "eval_mean_token_accuracy": 0.7873384729760592, |
| "eval_num_tokens": 6197683.0, |
| "eval_runtime": 16.2591, |
| "eval_samples_per_second": 299.217, |
| "eval_steps_per_second": 18.759, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.8527129173278809, |
| "epoch": 0.3548554613120997, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.888601805063719e-05, |
| "loss": 0.8221394348144532, |
| "mean_token_accuracy": 0.7878592002391815, |
| "num_tokens": 6353591.0, |
| "step": 1025 |
| }, |
| { |
| "entropy": 0.8309027600288391, |
| "epoch": 0.36351047256361435, |
| "grad_norm": 2.1875, |
| "learning_rate": 1.8820863753310812e-05, |
| "loss": 0.804949951171875, |
| "mean_token_accuracy": 0.7907659840583802, |
| "num_tokens": 6503522.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.8664905667304993, |
| "epoch": 0.372165483815129, |
| "grad_norm": 2.234375, |
| "learning_rate": 1.875397689905081e-05, |
| "loss": 0.8386592102050782, |
| "mean_token_accuracy": 0.783461629152298, |
| "num_tokens": 6657038.0, |
| "step": 1075 |
| }, |
| { |
| "entropy": 0.8521995890140533, |
| "epoch": 0.38082049506664356, |
| "grad_norm": 2.140625, |
| "learning_rate": 1.8685370625491582e-05, |
| "loss": 0.8274839782714843, |
| "mean_token_accuracy": 0.7857646262645721, |
| "num_tokens": 6813403.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.85324840426445, |
| "epoch": 0.3894755063181582, |
| "grad_norm": 2.03125, |
| "learning_rate": 1.861505840798864e-05, |
| "loss": 0.8251450347900391, |
| "mean_token_accuracy": 0.7872388279438018, |
| "num_tokens": 6969337.0, |
| "step": 1125 |
| }, |
| { |
| "entropy": 0.8578063642978668, |
| "epoch": 0.3981305175696728, |
| "grad_norm": 2.09375, |
| "learning_rate": 1.8543054056971815e-05, |
| "loss": 0.834039077758789, |
| "mean_token_accuracy": 0.7848770570755005, |
| "num_tokens": 7124334.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.8491312241554261, |
| "epoch": 0.40678552882118746, |
| "grad_norm": 2.0625, |
| "learning_rate": 1.8469371715232686e-05, |
| "loss": 0.8220222473144532, |
| "mean_token_accuracy": 0.7871887254714965, |
| "num_tokens": 7277475.0, |
| "step": 1175 |
| }, |
| { |
| "entropy": 0.8443890881538391, |
| "epoch": 0.4154405400727021, |
| "grad_norm": 2.203125, |
| "learning_rate": 1.839402585514671e-05, |
| "loss": 0.8166568756103516, |
| "mean_token_accuracy": 0.7897883796691895, |
| "num_tokens": 7428616.0, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.4154405400727021, |
| "eval_entropy": 0.8290919256014903, |
| "eval_loss": 0.8211944103240967, |
| "eval_mean_token_accuracy": 0.788363546621604, |
| "eval_num_tokens": 7428616.0, |
| "eval_runtime": 16.2451, |
| "eval_samples_per_second": 299.474, |
| "eval_steps_per_second": 18.775, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.8553554141521453, |
| "epoch": 0.4240955513242167, |
| "grad_norm": 2.140625, |
| "learning_rate": 1.8317031275830607e-05, |
| "loss": 0.8335649108886719, |
| "mean_token_accuracy": 0.7848791778087616, |
| "num_tokens": 7586653.0, |
| "step": 1225 |
| }, |
| { |
| "entropy": 0.8513217878341675, |
| "epoch": 0.43275056257573136, |
| "grad_norm": 2.078125, |
| "learning_rate": 1.8238403100235602e-05, |
| "loss": 0.8226930999755859, |
| "mean_token_accuracy": 0.7892848730087281, |
| "num_tokens": 7736619.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.8567298305034637, |
| "epoch": 0.441405573827246, |
| "grad_norm": 2.25, |
| "learning_rate": 1.8158156772177015e-05, |
| "loss": 0.8301739501953125, |
| "mean_token_accuracy": 0.7857687485218048, |
| "num_tokens": 7895674.0, |
| "step": 1275 |
| }, |
| { |
| "entropy": 0.8383160126209259, |
| "epoch": 0.4500605850787606, |
| "grad_norm": 2.28125, |
| "learning_rate": 1.807630805330087e-05, |
| "loss": 0.8143462371826172, |
| "mean_token_accuracy": 0.7904347920417786, |
| "num_tokens": 8052368.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.858016027212143, |
| "epoch": 0.45871559633027525, |
| "grad_norm": 2.328125, |
| "learning_rate": 1.7992873019988044e-05, |
| "loss": 0.8361105346679687, |
| "mean_token_accuracy": 0.7837494373321533, |
| "num_tokens": 8209037.0, |
| "step": 1325 |
| }, |
| { |
| "entropy": 0.8485116994380951, |
| "epoch": 0.46737060758178983, |
| "grad_norm": 2.078125, |
| "learning_rate": 1.7907868060196627e-05, |
| "loss": 0.8189002990722656, |
| "mean_token_accuracy": 0.7860724258422852, |
| "num_tokens": 8366536.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.8689926111698151, |
| "epoch": 0.47602561883330446, |
| "grad_norm": 2.28125, |
| "learning_rate": 1.7821309870243056e-05, |
| "loss": 0.8450764465332031, |
| "mean_token_accuracy": 0.7817527639865876, |
| "num_tokens": 8524698.0, |
| "step": 1375 |
| }, |
| { |
| "entropy": 0.8510251712799072, |
| "epoch": 0.4846806300848191, |
| "grad_norm": 2.03125, |
| "learning_rate": 1.77332154515227e-05, |
| "loss": 0.8352996063232422, |
| "mean_token_accuracy": 0.786122510433197, |
| "num_tokens": 8680719.0, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.4846806300848191, |
| "eval_entropy": 0.8555903234442727, |
| "eval_loss": 0.8170819878578186, |
| "eval_mean_token_accuracy": 0.7890570513537672, |
| "eval_num_tokens": 8680719.0, |
| "eval_runtime": 16.2534, |
| "eval_samples_per_second": 299.322, |
| "eval_steps_per_second": 18.765, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.8443090248107911, |
| "epoch": 0.49333564133633373, |
| "grad_norm": 2.34375, |
| "learning_rate": 1.764360210717051e-05, |
| "loss": 0.8160585784912109, |
| "mean_token_accuracy": 0.7888865935802459, |
| "num_tokens": 8839183.0, |
| "step": 1425 |
| }, |
| { |
| "entropy": 0.8397502684593201, |
| "epoch": 0.5019906525878484, |
| "grad_norm": 2.015625, |
| "learning_rate": 1.7552487438662432e-05, |
| "loss": 0.8148053741455078, |
| "mean_token_accuracy": 0.7885977244377136, |
| "num_tokens": 8995586.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.8441075325012207, |
| "epoch": 0.5106456638393629, |
| "grad_norm": 2.34375, |
| "learning_rate": 1.7459889342358176e-05, |
| "loss": 0.8217481994628906, |
| "mean_token_accuracy": 0.7888561117649079, |
| "num_tokens": 9146595.0, |
| "step": 1475 |
| }, |
| { |
| "entropy": 0.8394048309326172, |
| "epoch": 0.5193006750908776, |
| "grad_norm": 2.296875, |
| "learning_rate": 1.736582600598609e-05, |
| "loss": 0.8105857849121094, |
| "mean_token_accuracy": 0.7910154783725738, |
| "num_tokens": 9297789.0, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.8256420779228211, |
| "epoch": 0.5279556863423922, |
| "grad_norm": 2.140625, |
| "learning_rate": 1.7270315905070822e-05, |
| "loss": 0.7967173004150391, |
| "mean_token_accuracy": 0.7917508590221405, |
| "num_tokens": 9450532.0, |
| "step": 1525 |
| }, |
| { |
| "entropy": 0.817123122215271, |
| "epoch": 0.5366106975939069, |
| "grad_norm": 1.8828125, |
| "learning_rate": 1.717337779930441e-05, |
| "loss": 0.7964045715332031, |
| "mean_token_accuracy": 0.793145546913147, |
| "num_tokens": 9605286.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.8218163585662842, |
| "epoch": 0.5452657088454215, |
| "grad_norm": 2.25, |
| "learning_rate": 1.7075030728861602e-05, |
| "loss": 0.7938800048828125, |
| "mean_token_accuracy": 0.7939867222309113, |
| "num_tokens": 9755402.0, |
| "step": 1575 |
| }, |
| { |
| "entropy": 0.8473324692249298, |
| "epoch": 0.5539207200969362, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.697529401066006e-05, |
| "loss": 0.8210481262207031, |
| "mean_token_accuracy": 0.7879379904270172, |
| "num_tokens": 9915397.0, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.5539207200969362, |
| "eval_entropy": 0.8279588664164309, |
| "eval_loss": 0.8136925101280212, |
| "eval_mean_token_accuracy": 0.7897968792524495, |
| "eval_num_tokens": 9915397.0, |
| "eval_runtime": 16.2401, |
| "eval_samples_per_second": 299.567, |
| "eval_steps_per_second": 18.781, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.83288947224617, |
| "epoch": 0.5625757313484507, |
| "grad_norm": 2.125, |
| "learning_rate": 1.6874187234566198e-05, |
| "loss": 0.8104652404785156, |
| "mean_token_accuracy": 0.7881759464740753, |
| "num_tokens": 10066176.0, |
| "step": 1625 |
| }, |
| { |
| "entropy": 0.8406240797042847, |
| "epoch": 0.5712307425999654, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.677173025954745e-05, |
| "loss": 0.8155978393554687, |
| "mean_token_accuracy": 0.7885338914394379, |
| "num_tokens": 10216630.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.8361802196502686, |
| "epoch": 0.57988575385148, |
| "grad_norm": 2.078125, |
| "learning_rate": 1.6667943209771634e-05, |
| "loss": 0.8124593353271484, |
| "mean_token_accuracy": 0.7909375095367431, |
| "num_tokens": 10368775.0, |
| "step": 1675 |
| }, |
| { |
| "entropy": 0.8546354687213897, |
| "epoch": 0.5885407651029946, |
| "grad_norm": 2.390625, |
| "learning_rate": 1.6562846470654253e-05, |
| "loss": 0.8320878601074219, |
| "mean_token_accuracy": 0.7862278306484223, |
| "num_tokens": 10522306.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.8271597623825073, |
| "epoch": 0.5971957763545093, |
| "grad_norm": 2.359375, |
| "learning_rate": 1.6456460684854474e-05, |
| "loss": 0.802950210571289, |
| "mean_token_accuracy": 0.7914960932731628, |
| "num_tokens": 10677278.0, |
| "step": 1725 |
| }, |
| { |
| "entropy": 0.8680684280395508, |
| "epoch": 0.6058507876060238, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.6348806748220596e-05, |
| "loss": 0.8468040466308594, |
| "mean_token_accuracy": 0.7825022864341736, |
| "num_tokens": 10832759.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.8460379076004029, |
| "epoch": 0.6145057988575385, |
| "grad_norm": 1.90625, |
| "learning_rate": 1.6239905805685766e-05, |
| "loss": 0.81728515625, |
| "mean_token_accuracy": 0.7885628819465638, |
| "num_tokens": 10987982.0, |
| "step": 1775 |
| }, |
| { |
| "entropy": 0.8258041095733643, |
| "epoch": 0.6231608101090531, |
| "grad_norm": 1.9609375, |
| "learning_rate": 1.6129779247114776e-05, |
| "loss": 0.8052913665771484, |
| "mean_token_accuracy": 0.7923011314868927, |
| "num_tokens": 11142409.0, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.6231608101090531, |
| "eval_entropy": 0.8428774887421092, |
| "eval_loss": 0.8103926181793213, |
| "eval_mean_token_accuracy": 0.790385250967057, |
| "eval_num_tokens": 11142409.0, |
| "eval_runtime": 16.2616, |
| "eval_samples_per_second": 299.17, |
| "eval_steps_per_second": 18.756, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.8314215159416198, |
| "epoch": 0.6318158213605678, |
| "grad_norm": 2.09375, |
| "learning_rate": 1.601844870310277e-05, |
| "loss": 0.8011739349365234, |
| "mean_token_accuracy": 0.7905190980434418, |
| "num_tokens": 11295629.0, |
| "step": 1825 |
| }, |
| { |
| "entropy": 0.8174971842765808, |
| "epoch": 0.6404708326120824, |
| "grad_norm": 2.03125, |
| "learning_rate": 1.5905936040726636e-05, |
| "loss": 0.7869554138183594, |
| "mean_token_accuracy": 0.7940086150169372, |
| "num_tokens": 11449328.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.8386989748477935, |
| "epoch": 0.6491258438635971, |
| "grad_norm": 2.03125, |
| "learning_rate": 1.579226335924999e-05, |
| "loss": 0.8210865783691407, |
| "mean_token_accuracy": 0.7893707096576691, |
| "num_tokens": 11605530.0, |
| "step": 1875 |
| }, |
| { |
| "entropy": 0.8110369992256165, |
| "epoch": 0.6577808551151116, |
| "grad_norm": 2.3125, |
| "learning_rate": 1.5677452985782493e-05, |
| "loss": 0.7894587707519531, |
| "mean_token_accuracy": 0.7973484849929809, |
| "num_tokens": 11760213.0, |
| "step": 1900 |
| }, |
| { |
| "entropy": 0.8280953919887543, |
| "epoch": 0.6664358663666263, |
| "grad_norm": 2.203125, |
| "learning_rate": 1.5561527470894503e-05, |
| "loss": 0.7997794342041016, |
| "mean_token_accuracy": 0.7926359629631042, |
| "num_tokens": 11913380.0, |
| "step": 1925 |
| }, |
| { |
| "entropy": 0.8196328341960907, |
| "epoch": 0.6750908776181409, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.544450958418775e-05, |
| "loss": 0.7956825256347656, |
| "mean_token_accuracy": 0.7946672177314759, |
| "num_tokens": 12065793.0, |
| "step": 1950 |
| }, |
| { |
| "entropy": 0.8513234782218934, |
| "epoch": 0.6837458888696555, |
| "grad_norm": 1.8671875, |
| "learning_rate": 1.5326422309823046e-05, |
| "loss": 0.8301435089111329, |
| "mean_token_accuracy": 0.7851076364517212, |
| "num_tokens": 12229815.0, |
| "step": 1975 |
| }, |
| { |
| "entropy": 0.8259239780902863, |
| "epoch": 0.6924009001211702, |
| "grad_norm": 2.078125, |
| "learning_rate": 1.5207288842005813e-05, |
| "loss": 0.8005783081054687, |
| "mean_token_accuracy": 0.79151895403862, |
| "num_tokens": 12391391.0, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.6924009001211702, |
| "eval_entropy": 0.828711828149733, |
| "eval_loss": 0.8071080446243286, |
| "eval_mean_token_accuracy": 0.7911858433582744, |
| "eval_num_tokens": 12391391.0, |
| "eval_runtime": 16.2568, |
| "eval_samples_per_second": 299.259, |
| "eval_steps_per_second": 18.761, |
| "step": 2000 |
| }, |
| { |
| "entropy": 0.8361215782165528, |
| "epoch": 0.7010559113726847, |
| "grad_norm": 2.109375, |
| "learning_rate": 1.508713258043041e-05, |
| "loss": 0.8149121856689453, |
| "mean_token_accuracy": 0.7890836715698242, |
| "num_tokens": 12548530.0, |
| "step": 2025 |
| }, |
| { |
| "entropy": 0.8404046738147736, |
| "epoch": 0.7097109226241994, |
| "grad_norm": 2.0, |
| "learning_rate": 1.4965977125684018e-05, |
| "loss": 0.814455337524414, |
| "mean_token_accuracy": 0.7903144693374634, |
| "num_tokens": 12703495.0, |
| "step": 2050 |
| }, |
| { |
| "entropy": 0.8286141097545624, |
| "epoch": 0.718365933875714, |
| "grad_norm": 2.296875, |
| "learning_rate": 1.484384627461115e-05, |
| "loss": 0.8082135009765625, |
| "mean_token_accuracy": 0.7908019495010375, |
| "num_tokens": 12856585.0, |
| "step": 2075 |
| }, |
| { |
| "entropy": 0.8540136122703552, |
| "epoch": 0.7270209451272287, |
| "grad_norm": 2.09375, |
| "learning_rate": 1.4720764015639555e-05, |
| "loss": 0.8297102355957031, |
| "mean_token_accuracy": 0.7875827658176422, |
| "num_tokens": 13011641.0, |
| "step": 2100 |
| }, |
| { |
| "entropy": 0.8180252170562744, |
| "epoch": 0.7356759563787433, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.4596754524068515e-05, |
| "loss": 0.791978988647461, |
| "mean_token_accuracy": 0.7945426023006439, |
| "num_tokens": 13167258.0, |
| "step": 2125 |
| }, |
| { |
| "entropy": 0.8130170261859894, |
| "epoch": 0.744330967630258, |
| "grad_norm": 2.1875, |
| "learning_rate": 1.4471842157320443e-05, |
| "loss": 0.7898551940917968, |
| "mean_token_accuracy": 0.7971277523040772, |
| "num_tokens": 13315266.0, |
| "step": 2150 |
| }, |
| { |
| "entropy": 0.8508910453319549, |
| "epoch": 0.7529859788817725, |
| "grad_norm": 2.140625, |
| "learning_rate": 1.4346051450156689e-05, |
| "loss": 0.8204324340820313, |
| "mean_token_accuracy": 0.7856499540805817, |
| "num_tokens": 13473686.0, |
| "step": 2175 |
| }, |
| { |
| "entropy": 0.8430261933803558, |
| "epoch": 0.7616409901332871, |
| "grad_norm": 2.125, |
| "learning_rate": 1.4219407109858532e-05, |
| "loss": 0.8170342254638672, |
| "mean_token_accuracy": 0.7883759355545044, |
| "num_tokens": 13627766.0, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.7616409901332871, |
| "eval_entropy": 0.8279429495334625, |
| "eval_loss": 0.8048078417778015, |
| "eval_mean_token_accuracy": 0.7913855454960808, |
| "eval_num_tokens": 13627766.0, |
| "eval_runtime": 16.2774, |
| "eval_samples_per_second": 298.88, |
| "eval_steps_per_second": 18.738, |
| "step": 2200 |
| }, |
| { |
| "entropy": 0.8365470349788666, |
| "epoch": 0.7702960013848018, |
| "grad_norm": 2.3125, |
| "learning_rate": 1.4091934011374281e-05, |
| "loss": 0.8083656311035157, |
| "mean_token_accuracy": 0.7891746735572815, |
| "num_tokens": 13781809.0, |
| "step": 2225 |
| }, |
| { |
| "entropy": 0.8262781822681426, |
| "epoch": 0.7789510126363164, |
| "grad_norm": 2.265625, |
| "learning_rate": 1.396365719243344e-05, |
| "loss": 0.8065071868896484, |
| "mean_token_accuracy": 0.7918438529968261, |
| "num_tokens": 13937124.0, |
| "step": 2250 |
| }, |
| { |
| "entropy": 0.8354354929924012, |
| "epoch": 0.7876060238878311, |
| "grad_norm": 2.03125, |
| "learning_rate": 1.3834601848628897e-05, |
| "loss": 0.8083207702636719, |
| "mean_token_accuracy": 0.7916932463645935, |
| "num_tokens": 14088345.0, |
| "step": 2275 |
| }, |
| { |
| "entropy": 0.8217170929908753, |
| "epoch": 0.7962610351393457, |
| "grad_norm": 2.203125, |
| "learning_rate": 1.3704793328468122e-05, |
| "loss": 0.7991712951660156, |
| "mean_token_accuracy": 0.7924394917488098, |
| "num_tokens": 14241089.0, |
| "step": 2300 |
| }, |
| { |
| "entropy": 0.8267798566818237, |
| "epoch": 0.8049160463908603, |
| "grad_norm": 2.09375, |
| "learning_rate": 1.3574257128394315e-05, |
| "loss": 0.8018778228759765, |
| "mean_token_accuracy": 0.7922985780239106, |
| "num_tokens": 14392337.0, |
| "step": 2325 |
| }, |
| { |
| "entropy": 0.8261617541313171, |
| "epoch": 0.8135710576423749, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.3443018887778494e-05, |
| "loss": 0.8028769683837891, |
| "mean_token_accuracy": 0.7926994669437408, |
| "num_tokens": 14544382.0, |
| "step": 2350 |
| }, |
| { |
| "entropy": 0.8380094921588898, |
| "epoch": 0.8222260688938896, |
| "grad_norm": 2.3125, |
| "learning_rate": 1.3311104383883537e-05, |
| "loss": 0.8226547241210938, |
| "mean_token_accuracy": 0.7886342179775238, |
| "num_tokens": 14703460.0, |
| "step": 2375 |
| }, |
| { |
| "entropy": 0.8281042277812958, |
| "epoch": 0.8308810801454042, |
| "grad_norm": 2.296875, |
| "learning_rate": 1.3178539526801106e-05, |
| "loss": 0.8025864410400391, |
| "mean_token_accuracy": 0.7910430765151978, |
| "num_tokens": 14860294.0, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.8308810801454042, |
| "eval_entropy": 0.8327872855741469, |
| "eval_loss": 0.8026647567749023, |
| "eval_mean_token_accuracy": 0.7919463300314106, |
| "eval_num_tokens": 14860294.0, |
| "eval_runtime": 16.2486, |
| "eval_samples_per_second": 299.41, |
| "eval_steps_per_second": 18.771, |
| "step": 2400 |
| }, |
| { |
| "entropy": 0.811096111536026, |
| "epoch": 0.8395360913969189, |
| "grad_norm": 2.4375, |
| "learning_rate": 1.3045350354362495e-05, |
| "loss": 0.7822922515869141, |
| "mean_token_accuracy": 0.7949872577190399, |
| "num_tokens": 15015328.0, |
| "step": 2425 |
| }, |
| { |
| "entropy": 0.8189793074131012, |
| "epoch": 0.8481911026484334, |
| "grad_norm": 2.3125, |
| "learning_rate": 1.2911563027024392e-05, |
| "loss": 0.8003229522705078, |
| "mean_token_accuracy": 0.7943521749973297, |
| "num_tokens": 15170350.0, |
| "step": 2450 |
| }, |
| { |
| "entropy": 0.8213041162490845, |
| "epoch": 0.856846113899948, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.2777203822730557e-05, |
| "loss": 0.7970082092285157, |
| "mean_token_accuracy": 0.7927951776981353, |
| "num_tokens": 15324153.0, |
| "step": 2475 |
| }, |
| { |
| "entropy": 0.824720960855484, |
| "epoch": 0.8655011251514627, |
| "grad_norm": 2.125, |
| "learning_rate": 1.2642299131750417e-05, |
| "loss": 0.7908332824707032, |
| "mean_token_accuracy": 0.7957222282886505, |
| "num_tokens": 15473303.0, |
| "step": 2500 |
| }, |
| { |
| "entropy": 0.838714919090271, |
| "epoch": 0.8741561364029773, |
| "grad_norm": 2.09375, |
| "learning_rate": 1.250687545149559e-05, |
| "loss": 0.8137520599365234, |
| "mean_token_accuracy": 0.7890112113952636, |
| "num_tokens": 15629017.0, |
| "step": 2525 |
| }, |
| { |
| "entropy": 0.8311066961288452, |
| "epoch": 0.882811147654492, |
| "grad_norm": 2.1875, |
| "learning_rate": 1.2370959381315393e-05, |
| "loss": 0.8034115600585937, |
| "mean_token_accuracy": 0.7926496362686157, |
| "num_tokens": 15777876.0, |
| "step": 2550 |
| }, |
| { |
| "entropy": 0.8097344720363617, |
| "epoch": 0.8914661589060066, |
| "grad_norm": 2.203125, |
| "learning_rate": 1.2234577617272282e-05, |
| "loss": 0.786445083618164, |
| "mean_token_accuracy": 0.7966820681095124, |
| "num_tokens": 15929184.0, |
| "step": 2575 |
| }, |
| { |
| "entropy": 0.8254410099983215, |
| "epoch": 0.9001211701575212, |
| "grad_norm": 2.296875, |
| "learning_rate": 1.209775694689837e-05, |
| "loss": 0.8023792266845703, |
| "mean_token_accuracy": 0.7923547208309174, |
| "num_tokens": 16087411.0, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.9001211701575212, |
| "eval_entropy": 0.8213896471945966, |
| "eval_loss": 0.8008216619491577, |
| "eval_mean_token_accuracy": 0.7924871415388389, |
| "eval_num_tokens": 16087411.0, |
| "eval_runtime": 16.2722, |
| "eval_samples_per_second": 298.976, |
| "eval_steps_per_second": 18.744, |
| "step": 2600 |
| }, |
| { |
| "entropy": 0.8255444514751434, |
| "epoch": 0.9087761814090358, |
| "grad_norm": 1.9140625, |
| "learning_rate": 1.1960524243933875e-05, |
| "loss": 0.8017765808105469, |
| "mean_token_accuracy": 0.7921687912940979, |
| "num_tokens": 16244042.0, |
| "step": 2625 |
| }, |
| { |
| "entropy": 0.8252915942668915, |
| "epoch": 0.9174311926605505, |
| "grad_norm": 2.21875, |
| "learning_rate": 1.182290646304874e-05, |
| "loss": 0.8029595947265625, |
| "mean_token_accuracy": 0.7924495422840119, |
| "num_tokens": 16399768.0, |
| "step": 2650 |
| }, |
| { |
| "entropy": 0.8625216126441956, |
| "epoch": 0.9260862039120651, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.1684930634548287e-05, |
| "loss": 0.838686752319336, |
| "mean_token_accuracy": 0.7851159811019898, |
| "num_tokens": 16557371.0, |
| "step": 2675 |
| }, |
| { |
| "entropy": 0.8215836095809936, |
| "epoch": 0.9347412151635797, |
| "grad_norm": 2.265625, |
| "learning_rate": 1.1546623859064053e-05, |
| "loss": 0.7977464294433594, |
| "mean_token_accuracy": 0.7929444241523743, |
| "num_tokens": 16713045.0, |
| "step": 2700 |
| }, |
| { |
| "entropy": 0.8286124646663666, |
| "epoch": 0.9433962264150944, |
| "grad_norm": 2.078125, |
| "learning_rate": 1.1408013302230765e-05, |
| "loss": 0.802153091430664, |
| "mean_token_accuracy": 0.7924544358253479, |
| "num_tokens": 16867145.0, |
| "step": 2725 |
| }, |
| { |
| "entropy": 0.8237406086921691, |
| "epoch": 0.9520512376666089, |
| "grad_norm": 2.15625, |
| "learning_rate": 1.126912618935063e-05, |
| "loss": 0.8021310424804687, |
| "mean_token_accuracy": 0.7944400310516357, |
| "num_tokens": 17014455.0, |
| "step": 2750 |
| }, |
| { |
| "entropy": 0.826482676267624, |
| "epoch": 0.9607062489181236, |
| "grad_norm": 2.140625, |
| "learning_rate": 1.1129989800045817e-05, |
| "loss": 0.8052169036865234, |
| "mean_token_accuracy": 0.7911915636062622, |
| "num_tokens": 17175484.0, |
| "step": 2775 |
| }, |
| { |
| "entropy": 0.8201354122161866, |
| "epoch": 0.9693612601696382, |
| "grad_norm": 2.421875, |
| "learning_rate": 1.099063146290032e-05, |
| "loss": 0.7985061645507813, |
| "mean_token_accuracy": 0.7938509380817413, |
| "num_tokens": 17334068.0, |
| "step": 2800 |
| }, |
| { |
| "epoch": 0.9693612601696382, |
| "eval_entropy": 0.8281321289109402, |
| "eval_loss": 0.7994962334632874, |
| "eval_mean_token_accuracy": 0.792740264681519, |
| "eval_num_tokens": 17334068.0, |
| "eval_runtime": 16.2806, |
| "eval_samples_per_second": 298.822, |
| "eval_steps_per_second": 18.734, |
| "step": 2800 |
| }, |
| { |
| "entropy": 0.8046394038200378, |
| "epoch": 0.9780162714211529, |
| "grad_norm": 2.46875, |
| "learning_rate": 1.08510785500922e-05, |
| "loss": 0.7775811004638672, |
| "mean_token_accuracy": 0.7966744971275329, |
| "num_tokens": 17483883.0, |
| "step": 2825 |
| }, |
| { |
| "entropy": 0.8306651365756988, |
| "epoch": 0.9866712826726675, |
| "grad_norm": 2.359375, |
| "learning_rate": 1.0711358472017244e-05, |
| "loss": 0.7954608154296875, |
| "mean_token_accuracy": 0.7919278919696808, |
| "num_tokens": 17639972.0, |
| "step": 2850 |
| }, |
| { |
| "entropy": 0.8295222818851471, |
| "epoch": 0.9953262939241821, |
| "grad_norm": 2.125, |
| "learning_rate": 1.0571498671905146e-05, |
| "loss": 0.8072916412353516, |
| "mean_token_accuracy": 0.7907999694347382, |
| "num_tokens": 17794430.0, |
| "step": 2875 |
| }, |
| { |
| "entropy": 0.8070371369926297, |
| "epoch": 1.0038082049506665, |
| "grad_norm": 2.09375, |
| "learning_rate": 1.0431526620429207e-05, |
| "loss": 0.760282211303711, |
| "mean_token_accuracy": 0.7995899867038337, |
| "num_tokens": 17946140.0, |
| "step": 2900 |
| }, |
| { |
| "entropy": 0.7756562912464142, |
| "epoch": 1.0124632162021812, |
| "grad_norm": 2.140625, |
| "learning_rate": 1.0291469810310669e-05, |
| "loss": 0.741202621459961, |
| "mean_token_accuracy": 0.803013744354248, |
| "num_tokens": 18100839.0, |
| "step": 2925 |
| }, |
| { |
| "entropy": 0.7452013885974884, |
| "epoch": 1.0211182274536956, |
| "grad_norm": 2.15625, |
| "learning_rate": 1.0151355750918712e-05, |
| "loss": 0.7064004516601563, |
| "mean_token_accuracy": 0.8126378917694091, |
| "num_tokens": 18256691.0, |
| "step": 2950 |
| }, |
| { |
| "entropy": 0.7718643927574158, |
| "epoch": 1.0297732387052103, |
| "grad_norm": 1.8671875, |
| "learning_rate": 1.001121196286718e-05, |
| "loss": 0.7311404418945312, |
| "mean_token_accuracy": 0.8063716220855713, |
| "num_tokens": 18418483.0, |
| "step": 2975 |
| }, |
| { |
| "entropy": 0.765803085565567, |
| "epoch": 1.038428249956725, |
| "grad_norm": 2.0, |
| "learning_rate": 9.871065972609089e-06, |
| "loss": 0.7332243347167968, |
| "mean_token_accuracy": 0.8049838674068451, |
| "num_tokens": 18576253.0, |
| "step": 3000 |
| }, |
| { |
| "epoch": 1.038428249956725, |
| "eval_entropy": 0.7786700289757525, |
| "eval_loss": 0.8012996912002563, |
| "eval_mean_token_accuracy": 0.792937566022404, |
| "eval_num_tokens": 18576253.0, |
| "eval_runtime": 16.277, |
| "eval_samples_per_second": 298.888, |
| "eval_steps_per_second": 18.738, |
| "step": 3000 |
| }, |
| { |
| "entropy": 0.7836507785320282, |
| "epoch": 1.0470832612082395, |
| "grad_norm": 2.0625, |
| "learning_rate": 9.730945307030009e-06, |
| "loss": 0.7418560028076172, |
| "mean_token_accuracy": 0.8028053915500641, |
| "num_tokens": 18729564.0, |
| "step": 3025 |
| }, |
| { |
| "entropy": 0.7653721117973328, |
| "epoch": 1.0557382724597542, |
| "grad_norm": 2.03125, |
| "learning_rate": 9.590877488041336e-06, |
| "loss": 0.7263675689697265, |
| "mean_token_accuracy": 0.807190809249878, |
| "num_tokens": 18891538.0, |
| "step": 3050 |
| }, |
| { |
| "entropy": 0.7604654181003571, |
| "epoch": 1.0643932837112688, |
| "grad_norm": 2.296875, |
| "learning_rate": 9.45089002717457e-06, |
| "loss": 0.72638916015625, |
| "mean_token_accuracy": 0.8084564554691315, |
| "num_tokens": 19050606.0, |
| "step": 3075 |
| }, |
| { |
| "entropy": 0.7386153995990753, |
| "epoch": 1.0730482949627835, |
| "grad_norm": 1.9765625, |
| "learning_rate": 9.311010420177597e-06, |
| "loss": 0.706187973022461, |
| "mean_token_accuracy": 0.8120154666900635, |
| "num_tokens": 19208707.0, |
| "step": 3100 |
| }, |
| { |
| "entropy": 0.7525534296035766, |
| "epoch": 1.081703306214298, |
| "grad_norm": 2.171875, |
| "learning_rate": 9.171266141614112e-06, |
| "loss": 0.7135391235351562, |
| "mean_token_accuracy": 0.8123032712936401, |
| "num_tokens": 19362737.0, |
| "step": 3125 |
| }, |
| { |
| "entropy": 0.7687537455558777, |
| "epoch": 1.0903583174658127, |
| "grad_norm": 2.109375, |
| "learning_rate": 9.031684639467159e-06, |
| "loss": 0.734495849609375, |
| "mean_token_accuracy": 0.8068291318416595, |
| "num_tokens": 19521058.0, |
| "step": 3150 |
| }, |
| { |
| "entropy": 0.7806685280799865, |
| "epoch": 1.0990133287173274, |
| "grad_norm": 2.046875, |
| "learning_rate": 8.892293329747923e-06, |
| "loss": 0.7469673156738281, |
| "mean_token_accuracy": 0.8041970551013946, |
| "num_tokens": 19675603.0, |
| "step": 3175 |
| }, |
| { |
| "entropy": 0.7709089267253876, |
| "epoch": 1.1076683399688418, |
| "grad_norm": 2.15625, |
| "learning_rate": 8.753119591110802e-06, |
| "loss": 0.7363986206054688, |
| "mean_token_accuracy": 0.8061197555065155, |
| "num_tokens": 19828898.0, |
| "step": 3200 |
| }, |
| { |
| "epoch": 1.1076683399688418, |
| "eval_entropy": 0.7721671974072691, |
| "eval_loss": 0.8010231852531433, |
| "eval_mean_token_accuracy": 0.7929807965872717, |
| "eval_num_tokens": 19828898.0, |
| "eval_runtime": 16.2529, |
| "eval_samples_per_second": 299.331, |
| "eval_steps_per_second": 18.766, |
| "step": 3200 |
| }, |
| { |
| "entropy": 0.7211595928668976, |
| "epoch": 1.1163233512203565, |
| "grad_norm": 2.046875, |
| "learning_rate": 8.614190759475789e-06, |
| "loss": 0.6826175689697266, |
| "mean_token_accuracy": 0.8186909162998199, |
| "num_tokens": 19982072.0, |
| "step": 3225 |
| }, |
| { |
| "entropy": 0.7708668720722198, |
| "epoch": 1.1249783624718712, |
| "grad_norm": 2.15625, |
| "learning_rate": 8.475534122659275e-06, |
| "loss": 0.7388634490966797, |
| "mean_token_accuracy": 0.8042692136764527, |
| "num_tokens": 20137744.0, |
| "step": 3250 |
| }, |
| { |
| "entropy": 0.7538685095310211, |
| "epoch": 1.133633373723386, |
| "grad_norm": 2.171875, |
| "learning_rate": 8.337176915014293e-06, |
| "loss": 0.7132722473144532, |
| "mean_token_accuracy": 0.8105318355560303, |
| "num_tokens": 20290524.0, |
| "step": 3275 |
| }, |
| { |
| "entropy": 0.7487189078330994, |
| "epoch": 1.1422883849749004, |
| "grad_norm": 2.046875, |
| "learning_rate": 8.199146312081265e-06, |
| "loss": 0.7089384460449218, |
| "mean_token_accuracy": 0.8108018398284912, |
| "num_tokens": 20443485.0, |
| "step": 3300 |
| }, |
| { |
| "entropy": 0.7543841946125031, |
| "epoch": 1.150943396226415, |
| "grad_norm": 2.0625, |
| "learning_rate": 8.061469425250273e-06, |
| "loss": 0.7227223205566407, |
| "mean_token_accuracy": 0.8090525925159454, |
| "num_tokens": 20595797.0, |
| "step": 3325 |
| }, |
| { |
| "entropy": 0.7593919610977173, |
| "epoch": 1.1595984074779297, |
| "grad_norm": 2.015625, |
| "learning_rate": 7.924173296435987e-06, |
| "loss": 0.7215187072753906, |
| "mean_token_accuracy": 0.8093890881538391, |
| "num_tokens": 20749554.0, |
| "step": 3350 |
| }, |
| { |
| "entropy": 0.7829481947422028, |
| "epoch": 1.1682534187294444, |
| "grad_norm": 2.171875, |
| "learning_rate": 7.78728489276618e-06, |
| "loss": 0.7491289520263672, |
| "mean_token_accuracy": 0.8015105819702149, |
| "num_tokens": 20909534.0, |
| "step": 3375 |
| }, |
| { |
| "entropy": 0.7392001152038574, |
| "epoch": 1.176908429980959, |
| "grad_norm": 2.1875, |
| "learning_rate": 7.650831101284977e-06, |
| "loss": 0.7020195007324219, |
| "mean_token_accuracy": 0.8150424540042878, |
| "num_tokens": 21059158.0, |
| "step": 3400 |
| }, |
| { |
| "epoch": 1.176908429980959, |
| "eval_entropy": 0.7698653085309951, |
| "eval_loss": 0.8008679747581482, |
| "eval_mean_token_accuracy": 0.793300522155449, |
| "eval_num_tokens": 21059158.0, |
| "eval_runtime": 16.2799, |
| "eval_samples_per_second": 298.835, |
| "eval_steps_per_second": 18.735, |
| "step": 3400 |
| }, |
| { |
| "entropy": 0.768788868188858, |
| "epoch": 1.1855634412324736, |
| "grad_norm": 2.046875, |
| "learning_rate": 7.514838723671799e-06, |
| "loss": 0.7372692108154297, |
| "mean_token_accuracy": 0.8063059449195862, |
| "num_tokens": 21208688.0, |
| "step": 3425 |
| }, |
| { |
| "entropy": 0.7443365502357483, |
| "epoch": 1.1942184524839883, |
| "grad_norm": 1.875, |
| "learning_rate": 7.379334470977093e-06, |
| "loss": 0.710152587890625, |
| "mean_token_accuracy": 0.811426283121109, |
| "num_tokens": 21365731.0, |
| "step": 3450 |
| }, |
| { |
| "entropy": 0.7438737845420837, |
| "epoch": 1.2028734637355027, |
| "grad_norm": 1.96875, |
| "learning_rate": 7.244344958375881e-06, |
| "loss": 0.710167465209961, |
| "mean_token_accuracy": 0.8120112872123718, |
| "num_tokens": 21519688.0, |
| "step": 3475 |
| }, |
| { |
| "entropy": 0.7728407967090607, |
| "epoch": 1.2115284749870174, |
| "grad_norm": 2.046875, |
| "learning_rate": 7.109896699940084e-06, |
| "loss": 0.7461888885498047, |
| "mean_token_accuracy": 0.8037091171741486, |
| "num_tokens": 21676493.0, |
| "step": 3500 |
| }, |
| { |
| "entropy": 0.7640272963047028, |
| "epoch": 1.2201834862385321, |
| "grad_norm": 2.203125, |
| "learning_rate": 6.976016103430765e-06, |
| "loss": 0.7235647583007813, |
| "mean_token_accuracy": 0.8083086097240448, |
| "num_tokens": 21825664.0, |
| "step": 3525 |
| }, |
| { |
| "entropy": 0.7460694527626037, |
| "epoch": 1.2288384974900468, |
| "grad_norm": 2.03125, |
| "learning_rate": 6.842729465111224e-06, |
| "loss": 0.7100708770751953, |
| "mean_token_accuracy": 0.8116568970680237, |
| "num_tokens": 21976504.0, |
| "step": 3550 |
| }, |
| { |
| "entropy": 0.759437519311905, |
| "epoch": 1.2374935087415613, |
| "grad_norm": 2.609375, |
| "learning_rate": 6.710062964581988e-06, |
| "loss": 0.722847900390625, |
| "mean_token_accuracy": 0.807286411523819, |
| "num_tokens": 22133165.0, |
| "step": 3575 |
| }, |
| { |
| "entropy": 0.7708945345878601, |
| "epoch": 1.246148519993076, |
| "grad_norm": 2.015625, |
| "learning_rate": 6.578042659638726e-06, |
| "loss": 0.7426469421386719, |
| "mean_token_accuracy": 0.8035933816432953, |
| "num_tokens": 22290456.0, |
| "step": 3600 |
| }, |
| { |
| "epoch": 1.246148519993076, |
| "eval_entropy": 0.7726592179204597, |
| "eval_loss": 0.8001708984375, |
| "eval_mean_token_accuracy": 0.7932597899046101, |
| "eval_num_tokens": 22290456.0, |
| "eval_runtime": 16.2656, |
| "eval_samples_per_second": 299.097, |
| "eval_steps_per_second": 18.751, |
| "step": 3600 |
| }, |
| { |
| "entropy": 0.7578576147556305, |
| "epoch": 1.2548035312445907, |
| "grad_norm": 2.078125, |
| "learning_rate": 6.446694481154104e-06, |
| "loss": 0.7225240325927734, |
| "mean_token_accuracy": 0.8097064507007599, |
| "num_tokens": 22446824.0, |
| "step": 3625 |
| }, |
| { |
| "entropy": 0.7762425947189331, |
| "epoch": 1.2634585424961053, |
| "grad_norm": 2.265625, |
| "learning_rate": 6.3160442279845206e-06, |
| "loss": 0.7412718200683593, |
| "mean_token_accuracy": 0.805392199754715, |
| "num_tokens": 22602588.0, |
| "step": 3650 |
| }, |
| { |
| "entropy": 0.7637201595306397, |
| "epoch": 1.2721135537476198, |
| "grad_norm": 2.25, |
| "learning_rate": 6.186117561902836e-06, |
| "loss": 0.7297930908203125, |
| "mean_token_accuracy": 0.8078333747386932, |
| "num_tokens": 22752707.0, |
| "step": 3675 |
| }, |
| { |
| "entropy": 0.7704725599288941, |
| "epoch": 1.2807685649991345, |
| "grad_norm": 2.125, |
| "learning_rate": 6.056940002557969e-06, |
| "loss": 0.7372798919677734, |
| "mean_token_accuracy": 0.8056825184822083, |
| "num_tokens": 22907803.0, |
| "step": 3700 |
| }, |
| { |
| "entropy": 0.7552802407741547, |
| "epoch": 1.2894235762506492, |
| "grad_norm": 2.15625, |
| "learning_rate": 5.928536922462449e-06, |
| "loss": 0.7166344451904297, |
| "mean_token_accuracy": 0.8109099209308624, |
| "num_tokens": 23059819.0, |
| "step": 3725 |
| }, |
| { |
| "entropy": 0.7541340529918671, |
| "epoch": 1.2980785875021636, |
| "grad_norm": 2.3125, |
| "learning_rate": 5.80093354200886e-06, |
| "loss": 0.7238423919677734, |
| "mean_token_accuracy": 0.809191288948059, |
| "num_tokens": 23217392.0, |
| "step": 3750 |
| }, |
| { |
| "entropy": 0.7676697385311126, |
| "epoch": 1.3067335987536783, |
| "grad_norm": 2.21875, |
| "learning_rate": 5.674154924516129e-06, |
| "loss": 0.7366065979003906, |
| "mean_token_accuracy": 0.8054041469097137, |
| "num_tokens": 23371073.0, |
| "step": 3775 |
| }, |
| { |
| "entropy": 0.7785192513465882, |
| "epoch": 1.315388610005193, |
| "grad_norm": 1.9140625, |
| "learning_rate": 5.548225971306731e-06, |
| "loss": 0.7439189147949219, |
| "mean_token_accuracy": 0.8034746491909027, |
| "num_tokens": 23524592.0, |
| "step": 3800 |
| }, |
| { |
| "epoch": 1.315388610005193, |
| "eval_entropy": 0.7713183484116538, |
| "eval_loss": 0.7998276352882385, |
| "eval_mean_token_accuracy": 0.7933589818047695, |
| "eval_num_tokens": 23524592.0, |
| "eval_runtime": 16.2844, |
| "eval_samples_per_second": 298.753, |
| "eval_steps_per_second": 18.73, |
| "step": 3800 |
| }, |
| { |
| "entropy": 0.7570435631275177, |
| "epoch": 1.3240436212567077, |
| "grad_norm": 2.078125, |
| "learning_rate": 5.423171416815651e-06, |
| "loss": 0.7266172790527343, |
| "mean_token_accuracy": 0.8094574224948883, |
| "num_tokens": 23682141.0, |
| "step": 3825 |
| }, |
| { |
| "entropy": 0.76507066488266, |
| "epoch": 1.3326986325082222, |
| "grad_norm": 2.15625, |
| "learning_rate": 5.299015823732163e-06, |
| "loss": 0.7294864654541016, |
| "mean_token_accuracy": 0.8064786922931672, |
| "num_tokens": 23839050.0, |
| "step": 3850 |
| }, |
| { |
| "entropy": 0.7562107217311859, |
| "epoch": 1.3413536437597369, |
| "grad_norm": 2.0625, |
| "learning_rate": 5.175783578175309e-06, |
| "loss": 0.719031982421875, |
| "mean_token_accuracy": 0.8091187167167664, |
| "num_tokens": 23992205.0, |
| "step": 3875 |
| }, |
| { |
| "entropy": 0.7576862835884094, |
| "epoch": 1.3500086550112516, |
| "grad_norm": 2.109375, |
| "learning_rate": 5.053498884904122e-06, |
| "loss": 0.7270169830322266, |
| "mean_token_accuracy": 0.8094841420650483, |
| "num_tokens": 24139859.0, |
| "step": 3900 |
| }, |
| { |
| "entropy": 0.7541490840911865, |
| "epoch": 1.3586636662627662, |
| "grad_norm": 2.109375, |
| "learning_rate": 4.932185762563381e-06, |
| "loss": 0.7113311004638672, |
| "mean_token_accuracy": 0.8112999868392944, |
| "num_tokens": 24289196.0, |
| "step": 3925 |
| }, |
| { |
| "entropy": 0.7688007843494415, |
| "epoch": 1.3673186775142807, |
| "grad_norm": 2.1875, |
| "learning_rate": 4.811868038965999e-06, |
| "loss": 0.745551986694336, |
| "mean_token_accuracy": 0.8050249302387238, |
| "num_tokens": 24447938.0, |
| "step": 3950 |
| }, |
| { |
| "entropy": 0.7551292729377747, |
| "epoch": 1.3759736887657954, |
| "grad_norm": 2.1875, |
| "learning_rate": 4.692569346412864e-06, |
| "loss": 0.7206728363037109, |
| "mean_token_accuracy": 0.8098892831802368, |
| "num_tokens": 24598742.0, |
| "step": 3975 |
| }, |
| { |
| "entropy": 0.7587357807159424, |
| "epoch": 1.38462870001731, |
| "grad_norm": 2.125, |
| "learning_rate": 4.574313117051059e-06, |
| "loss": 0.7276847076416015, |
| "mean_token_accuracy": 0.8077455127239227, |
| "num_tokens": 24750406.0, |
| "step": 4000 |
| }, |
| { |
| "epoch": 1.38462870001731, |
| "eval_entropy": 0.7717335020909544, |
| "eval_loss": 0.7996781468391418, |
| "eval_mean_token_accuracy": 0.7934077425081222, |
| "eval_num_tokens": 24750406.0, |
| "eval_runtime": 16.2676, |
| "eval_samples_per_second": 299.061, |
| "eval_steps_per_second": 18.749, |
| "step": 4000 |
| }, |
| { |
| "entropy": 0.7805033266544342, |
| "epoch": 1.3932837112688246, |
| "grad_norm": 2.078125, |
| "learning_rate": 4.457122578271457e-06, |
| "loss": 0.7509593200683594, |
| "mean_token_accuracy": 0.8028152561187745, |
| "num_tokens": 24906659.0, |
| "step": 4025 |
| }, |
| { |
| "entropy": 0.7705829620361329, |
| "epoch": 1.4019387225203392, |
| "grad_norm": 2.125, |
| "learning_rate": 4.341020748146449e-06, |
| "loss": 0.7392497253417969, |
| "mean_token_accuracy": 0.8049503290653228, |
| "num_tokens": 25066654.0, |
| "step": 4050 |
| }, |
| { |
| "entropy": 0.7581047600507737, |
| "epoch": 1.410593733771854, |
| "grad_norm": 2.0, |
| "learning_rate": 4.226030430908869e-06, |
| "loss": 0.7285484313964844, |
| "mean_token_accuracy": 0.8085175144672394, |
| "num_tokens": 25220415.0, |
| "step": 4075 |
| }, |
| { |
| "entropy": 0.7656786620616913, |
| "epoch": 1.4192487450233684, |
| "grad_norm": 2.21875, |
| "learning_rate": 4.112174212472864e-06, |
| "loss": 0.7323892211914063, |
| "mean_token_accuracy": 0.8051603281497955, |
| "num_tokens": 25374458.0, |
| "step": 4100 |
| }, |
| { |
| "entropy": 0.7522096228599549, |
| "epoch": 1.427903756274883, |
| "grad_norm": 2.171875, |
| "learning_rate": 3.99947445599767e-06, |
| "loss": 0.7212752532958985, |
| "mean_token_accuracy": 0.8096666383743286, |
| "num_tokens": 25525969.0, |
| "step": 4125 |
| }, |
| { |
| "entropy": 0.7608710610866547, |
| "epoch": 1.4365587675263978, |
| "grad_norm": 2.40625, |
| "learning_rate": 3.887953297495121e-06, |
| "loss": 0.7316089630126953, |
| "mean_token_accuracy": 0.8074719154834747, |
| "num_tokens": 25681452.0, |
| "step": 4150 |
| }, |
| { |
| "entropy": 0.7639850771427155, |
| "epoch": 1.4452137787779125, |
| "grad_norm": 2.140625, |
| "learning_rate": 3.77763264148181e-06, |
| "loss": 0.7320518493652344, |
| "mean_token_accuracy": 0.809201009273529, |
| "num_tokens": 25838398.0, |
| "step": 4175 |
| }, |
| { |
| "entropy": 0.7513358175754548, |
| "epoch": 1.4538687900294271, |
| "grad_norm": 2.21875, |
| "learning_rate": 3.6685341566766785e-06, |
| "loss": 0.7206896209716797, |
| "mean_token_accuracy": 0.8095373523235321, |
| "num_tokens": 25992640.0, |
| "step": 4200 |
| }, |
| { |
| "epoch": 1.4538687900294271, |
| "eval_entropy": 0.7707103953986871, |
| "eval_loss": 0.7995045185089111, |
| "eval_mean_token_accuracy": 0.7933821330305005, |
| "eval_num_tokens": 25992640.0, |
| "eval_runtime": 16.2677, |
| "eval_samples_per_second": 299.058, |
| "eval_steps_per_second": 18.749, |
| "step": 4200 |
| }, |
| { |
| "entropy": 0.7595930647850037, |
| "epoch": 1.4625238012809416, |
| "grad_norm": 2.171875, |
| "learning_rate": 3.5606792717449455e-06, |
| "loss": 0.7264796447753906, |
| "mean_token_accuracy": 0.8094517874717713, |
| "num_tokens": 26141304.0, |
| "step": 4225 |
| }, |
| { |
| "entropy": 0.7630014491081237, |
| "epoch": 1.4711788125324563, |
| "grad_norm": 2.15625, |
| "learning_rate": 3.4540891710891768e-06, |
| "loss": 0.7287496185302734, |
| "mean_token_accuracy": 0.808629115819931, |
| "num_tokens": 26295315.0, |
| "step": 4250 |
| }, |
| { |
| "entropy": 0.7566706609725952, |
| "epoch": 1.479833823783971, |
| "grad_norm": 2.34375, |
| "learning_rate": 3.3487847906883232e-06, |
| "loss": 0.7267755126953125, |
| "mean_token_accuracy": 0.8091002118587494, |
| "num_tokens": 26450821.0, |
| "step": 4275 |
| }, |
| { |
| "entropy": 0.7702415359020233, |
| "epoch": 1.4884888350354855, |
| "grad_norm": 2.09375, |
| "learning_rate": 3.2447868139855954e-06, |
| "loss": 0.74352294921875, |
| "mean_token_accuracy": 0.8064157390594482, |
| "num_tokens": 26610364.0, |
| "step": 4300 |
| }, |
| { |
| "entropy": 0.78652991771698, |
| "epoch": 1.4971438462870001, |
| "grad_norm": 2.359375, |
| "learning_rate": 3.142115667825868e-06, |
| "loss": 0.7553633117675781, |
| "mean_token_accuracy": 0.8014207696914672, |
| "num_tokens": 26766393.0, |
| "step": 4325 |
| }, |
| { |
| "entropy": 0.7758943355083465, |
| "epoch": 1.5057988575385148, |
| "grad_norm": 2.0, |
| "learning_rate": 3.0407915184435544e-06, |
| "loss": 0.7414107513427735, |
| "mean_token_accuracy": 0.8034427273273468, |
| "num_tokens": 26928821.0, |
| "step": 4350 |
| }, |
| { |
| "entropy": 0.7746455824375152, |
| "epoch": 1.5144538687900293, |
| "grad_norm": 2.15625, |
| "learning_rate": 2.9408342675016288e-06, |
| "loss": 0.7338740539550781, |
| "mean_token_accuracy": 0.8053066456317901, |
| "num_tokens": 27081955.0, |
| "step": 4375 |
| }, |
| { |
| "entropy": 0.7544065153598786, |
| "epoch": 1.5231088800415442, |
| "grad_norm": 2.125, |
| "learning_rate": 2.84226354818263e-06, |
| "loss": 0.7168594360351562, |
| "mean_token_accuracy": 0.8104126238822937, |
| "num_tokens": 27233223.0, |
| "step": 4400 |
| }, |
| { |
| "epoch": 1.5231088800415442, |
| "eval_entropy": 0.770115079156688, |
| "eval_loss": 0.7993313074111938, |
| "eval_mean_token_accuracy": 0.7934993093131019, |
| "eval_num_tokens": 27233223.0, |
| "eval_runtime": 16.2903, |
| "eval_samples_per_second": 298.644, |
| "eval_steps_per_second": 18.723, |
| "step": 4400 |
| }, |
| { |
| "entropy": 0.7756598269939423, |
| "epoch": 1.5317638912930587, |
| "grad_norm": 2.15625, |
| "learning_rate": 2.74509872133238e-06, |
| "loss": 0.750801010131836, |
| "mean_token_accuracy": 0.8043915629386902, |
| "num_tokens": 27385671.0, |
| "step": 4425 |
| }, |
| { |
| "entropy": 0.7870103931427002, |
| "epoch": 1.5404189025445731, |
| "grad_norm": 2.46875, |
| "learning_rate": 2.6493588716572306e-06, |
| "loss": 0.7561086273193359, |
| "mean_token_accuracy": 0.8031144332885742, |
| "num_tokens": 27542323.0, |
| "step": 4450 |
| }, |
| { |
| "entropy": 0.7758941638469696, |
| "epoch": 1.549073913796088, |
| "grad_norm": 2.359375, |
| "learning_rate": 2.555062803975512e-06, |
| "loss": 0.7479061889648437, |
| "mean_token_accuracy": 0.8041390895843505, |
| "num_tokens": 27695755.0, |
| "step": 4475 |
| }, |
| { |
| "entropy": 0.754768637418747, |
| "epoch": 1.5577289250476025, |
| "grad_norm": 2.4375, |
| "learning_rate": 2.4622290395239613e-06, |
| "loss": 0.7247459411621093, |
| "mean_token_accuracy": 0.8090137577056885, |
| "num_tokens": 27847694.0, |
| "step": 4500 |
| }, |
| { |
| "entropy": 0.7636888045072555, |
| "epoch": 1.5663839362991172, |
| "grad_norm": 2.140625, |
| "learning_rate": 2.3708758123198904e-06, |
| "loss": 0.7294938659667969, |
| "mean_token_accuracy": 0.8078456258773804, |
| "num_tokens": 28001389.0, |
| "step": 4525 |
| }, |
| { |
| "entropy": 0.7723946702480317, |
| "epoch": 1.575038947550632, |
| "grad_norm": 1.9453125, |
| "learning_rate": 2.2810210655796915e-06, |
| "loss": 0.7421239471435547, |
| "mean_token_accuracy": 0.8051141202449799, |
| "num_tokens": 28155633.0, |
| "step": 4550 |
| }, |
| { |
| "entropy": 0.7729098165035247, |
| "epoch": 1.5836939588021464, |
| "grad_norm": 1.9375, |
| "learning_rate": 2.192682448194555e-06, |
| "loss": 0.7385528564453125, |
| "mean_token_accuracy": 0.8061604249477387, |
| "num_tokens": 28316815.0, |
| "step": 4575 |
| }, |
| { |
| "entropy": 0.7528582894802094, |
| "epoch": 1.592348970053661, |
| "grad_norm": 2.234375, |
| "learning_rate": 2.1058773112639054e-06, |
| "loss": 0.719280014038086, |
| "mean_token_accuracy": 0.8096978998184204, |
| "num_tokens": 28473438.0, |
| "step": 4600 |
| }, |
| { |
| "epoch": 1.592348970053661, |
| "eval_entropy": 0.7718443285246365, |
| "eval_loss": 0.7991446852684021, |
| "eval_mean_token_accuracy": 0.7934910416603088, |
| "eval_num_tokens": 28473438.0, |
| "eval_runtime": 16.2845, |
| "eval_samples_per_second": 298.751, |
| "eval_steps_per_second": 18.729, |
| "step": 4600 |
| }, |
| { |
| "entropy": 0.7752782785892487, |
| "epoch": 1.6010039813051757, |
| "grad_norm": 2.203125, |
| "learning_rate": 2.020622704687398e-06, |
| "loss": 0.7455803680419922, |
| "mean_token_accuracy": 0.8029440557956695, |
| "num_tokens": 28629933.0, |
| "step": 4625 |
| }, |
| { |
| "entropy": 0.7682564723491668, |
| "epoch": 1.6096589925566902, |
| "grad_norm": 2.28125, |
| "learning_rate": 1.936935373816046e-06, |
| "loss": 0.7349263000488281, |
| "mean_token_accuracy": 0.8075251269340515, |
| "num_tokens": 28779447.0, |
| "step": 4650 |
| }, |
| { |
| "entropy": 0.7602540409564972, |
| "epoch": 1.6183140038082051, |
| "grad_norm": 2.15625, |
| "learning_rate": 1.8548317561631534e-06, |
| "loss": 0.722344741821289, |
| "mean_token_accuracy": 0.809203851222992, |
| "num_tokens": 28935999.0, |
| "step": 4675 |
| }, |
| { |
| "entropy": 0.7502141880989075, |
| "epoch": 1.6269690150597196, |
| "grad_norm": 1.984375, |
| "learning_rate": 1.7743279781757493e-06, |
| "loss": 0.718406982421875, |
| "mean_token_accuracy": 0.8091218245029449, |
| "num_tokens": 29092219.0, |
| "step": 4700 |
| }, |
| { |
| "entropy": 0.7485624659061432, |
| "epoch": 1.635624026311234, |
| "grad_norm": 1.9375, |
| "learning_rate": 1.6954398520670867e-06, |
| "loss": 0.7138198852539063, |
| "mean_token_accuracy": 0.8104638087749482, |
| "num_tokens": 29249386.0, |
| "step": 4725 |
| }, |
| { |
| "entropy": 0.7689664399623871, |
| "epoch": 1.644279037562749, |
| "grad_norm": 2.0625, |
| "learning_rate": 1.6181828727108772e-06, |
| "loss": 0.7358683776855469, |
| "mean_token_accuracy": 0.8066857266426086, |
| "num_tokens": 29406880.0, |
| "step": 4750 |
| }, |
| { |
| "entropy": 0.7672972738742828, |
| "epoch": 1.6529340488142634, |
| "grad_norm": 2.40625, |
| "learning_rate": 1.542572214597844e-06, |
| "loss": 0.7319490051269532, |
| "mean_token_accuracy": 0.8055110704898835, |
| "num_tokens": 29558251.0, |
| "step": 4775 |
| }, |
| { |
| "entropy": 0.7510581874847412, |
| "epoch": 1.6615890600657781, |
| "grad_norm": 2.078125, |
| "learning_rate": 1.4686227288552357e-06, |
| "loss": 0.7089730072021484, |
| "mean_token_accuracy": 0.811128660440445, |
| "num_tokens": 29713889.0, |
| "step": 4800 |
| }, |
| { |
| "epoch": 1.6615890600657781, |
| "eval_entropy": 0.7707234706057877, |
| "eval_loss": 0.7992180585861206, |
| "eval_mean_token_accuracy": 0.7935016180648178, |
| "eval_num_tokens": 29713889.0, |
| "eval_runtime": 16.3186, |
| "eval_samples_per_second": 298.126, |
| "eval_steps_per_second": 18.69, |
| "step": 4800 |
| }, |
| { |
| "entropy": 0.7817390990257264, |
| "epoch": 1.6702440713172928, |
| "grad_norm": 2.25, |
| "learning_rate": 1.3963489403297992e-06, |
| "loss": 0.7543077087402343, |
| "mean_token_accuracy": 0.8011208462715149, |
| "num_tokens": 29870315.0, |
| "step": 4825 |
| }, |
| { |
| "entropy": 0.7862332105636597, |
| "epoch": 1.6788990825688073, |
| "grad_norm": 2.125, |
| "learning_rate": 1.3257650447349035e-06, |
| "loss": 0.7527214050292969, |
| "mean_token_accuracy": 0.8031284868717193, |
| "num_tokens": 30023428.0, |
| "step": 4850 |
| }, |
| { |
| "entropy": 0.7391238558292389, |
| "epoch": 1.687554093820322, |
| "grad_norm": 2.046875, |
| "learning_rate": 1.2568849058622435e-06, |
| "loss": 0.7039041137695312, |
| "mean_token_accuracy": 0.811707170009613, |
| "num_tokens": 30181568.0, |
| "step": 4875 |
| }, |
| { |
| "entropy": 0.7737973690032959, |
| "epoch": 1.6962091050718366, |
| "grad_norm": 2.125, |
| "learning_rate": 1.1897220528588005e-06, |
| "loss": 0.7400728607177735, |
| "mean_token_accuracy": 0.8040676033496856, |
| "num_tokens": 30334041.0, |
| "step": 4900 |
| }, |
| { |
| "entropy": 0.7637208473682403, |
| "epoch": 1.704864116323351, |
| "grad_norm": 2.265625, |
| "learning_rate": 1.1242896775694923e-06, |
| "loss": 0.7268269348144532, |
| "mean_token_accuracy": 0.8077472651004791, |
| "num_tokens": 30488073.0, |
| "step": 4925 |
| }, |
| { |
| "entropy": 0.7567976391315461, |
| "epoch": 1.7135191275748658, |
| "grad_norm": 1.984375, |
| "learning_rate": 1.0606006319460794e-06, |
| "loss": 0.7226065826416016, |
| "mean_token_accuracy": 0.8086667931079865, |
| "num_tokens": 30643659.0, |
| "step": 4950 |
| }, |
| { |
| "entropy": 0.7528193962574005, |
| "epoch": 1.7221741388263805, |
| "grad_norm": 2.09375, |
| "learning_rate": 9.986674255228546e-07, |
| "loss": 0.7181925964355469, |
| "mean_token_accuracy": 0.8110329484939576, |
| "num_tokens": 30794462.0, |
| "step": 4975 |
| }, |
| { |
| "entropy": 0.7771501529216767, |
| "epoch": 1.730829150077895, |
| "grad_norm": 2.203125, |
| "learning_rate": 9.385022229595631e-07, |
| "loss": 0.7418533325195312, |
| "mean_token_accuracy": 0.8045794427394867, |
| "num_tokens": 30945283.0, |
| "step": 5000 |
| }, |
| { |
| "epoch": 1.730829150077895, |
| "eval_entropy": 0.7709188702653667, |
| "eval_loss": 0.7991884350776672, |
| "eval_mean_token_accuracy": 0.7934240538565839, |
| "eval_num_tokens": 30945283.0, |
| "eval_runtime": 16.2662, |
| "eval_samples_per_second": 299.086, |
| "eval_steps_per_second": 18.751, |
| "step": 5000 |
| }, |
| { |
| "entropy": 0.7722408485412597, |
| "epoch": 1.7394841613294099, |
| "grad_norm": 2.0625, |
| "learning_rate": 8.801168416520733e-07, |
| "loss": 0.7394268035888671, |
| "mean_token_accuracy": 0.806780993938446, |
| "num_tokens": 31105352.0, |
| "step": 5025 |
| }, |
| { |
| "entropy": 0.7758919060230255, |
| "epoch": 1.7481391725809243, |
| "grad_norm": 2.0625, |
| "learning_rate": 8.235227494112463e-07, |
| "loss": 0.7348842620849609, |
| "mean_token_accuracy": 0.8069607448577881, |
| "num_tokens": 31253216.0, |
| "step": 5050 |
| }, |
| { |
| "entropy": 0.7573688685894012, |
| "epoch": 1.756794183832439, |
| "grad_norm": 2.0, |
| "learning_rate": 7.687310622104982e-07, |
| "loss": 0.7183684539794922, |
| "mean_token_accuracy": 0.8097042238712311, |
| "num_tokens": 31404175.0, |
| "step": 5075 |
| }, |
| { |
| "entropy": 0.7699829995632171, |
| "epoch": 1.7654491950839537, |
| "grad_norm": 2.21875, |
| "learning_rate": 7.157525420024181e-07, |
| "loss": 0.7363372802734375, |
| "mean_token_accuracy": 0.8065412914752961, |
| "num_tokens": 31553043.0, |
| "step": 5100 |
| }, |
| { |
| "entropy": 0.7569133055210113, |
| "epoch": 1.7741042063354682, |
| "grad_norm": 1.9609375, |
| "learning_rate": 6.645975946049732e-07, |
| "loss": 0.7242645263671875, |
| "mean_token_accuracy": 0.8096802568435669, |
| "num_tokens": 31708719.0, |
| "step": 5125 |
| }, |
| { |
| "entropy": 0.763712112903595, |
| "epoch": 1.7827592175869829, |
| "grad_norm": 2.125, |
| "learning_rate": 6.152762676576352e-07, |
| "loss": 0.7330033111572266, |
| "mean_token_accuracy": 0.8064149105548859, |
| "num_tokens": 31869942.0, |
| "step": 5150 |
| }, |
| { |
| "entropy": 0.7759097361564636, |
| "epoch": 1.7914142288384975, |
| "grad_norm": 2.203125, |
| "learning_rate": 5.677982486478595e-07, |
| "loss": 0.7459344482421875, |
| "mean_token_accuracy": 0.8050018990039826, |
| "num_tokens": 32024066.0, |
| "step": 5175 |
| }, |
| { |
| "entropy": 0.7608749055862427, |
| "epoch": 1.800069240090012, |
| "grad_norm": 2.09375, |
| "learning_rate": 5.221728630083201e-07, |
| "loss": 0.7280139923095703, |
| "mean_token_accuracy": 0.8076766312122345, |
| "num_tokens": 32180495.0, |
| "step": 5200 |
| }, |
| { |
| "epoch": 1.800069240090012, |
| "eval_entropy": 0.7705254261610938, |
| "eval_loss": 0.7991676330566406, |
| "eval_mean_token_accuracy": 0.7934689746528375, |
| "eval_num_tokens": 32180495.0, |
| "eval_runtime": 16.2695, |
| "eval_samples_per_second": 299.026, |
| "eval_steps_per_second": 18.747, |
| "step": 5200 |
| }, |
| { |
| "entropy": 0.7587528014183045, |
| "epoch": 1.8087242513415267, |
| "grad_norm": 2.3125, |
| "learning_rate": 4.784090722852353e-07, |
| "loss": 0.7158480834960937, |
| "mean_token_accuracy": 0.809622370004654, |
| "num_tokens": 32338906.0, |
| "step": 5225 |
| }, |
| { |
| "entropy": 0.7616716659069062, |
| "epoch": 1.8173792625930414, |
| "grad_norm": 2.109375, |
| "learning_rate": 4.365154723781895e-07, |
| "loss": 0.7293673706054687, |
| "mean_token_accuracy": 0.8077958106994629, |
| "num_tokens": 32495631.0, |
| "step": 5250 |
| }, |
| { |
| "entropy": 0.737936383485794, |
| "epoch": 1.8260342738445559, |
| "grad_norm": 2.046875, |
| "learning_rate": 3.965002918517569e-07, |
| "loss": 0.7038777160644532, |
| "mean_token_accuracy": 0.8131058704853058, |
| "num_tokens": 32651647.0, |
| "step": 5275 |
| }, |
| { |
| "entropy": 0.7654093158245087, |
| "epoch": 1.8346892850960708, |
| "grad_norm": 2.21875, |
| "learning_rate": 3.5837139031928315e-07, |
| "loss": 0.7362342071533203, |
| "mean_token_accuracy": 0.8066077816486359, |
| "num_tokens": 32809688.0, |
| "step": 5300 |
| }, |
| { |
| "entropy": 0.7611633980274201, |
| "epoch": 1.8433442963475852, |
| "grad_norm": 2.25, |
| "learning_rate": 3.221362568991293e-07, |
| "loss": 0.7225106811523437, |
| "mean_token_accuracy": 0.809012303352356, |
| "num_tokens": 32962531.0, |
| "step": 5325 |
| }, |
| { |
| "entropy": 0.7749582767486572, |
| "epoch": 1.8519993075991, |
| "grad_norm": 2.28125, |
| "learning_rate": 2.8780200874370165e-07, |
| "loss": 0.7392816925048828, |
| "mean_token_accuracy": 0.803524352312088, |
| "num_tokens": 33122116.0, |
| "step": 5350 |
| }, |
| { |
| "entropy": 0.7610144579410553, |
| "epoch": 1.8606543188506146, |
| "grad_norm": 2.28125, |
| "learning_rate": 2.5537538964151676e-07, |
| "loss": 0.7229084014892578, |
| "mean_token_accuracy": 0.8083787798881531, |
| "num_tokens": 33273413.0, |
| "step": 5375 |
| }, |
| { |
| "entropy": 0.791879060268402, |
| "epoch": 1.869309330102129, |
| "grad_norm": 2.234375, |
| "learning_rate": 2.248627686926219e-07, |
| "loss": 0.7606393432617188, |
| "mean_token_accuracy": 0.8025156807899475, |
| "num_tokens": 33421728.0, |
| "step": 5400 |
| }, |
| { |
| "epoch": 1.869309330102129, |
| "eval_entropy": 0.770491715630547, |
| "eval_loss": 0.7991881370544434, |
| "eval_mean_token_accuracy": 0.7935233919346919, |
| "eval_num_tokens": 33421728.0, |
| "eval_runtime": 16.2639, |
| "eval_samples_per_second": 299.128, |
| "eval_steps_per_second": 18.753, |
| "step": 5400 |
| }, |
| { |
| "entropy": 0.7707629156112671, |
| "epoch": 1.8779643413536438, |
| "grad_norm": 2.265625, |
| "learning_rate": 1.9627013905760605e-07, |
| "loss": 0.7430142974853515, |
| "mean_token_accuracy": 0.8047948408126832, |
| "num_tokens": 33576359.0, |
| "step": 5425 |
| }, |
| { |
| "entropy": 0.7577464747428894, |
| "epoch": 1.8866193526051585, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.6960311678044018e-07, |
| "loss": 0.7168014526367188, |
| "mean_token_accuracy": 0.8094685411453247, |
| "num_tokens": 33723770.0, |
| "step": 5450 |
| }, |
| { |
| "entropy": 0.7747404301166534, |
| "epoch": 1.895274363856673, |
| "grad_norm": 2.0625, |
| "learning_rate": 1.4486693968540743e-07, |
| "loss": 0.7389591979980469, |
| "mean_token_accuracy": 0.8045553112030029, |
| "num_tokens": 33879362.0, |
| "step": 5475 |
| }, |
| { |
| "entropy": 0.7680295896530152, |
| "epoch": 1.9039293751081876, |
| "grad_norm": 2.046875, |
| "learning_rate": 1.22066466348304e-07, |
| "loss": 0.7351342010498046, |
| "mean_token_accuracy": 0.8047092247009278, |
| "num_tokens": 34035862.0, |
| "step": 5500 |
| }, |
| { |
| "entropy": 0.7560386550426483, |
| "epoch": 1.9125843863597023, |
| "grad_norm": 2.15625, |
| "learning_rate": 1.0120617514214115e-07, |
| "loss": 0.7233296966552735, |
| "mean_token_accuracy": 0.8088453054428101, |
| "num_tokens": 34192367.0, |
| "step": 5525 |
| }, |
| { |
| "entropy": 0.749894061088562, |
| "epoch": 1.9212393976112168, |
| "grad_norm": 2.234375, |
| "learning_rate": 8.229016335752238e-08, |
| "loss": 0.721561279296875, |
| "mean_token_accuracy": 0.8098326814174652, |
| "num_tokens": 34346639.0, |
| "step": 5550 |
| }, |
| { |
| "entropy": 0.7512237560749054, |
| "epoch": 1.9298944088627317, |
| "grad_norm": 2.203125, |
| "learning_rate": 6.532214639786926e-08, |
| "loss": 0.7133358001708985, |
| "mean_token_accuracy": 0.8117510068416596, |
| "num_tokens": 34501969.0, |
| "step": 5575 |
| }, |
| { |
| "entropy": 0.7577315938472747, |
| "epoch": 1.9385494201142461, |
| "grad_norm": 1.984375, |
| "learning_rate": 5.03054570496575e-08, |
| "loss": 0.7302328491210938, |
| "mean_token_accuracy": 0.8082369923591614, |
| "num_tokens": 34658561.0, |
| "step": 5600 |
| }, |
| { |
| "epoch": 1.9385494201142461, |
| "eval_entropy": 0.7700900946484237, |
| "eval_loss": 0.7992120981216431, |
| "eval_mean_token_accuracy": 0.7935370582049011, |
| "eval_num_tokens": 34658561.0, |
| "eval_runtime": 16.284, |
| "eval_samples_per_second": 298.76, |
| "eval_steps_per_second": 18.73, |
| "step": 5600 |
| }, |
| { |
| "entropy": 0.7591597294807434, |
| "epoch": 1.9472044313657608, |
| "grad_norm": 2.203125, |
| "learning_rate": 3.724304482781049e-08, |
| "loss": 0.7255685424804688, |
| "mean_token_accuracy": 0.808221675157547, |
| "num_tokens": 34809418.0, |
| "step": 5625 |
| }, |
| { |
| "entropy": 0.7517114734649658, |
| "epoch": 1.9558594426172755, |
| "grad_norm": 2.078125, |
| "learning_rate": 2.613747539635725e-08, |
| "loss": 0.7198385620117187, |
| "mean_token_accuracy": 0.8097265243530274, |
| "num_tokens": 34958876.0, |
| "step": 5650 |
| }, |
| { |
| "entropy": 0.7789162063598633, |
| "epoch": 1.96451445386879, |
| "grad_norm": 2.15625, |
| "learning_rate": 1.6990930064507694e-08, |
| "loss": 0.7432637786865235, |
| "mean_token_accuracy": 0.8060086870193481, |
| "num_tokens": 35115995.0, |
| "step": 5675 |
| }, |
| { |
| "entropy": 0.7361849272251129, |
| "epoch": 1.9731694651203047, |
| "grad_norm": 2.015625, |
| "learning_rate": 9.805205358197622e-09, |
| "loss": 0.7050675201416016, |
| "mean_token_accuracy": 0.8122973215579986, |
| "num_tokens": 35274445.0, |
| "step": 5700 |
| }, |
| { |
| "entropy": 0.7350089454650879, |
| "epoch": 1.9818244763718194, |
| "grad_norm": 2.15625, |
| "learning_rate": 4.5817126672331735e-09, |
| "loss": 0.697210922241211, |
| "mean_token_accuracy": 0.816340823173523, |
| "num_tokens": 35428606.0, |
| "step": 5725 |
| }, |
| { |
| "entropy": 0.7550158965587616, |
| "epoch": 1.9904794876233338, |
| "grad_norm": 2.171875, |
| "learning_rate": 1.3214779680637002e-09, |
| "loss": 0.71933349609375, |
| "mean_token_accuracy": 0.8101057040691376, |
| "num_tokens": 35582057.0, |
| "step": 5750 |
| }, |
| { |
| "entropy": 0.7830293524265289, |
| "epoch": 1.9991344988748485, |
| "grad_norm": 2.171875, |
| "learning_rate": 2.514162226741057e-11, |
| "loss": 0.7570413970947265, |
| "mean_token_accuracy": 0.8038502883911133, |
| "num_tokens": 35740061.0, |
| "step": 5775 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_entropy": 0.7710395187628074, |
| "eval_loss": 0.7991825342178345, |
| "eval_mean_token_accuracy": 0.7934717604371367, |
| "eval_num_tokens": 35756220.0, |
| "eval_runtime": 16.2915, |
| "eval_samples_per_second": 298.623, |
| "eval_steps_per_second": 18.721, |
| "step": 5778 |
| } |
| ], |
| "logging_steps": 25, |
| "max_steps": 5778, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 200, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.4611218403146547e+17, |
| "train_batch_size": 16, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
