| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.0, |
| "eval_steps": 500, |
| "global_step": 1890, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 0.9484764046967029, |
| "epoch": 0.05291005291005291, |
| "grad_norm": 3.640625, |
| "learning_rate": 3.2142857142857147e-06, |
| "loss": 0.8941335678100586, |
| "mean_token_accuracy": 0.7770709492266178, |
| "num_tokens": 710093.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 0.7449443437159061, |
| "epoch": 0.10582010582010581, |
| "grad_norm": 1.953125, |
| "learning_rate": 6.785714285714287e-06, |
| "loss": 0.760670518875122, |
| "mean_token_accuracy": 0.8012176774442196, |
| "num_tokens": 1429271.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.6568208456039428, |
| "epoch": 0.15873015873015872, |
| "grad_norm": 1.6328125, |
| "learning_rate": 1.0357142857142859e-05, |
| "loss": 0.6633346080780029, |
| "mean_token_accuracy": 0.8180790811777114, |
| "num_tokens": 2149735.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.6142221957445144, |
| "epoch": 0.21164021164021163, |
| "grad_norm": 1.4765625, |
| "learning_rate": 1.3928571428571429e-05, |
| "loss": 0.6138572692871094, |
| "mean_token_accuracy": 0.8293911524116993, |
| "num_tokens": 2868867.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.6016493506729603, |
| "epoch": 0.26455026455026454, |
| "grad_norm": 1.40625, |
| "learning_rate": 1.7500000000000002e-05, |
| "loss": 0.6028392791748047, |
| "mean_token_accuracy": 0.8320971228182316, |
| "num_tokens": 3582476.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.5878168147057294, |
| "epoch": 0.31746031746031744, |
| "grad_norm": 1.4453125, |
| "learning_rate": 1.999986795783372e-05, |
| "loss": 0.5865016937255859, |
| "mean_token_accuracy": 0.8350781731307506, |
| "num_tokens": 4287689.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.5819817841053009, |
| "epoch": 0.37037037037037035, |
| "grad_norm": 1.4765625, |
| "learning_rate": 1.9997520638548385e-05, |
| "loss": 0.5845561981201172, |
| "mean_token_accuracy": 0.8354875661432744, |
| "num_tokens": 5011191.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.575481166690588, |
| "epoch": 0.42328042328042326, |
| "grad_norm": 1.4921875, |
| "learning_rate": 1.9992239841688418e-05, |
| "loss": 0.573726749420166, |
| "mean_token_accuracy": 0.8390881218016147, |
| "num_tokens": 5714842.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.5651492070406675, |
| "epoch": 0.47619047619047616, |
| "grad_norm": 1.359375, |
| "learning_rate": 1.9984027116747914e-05, |
| "loss": 0.5669726371765137, |
| "mean_token_accuracy": 0.8404249772429466, |
| "num_tokens": 6424614.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.561945453286171, |
| "epoch": 0.5291005291005291, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.997288487350878e-05, |
| "loss": 0.5634259223937989, |
| "mean_token_accuracy": 0.8407754339277744, |
| "num_tokens": 7139778.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.5613414458930492, |
| "epoch": 0.582010582010582, |
| "grad_norm": 1.375, |
| "learning_rate": 1.9958816381333684e-05, |
| "loss": 0.560793685913086, |
| "mean_token_accuracy": 0.8415599942207337, |
| "num_tokens": 7852540.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.5559187311679125, |
| "epoch": 0.6349206349206349, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.994182576820673e-05, |
| "loss": 0.5539142608642578, |
| "mean_token_accuracy": 0.8423453107476234, |
| "num_tokens": 8568056.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.5548914466053247, |
| "epoch": 0.6878306878306878, |
| "grad_norm": 1.4375, |
| "learning_rate": 1.9921918019522238e-05, |
| "loss": 0.5571381568908691, |
| "mean_token_accuracy": 0.8423242166638374, |
| "num_tokens": 9276742.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.5476290043443441, |
| "epoch": 0.7407407407407407, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.9899098976621922e-05, |
| "loss": 0.5478506088256836, |
| "mean_token_accuracy": 0.8449021965265274, |
| "num_tokens": 9990163.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.5469458319246769, |
| "epoch": 0.7936507936507936, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.9873375335080915e-05, |
| "loss": 0.5464024543762207, |
| "mean_token_accuracy": 0.8450517050921917, |
| "num_tokens": 10710006.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.5466685622930527, |
| "epoch": 0.8465608465608465, |
| "grad_norm": 1.3359375, |
| "learning_rate": 1.9844754642743164e-05, |
| "loss": 0.5499144554138183, |
| "mean_token_accuracy": 0.8435575105249882, |
| "num_tokens": 11416822.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.5505413465201855, |
| "epoch": 0.8994708994708994, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.981324529750672e-05, |
| "loss": 0.5523758888244629, |
| "mean_token_accuracy": 0.8440131455659866, |
| "num_tokens": 12123118.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.5433717604726553, |
| "epoch": 0.9523809523809523, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.9778856544859647e-05, |
| "loss": 0.5418473243713379, |
| "mean_token_accuracy": 0.8456938855350018, |
| "num_tokens": 12831058.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.5340474512428045, |
| "epoch": 1.0052910052910053, |
| "grad_norm": 1.6015625, |
| "learning_rate": 1.9741598475167173e-05, |
| "loss": 0.5263125419616699, |
| "mean_token_accuracy": 0.849914176762104, |
| "num_tokens": 13547943.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.4664573803544044, |
| "epoch": 1.0582010582010581, |
| "grad_norm": 1.78125, |
| "learning_rate": 1.9701482020711004e-05, |
| "loss": 0.4695021629333496, |
| "mean_token_accuracy": 0.8637881852686405, |
| "num_tokens": 14264772.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.46051127538084985, |
| "epoch": 1.1111111111111112, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.9658518952481543e-05, |
| "loss": 0.4594545364379883, |
| "mean_token_accuracy": 0.8664374232292176, |
| "num_tokens": 14985565.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.46866549737751484, |
| "epoch": 1.164021164021164, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.9612721876724054e-05, |
| "loss": 0.46833486557006837, |
| "mean_token_accuracy": 0.8642570786178112, |
| "num_tokens": 15701796.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.4589692160487175, |
| "epoch": 1.216931216931217, |
| "grad_norm": 1.5, |
| "learning_rate": 1.9564104231239726e-05, |
| "loss": 0.45885400772094725, |
| "mean_token_accuracy": 0.8653957776725292, |
| "num_tokens": 16412522.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.4667964044958353, |
| "epoch": 1.2698412698412698, |
| "grad_norm": 1.3671875, |
| "learning_rate": 1.9512680281442745e-05, |
| "loss": 0.4659292697906494, |
| "mean_token_accuracy": 0.8646234408020973, |
| "num_tokens": 17126523.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.4641918484121561, |
| "epoch": 1.3227513227513228, |
| "grad_norm": 1.421875, |
| "learning_rate": 1.9458465116174534e-05, |
| "loss": 0.4644251823425293, |
| "mean_token_accuracy": 0.8649596966803074, |
| "num_tokens": 17834934.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.46363505683839323, |
| "epoch": 1.3756613756613756, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.940147464327637e-05, |
| "loss": 0.464162540435791, |
| "mean_token_accuracy": 0.8648634098470211, |
| "num_tokens": 18553413.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.460828198119998, |
| "epoch": 1.4285714285714286, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.9341725584921712e-05, |
| "loss": 0.4588433265686035, |
| "mean_token_accuracy": 0.8664663791656494, |
| "num_tokens": 19270191.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.4633682146668434, |
| "epoch": 1.4814814814814814, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.9279235472709554e-05, |
| "loss": 0.46491570472717286, |
| "mean_token_accuracy": 0.8642700470983982, |
| "num_tokens": 19976552.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.47140882574021814, |
| "epoch": 1.5343915343915344, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.92140226425203e-05, |
| "loss": 0.4693737030029297, |
| "mean_token_accuracy": 0.8627676844596863, |
| "num_tokens": 20676613.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.4692466512322426, |
| "epoch": 1.5873015873015874, |
| "grad_norm": 1.34375, |
| "learning_rate": 1.914610622913566e-05, |
| "loss": 0.46976585388183595, |
| "mean_token_accuracy": 0.8638280831277371, |
| "num_tokens": 21385978.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.46264285817742345, |
| "epoch": 1.6402116402116402, |
| "grad_norm": 1.21875, |
| "learning_rate": 1.9075506160624068e-05, |
| "loss": 0.462631893157959, |
| "mean_token_accuracy": 0.8651020877063275, |
| "num_tokens": 22094336.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.4625538159161806, |
| "epoch": 1.693121693121693, |
| "grad_norm": 1.34375, |
| "learning_rate": 1.900224315249343e-05, |
| "loss": 0.45883865356445314, |
| "mean_token_accuracy": 0.8657490260899067, |
| "num_tokens": 22798906.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.45813236869871615, |
| "epoch": 1.746031746031746, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.8926338701612738e-05, |
| "loss": 0.45924129486083987, |
| "mean_token_accuracy": 0.8655025579035283, |
| "num_tokens": 23511239.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.47067525461316106, |
| "epoch": 1.798941798941799, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.884781507990445e-05, |
| "loss": 0.4689933300018311, |
| "mean_token_accuracy": 0.8637023285031319, |
| "num_tokens": 24231185.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.4629125606268644, |
| "epoch": 1.8518518518518519, |
| "grad_norm": 1.3359375, |
| "learning_rate": 1.8766695327809463e-05, |
| "loss": 0.4614237308502197, |
| "mean_token_accuracy": 0.8654480330646038, |
| "num_tokens": 24944285.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.45222217179834845, |
| "epoch": 1.9047619047619047, |
| "grad_norm": 1.2734375, |
| "learning_rate": 1.8683003247526567e-05, |
| "loss": 0.4537208557128906, |
| "mean_token_accuracy": 0.8670621022582055, |
| "num_tokens": 25670441.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.4666783962398767, |
| "epoch": 1.9576719576719577, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.8596763396028384e-05, |
| "loss": 0.46645917892456057, |
| "mean_token_accuracy": 0.8638082660734654, |
| "num_tokens": 26379786.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.45693925954401493, |
| "epoch": 2.0105820105820107, |
| "grad_norm": 1.625, |
| "learning_rate": 1.8508001077855876e-05, |
| "loss": 0.4430360794067383, |
| "mean_token_accuracy": 0.8704046815633774, |
| "num_tokens": 27097166.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.36109724789857867, |
| "epoch": 2.0634920634920633, |
| "grad_norm": 1.4609375, |
| "learning_rate": 1.8416742337693436e-05, |
| "loss": 0.362947416305542, |
| "mean_token_accuracy": 0.8918761402368546, |
| "num_tokens": 27818991.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.36769300401210786, |
| "epoch": 2.1164021164021163, |
| "grad_norm": 1.53125, |
| "learning_rate": 1.8323013952726877e-05, |
| "loss": 0.363817024230957, |
| "mean_token_accuracy": 0.8912317991256714, |
| "num_tokens": 28524822.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.3584685638546944, |
| "epoch": 2.1693121693121693, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.8226843424786434e-05, |
| "loss": 0.3574815273284912, |
| "mean_token_accuracy": 0.8925611227750778, |
| "num_tokens": 29245634.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.36277062855660913, |
| "epoch": 2.2222222222222223, |
| "grad_norm": 1.4375, |
| "learning_rate": 1.8128258972277176e-05, |
| "loss": 0.3591357707977295, |
| "mean_token_accuracy": 0.8924785807728768, |
| "num_tokens": 29967135.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.37200981862843036, |
| "epoch": 2.2751322751322753, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.8027289521899143e-05, |
| "loss": 0.37147958278656007, |
| "mean_token_accuracy": 0.8890270739793777, |
| "num_tokens": 30680274.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.37133107893168926, |
| "epoch": 2.328042328042328, |
| "grad_norm": 1.4921875, |
| "learning_rate": 1.7923964700159658e-05, |
| "loss": 0.36978840827941895, |
| "mean_token_accuracy": 0.8895292244851589, |
| "num_tokens": 31386859.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.3785821709781885, |
| "epoch": 2.380952380952381, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.78183148246803e-05, |
| "loss": 0.37757096290588377, |
| "mean_token_accuracy": 0.8880668036639691, |
| "num_tokens": 32111007.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.3745276965200901, |
| "epoch": 2.433862433862434, |
| "grad_norm": 1.4453125, |
| "learning_rate": 1.77103708953011e-05, |
| "loss": 0.37161974906921386, |
| "mean_token_accuracy": 0.8889836438000203, |
| "num_tokens": 32819180.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.37825888618826864, |
| "epoch": 2.4867724867724865, |
| "grad_norm": 1.4140625, |
| "learning_rate": 1.7600164584984548e-05, |
| "loss": 0.37395377159118653, |
| "mean_token_accuracy": 0.8877906076610088, |
| "num_tokens": 33523113.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.3676387261599302, |
| "epoch": 2.5396825396825395, |
| "grad_norm": 1.375, |
| "learning_rate": 1.7487728230522085e-05, |
| "loss": 0.36518378257751466, |
| "mean_token_accuracy": 0.8908188074827195, |
| "num_tokens": 34240357.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.37763176299631596, |
| "epoch": 2.5925925925925926, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.737309482304585e-05, |
| "loss": 0.37689025402069093, |
| "mean_token_accuracy": 0.8876163229346276, |
| "num_tokens": 34958032.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.37179469019174577, |
| "epoch": 2.6455026455026456, |
| "grad_norm": 1.4921875, |
| "learning_rate": 1.7256297998348388e-05, |
| "loss": 0.3708032608032227, |
| "mean_token_accuracy": 0.889136116951704, |
| "num_tokens": 35658658.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.3762959886342287, |
| "epoch": 2.6984126984126986, |
| "grad_norm": 1.4609375, |
| "learning_rate": 1.7137372027013212e-05, |
| "loss": 0.37524843215942383, |
| "mean_token_accuracy": 0.8876962333917617, |
| "num_tokens": 36364754.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.37720581479370596, |
| "epoch": 2.751322751322751, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.7016351804359134e-05, |
| "loss": 0.3740926504135132, |
| "mean_token_accuracy": 0.8886674299836159, |
| "num_tokens": 37074678.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.38640342801809313, |
| "epoch": 2.804232804232804, |
| "grad_norm": 1.421875, |
| "learning_rate": 1.6893272840201243e-05, |
| "loss": 0.3842795372009277, |
| "mean_token_accuracy": 0.8857218272984028, |
| "num_tokens": 37787128.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.3735116682946682, |
| "epoch": 2.857142857142857, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.6768171248431603e-05, |
| "loss": 0.37361559867858884, |
| "mean_token_accuracy": 0.8879577770829201, |
| "num_tokens": 38504013.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.3714714240282774, |
| "epoch": 2.91005291005291, |
| "grad_norm": 1.421875, |
| "learning_rate": 1.66410837364227e-05, |
| "loss": 0.36954660415649415, |
| "mean_token_accuracy": 0.8894423045217991, |
| "num_tokens": 39212782.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.3719613593071699, |
| "epoch": 2.962962962962963, |
| "grad_norm": 1.40625, |
| "learning_rate": 1.651204759425674e-05, |
| "loss": 0.3682867527008057, |
| "mean_token_accuracy": 0.8899695724248886, |
| "num_tokens": 39931947.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.36400863490998747, |
| "epoch": 3.015873015873016, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.6381100683783984e-05, |
| "loss": 0.3504857778549194, |
| "mean_token_accuracy": 0.895585473626852, |
| "num_tokens": 40643225.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 0.28887790013104675, |
| "epoch": 3.068783068783069, |
| "grad_norm": 1.765625, |
| "learning_rate": 1.6248281427513293e-05, |
| "loss": 0.2926169395446777, |
| "mean_token_accuracy": 0.9128798201680184, |
| "num_tokens": 41359577.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.28943246975541115, |
| "epoch": 3.121693121693122, |
| "grad_norm": 1.65625, |
| "learning_rate": 1.6113628797338182e-05, |
| "loss": 0.28583292961120604, |
| "mean_token_accuracy": 0.9146523788571358, |
| "num_tokens": 42068599.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.28670772518962623, |
| "epoch": 3.1746031746031744, |
| "grad_norm": 1.6796875, |
| "learning_rate": 1.597718230310167e-05, |
| "loss": 0.28011598587036135, |
| "mean_token_accuracy": 0.9156787902116775, |
| "num_tokens": 42785175.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.284271296299994, |
| "epoch": 3.2275132275132274, |
| "grad_norm": 1.5703125, |
| "learning_rate": 1.5838981981003273e-05, |
| "loss": 0.2793328523635864, |
| "mean_token_accuracy": 0.9163288593292236, |
| "num_tokens": 43504295.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 0.2897234294563532, |
| "epoch": 3.2804232804232805, |
| "grad_norm": 1.5703125, |
| "learning_rate": 1.5699068381851576e-05, |
| "loss": 0.28597168922424315, |
| "mean_token_accuracy": 0.9139632828533649, |
| "num_tokens": 44210262.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 0.2917938305065036, |
| "epoch": 3.3333333333333335, |
| "grad_norm": 1.59375, |
| "learning_rate": 1.5557482559165777e-05, |
| "loss": 0.2878067970275879, |
| "mean_token_accuracy": 0.9133708864450455, |
| "num_tokens": 44921899.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.2872640796005726, |
| "epoch": 3.386243386243386, |
| "grad_norm": 1.5703125, |
| "learning_rate": 1.541426605712975e-05, |
| "loss": 0.28473606109619143, |
| "mean_token_accuracy": 0.9144375674426556, |
| "num_tokens": 45631735.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.29471687972545624, |
| "epoch": 3.439153439153439, |
| "grad_norm": 1.5703125, |
| "learning_rate": 1.526946089840214e-05, |
| "loss": 0.2887606620788574, |
| "mean_token_accuracy": 0.9132438965141774, |
| "num_tokens": 46339604.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.28842143099755047, |
| "epoch": 3.492063492063492, |
| "grad_norm": 1.578125, |
| "learning_rate": 1.5123109571786034e-05, |
| "loss": 0.28373041152954104, |
| "mean_token_accuracy": 0.9149374738335609, |
| "num_tokens": 47060193.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.2937064912170172, |
| "epoch": 3.544973544973545, |
| "grad_norm": 1.671875, |
| "learning_rate": 1.4975255019761896e-05, |
| "loss": 0.290342903137207, |
| "mean_token_accuracy": 0.9126849636435509, |
| "num_tokens": 47768090.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.2903562230989337, |
| "epoch": 3.597883597883598, |
| "grad_norm": 1.59375, |
| "learning_rate": 1.4825940625887343e-05, |
| "loss": 0.2838123798370361, |
| "mean_token_accuracy": 0.9145641170442105, |
| "num_tokens": 48486016.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.2933823145925999, |
| "epoch": 3.6507936507936507, |
| "grad_norm": 1.59375, |
| "learning_rate": 1.467521020206753e-05, |
| "loss": 0.29143826961517333, |
| "mean_token_accuracy": 0.9123459354043006, |
| "num_tokens": 49196144.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.2923125779256225, |
| "epoch": 3.7037037037037037, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.4523107975699833e-05, |
| "loss": 0.28827059268951416, |
| "mean_token_accuracy": 0.9130347467958927, |
| "num_tokens": 49912021.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.297353445366025, |
| "epoch": 3.7566137566137567, |
| "grad_norm": 1.6328125, |
| "learning_rate": 1.4369678576696633e-05, |
| "loss": 0.2925769329071045, |
| "mean_token_accuracy": 0.911816319823265, |
| "num_tokens": 50634262.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.29219231754541397, |
| "epoch": 3.8095238095238093, |
| "grad_norm": 1.59375, |
| "learning_rate": 1.4214967024389978e-05, |
| "loss": 0.2877782344818115, |
| "mean_token_accuracy": 0.9128481931984425, |
| "num_tokens": 51343452.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.2929087163880467, |
| "epoch": 3.8624338624338623, |
| "grad_norm": 1.546875, |
| "learning_rate": 1.4059018714321986e-05, |
| "loss": 0.28854234218597413, |
| "mean_token_accuracy": 0.9127740956842899, |
| "num_tokens": 52046363.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.29222483895719054, |
| "epoch": 3.9153439153439153, |
| "grad_norm": 1.6328125, |
| "learning_rate": 1.390187940492486e-05, |
| "loss": 0.2873117446899414, |
| "mean_token_accuracy": 0.9136057265102864, |
| "num_tokens": 52755135.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.28353314101696014, |
| "epoch": 3.9682539682539684, |
| "grad_norm": 1.578125, |
| "learning_rate": 1.3743595204094442e-05, |
| "loss": 0.2814703702926636, |
| "mean_token_accuracy": 0.9146252907812595, |
| "num_tokens": 53473566.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.2658627746626735, |
| "epoch": 4.021164021164021, |
| "grad_norm": 1.8046875, |
| "learning_rate": 1.3584212555661187e-05, |
| "loss": 0.2556802272796631, |
| "mean_token_accuracy": 0.9232950232923031, |
| "num_tokens": 54190492.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.2240922437980771, |
| "epoch": 4.074074074074074, |
| "grad_norm": 1.7109375, |
| "learning_rate": 1.3423778225762609e-05, |
| "loss": 0.21764779090881348, |
| "mean_token_accuracy": 0.9355925224721432, |
| "num_tokens": 54897936.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.22832553032785655, |
| "epoch": 4.1269841269841265, |
| "grad_norm": 1.9140625, |
| "learning_rate": 1.3262339289121129e-05, |
| "loss": 0.2187000036239624, |
| "mean_token_accuracy": 0.9350209765136241, |
| "num_tokens": 55609388.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.23423405233770608, |
| "epoch": 4.1798941798941796, |
| "grad_norm": 1.7421875, |
| "learning_rate": 1.3099943115231386e-05, |
| "loss": 0.2247084617614746, |
| "mean_token_accuracy": 0.932844078540802, |
| "num_tokens": 56329299.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.22562892884016036, |
| "epoch": 4.232804232804233, |
| "grad_norm": 1.6796875, |
| "learning_rate": 1.2936637354461064e-05, |
| "loss": 0.21526126861572265, |
| "mean_token_accuracy": 0.9357499599456787, |
| "num_tokens": 57042704.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.22767273224890233, |
| "epoch": 4.285714285714286, |
| "grad_norm": 1.6875, |
| "learning_rate": 1.2772469924069285e-05, |
| "loss": 0.22047383785247804, |
| "mean_token_accuracy": 0.9339667908847332, |
| "num_tokens": 57755769.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.2263598322868347, |
| "epoch": 4.338624338624339, |
| "grad_norm": 1.7109375, |
| "learning_rate": 1.2607488994146703e-05, |
| "loss": 0.2171053409576416, |
| "mean_token_accuracy": 0.9352345928549767, |
| "num_tokens": 58467967.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.23044093661010265, |
| "epoch": 4.391534391534392, |
| "grad_norm": 1.7734375, |
| "learning_rate": 1.2441742973481432e-05, |
| "loss": 0.22197408676147462, |
| "mean_token_accuracy": 0.933641043305397, |
| "num_tokens": 59188579.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.22965663019567728, |
| "epoch": 4.444444444444445, |
| "grad_norm": 1.8046875, |
| "learning_rate": 1.2275280495354889e-05, |
| "loss": 0.21916933059692384, |
| "mean_token_accuracy": 0.9343777082860469, |
| "num_tokens": 59899589.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.22166559360921384, |
| "epoch": 4.497354497354498, |
| "grad_norm": 1.6875, |
| "learning_rate": 1.21081504032718e-05, |
| "loss": 0.213417911529541, |
| "mean_token_accuracy": 0.9358938954770565, |
| "num_tokens": 60620913.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.23459077198058367, |
| "epoch": 4.550264550264551, |
| "grad_norm": 1.7109375, |
| "learning_rate": 1.1940401736628524e-05, |
| "loss": 0.22437291145324706, |
| "mean_token_accuracy": 0.9326319947838784, |
| "num_tokens": 61330675.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.22559807691723108, |
| "epoch": 4.603174603174603, |
| "grad_norm": 1.921875, |
| "learning_rate": 1.1772083716323878e-05, |
| "loss": 0.21871533393859863, |
| "mean_token_accuracy": 0.9344435475766659, |
| "num_tokens": 62041645.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.23360109385102987, |
| "epoch": 4.656084656084656, |
| "grad_norm": 1.6953125, |
| "learning_rate": 1.1603245730316724e-05, |
| "loss": 0.22537636756896973, |
| "mean_token_accuracy": 0.9321599870920181, |
| "num_tokens": 62750446.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.22814673688262702, |
| "epoch": 4.708994708994709, |
| "grad_norm": 1.75, |
| "learning_rate": 1.1433937319134512e-05, |
| "loss": 0.2192002534866333, |
| "mean_token_accuracy": 0.9344004705548287, |
| "num_tokens": 63460008.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.22613348066806793, |
| "epoch": 4.761904761904762, |
| "grad_norm": 1.6875, |
| "learning_rate": 1.1264208161337091e-05, |
| "loss": 0.21817412376403808, |
| "mean_token_accuracy": 0.9343988470733166, |
| "num_tokens": 64164803.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.2269863199442625, |
| "epoch": 4.814814814814815, |
| "grad_norm": 1.6484375, |
| "learning_rate": 1.1094108058940005e-05, |
| "loss": 0.21749932765960694, |
| "mean_token_accuracy": 0.9347520999610424, |
| "num_tokens": 64878163.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.2237796787172556, |
| "epoch": 4.867724867724868, |
| "grad_norm": 1.7265625, |
| "learning_rate": 1.0923686922801537e-05, |
| "loss": 0.21339073181152343, |
| "mean_token_accuracy": 0.9363370195031167, |
| "num_tokens": 65599878.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.22895934320986272, |
| "epoch": 4.920634920634921, |
| "grad_norm": 1.6796875, |
| "learning_rate": 1.0752994757977862e-05, |
| "loss": 0.22032537460327148, |
| "mean_token_accuracy": 0.9343368001282215, |
| "num_tokens": 66312478.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.23594301883131266, |
| "epoch": 4.973544973544973, |
| "grad_norm": 1.7421875, |
| "learning_rate": 1.058208164905052e-05, |
| "loss": 0.23022885322570802, |
| "mean_token_accuracy": 0.9311877243220806, |
| "num_tokens": 67026652.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.21192003265023232, |
| "epoch": 5.026455026455026, |
| "grad_norm": 2.125, |
| "learning_rate": 1.041099774543059e-05, |
| "loss": 0.19697930812835693, |
| "mean_token_accuracy": 0.9413417495787144, |
| "num_tokens": 67743837.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.1835236642509699, |
| "epoch": 5.079365079365079, |
| "grad_norm": 1.6953125, |
| "learning_rate": 1.023979324664382e-05, |
| "loss": 0.16840988397598267, |
| "mean_token_accuracy": 0.9505717150866986, |
| "num_tokens": 68459077.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.18639093749225139, |
| "epoch": 5.132275132275132, |
| "grad_norm": 1.65625, |
| "learning_rate": 1.0068518387601038e-05, |
| "loss": 0.17528188228607178, |
| "mean_token_accuracy": 0.9483132407069206, |
| "num_tokens": 69175886.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.17816722057759762, |
| "epoch": 5.185185185185185, |
| "grad_norm": 1.59375, |
| "learning_rate": 9.897223423858226e-06, |
| "loss": 0.16518499851226806, |
| "mean_token_accuracy": 0.951267096400261, |
| "num_tokens": 69888377.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.18737060986459256, |
| "epoch": 5.238095238095238, |
| "grad_norm": 1.6796875, |
| "learning_rate": 9.725958616870495e-06, |
| "loss": 0.1752444624900818, |
| "mean_token_accuracy": 0.9481210887432099, |
| "num_tokens": 70604584.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.19039766546338796, |
| "epoch": 5.291005291005291, |
| "grad_norm": 1.734375, |
| "learning_rate": 9.554774219244344e-06, |
| "loss": 0.17676937580108643, |
| "mean_token_accuracy": 0.9481188610196114, |
| "num_tokens": 71321169.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.19046760462224482, |
| "epoch": 5.343915343915344, |
| "grad_norm": 1.59375, |
| "learning_rate": 9.38372045999252e-06, |
| "loss": 0.1771212935447693, |
| "mean_token_accuracy": 0.9478032320737839, |
| "num_tokens": 72041175.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.18601095546036958, |
| "epoch": 5.396825396825397, |
| "grad_norm": 1.8125, |
| "learning_rate": 9.212847529795784e-06, |
| "loss": 0.17073088884353638, |
| "mean_token_accuracy": 0.9491294145584106, |
| "num_tokens": 72749079.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.18668258730322124, |
| "epoch": 5.449735449735449, |
| "grad_norm": 1.6171875, |
| "learning_rate": 9.042205566275952e-06, |
| "loss": 0.17535300254821778, |
| "mean_token_accuracy": 0.9484463959932328, |
| "num_tokens": 73462893.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.19089560490101576, |
| "epoch": 5.502645502645502, |
| "grad_norm": 1.75, |
| "learning_rate": 8.871844639284479e-06, |
| "loss": 0.17920498847961425, |
| "mean_token_accuracy": 0.9474038317799568, |
| "num_tokens": 74167133.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.19047440169379115, |
| "epoch": 5.555555555555555, |
| "grad_norm": 1.703125, |
| "learning_rate": 8.701814736210933e-06, |
| "loss": 0.17654495239257811, |
| "mean_token_accuracy": 0.9476719535887241, |
| "num_tokens": 74870399.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.19135245606303214, |
| "epoch": 5.608465608465608, |
| "grad_norm": 1.703125, |
| "learning_rate": 8.532165747315686e-06, |
| "loss": 0.17768774032592774, |
| "mean_token_accuracy": 0.9471674285829067, |
| "num_tokens": 75590782.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.1910527590662241, |
| "epoch": 5.661375661375661, |
| "grad_norm": 1.6640625, |
| "learning_rate": 8.362947451091072e-06, |
| "loss": 0.17710025310516359, |
| "mean_token_accuracy": 0.947571088373661, |
| "num_tokens": 76305788.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.18642712617293, |
| "epoch": 5.714285714285714, |
| "grad_norm": 1.5546875, |
| "learning_rate": 8.194209499655364e-06, |
| "loss": 0.1747068405151367, |
| "mean_token_accuracy": 0.9487118482589721, |
| "num_tokens": 77023579.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 0.18968199249356985, |
| "epoch": 5.767195767195767, |
| "grad_norm": 1.7890625, |
| "learning_rate": 8.026001404183831e-06, |
| "loss": 0.1767462372779846, |
| "mean_token_accuracy": 0.9479191042482853, |
| "num_tokens": 77720821.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.18559476286172866, |
| "epoch": 5.8201058201058204, |
| "grad_norm": 1.5703125, |
| "learning_rate": 7.85837252038112e-06, |
| "loss": 0.1724246025085449, |
| "mean_token_accuracy": 0.9496469184756279, |
| "num_tokens": 78431671.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.18607105426490306, |
| "epoch": 5.8730158730158735, |
| "grad_norm": 1.7109375, |
| "learning_rate": 7.69137203399931e-06, |
| "loss": 0.1711591362953186, |
| "mean_token_accuracy": 0.9495569460093976, |
| "num_tokens": 79141393.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.18680871110409497, |
| "epoch": 5.925925925925926, |
| "grad_norm": 1.6015625, |
| "learning_rate": 7.525048946405791e-06, |
| "loss": 0.173135244846344, |
| "mean_token_accuracy": 0.9489903882145881, |
| "num_tokens": 79858356.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 0.19268490094691515, |
| "epoch": 5.978835978835979, |
| "grad_norm": 1.6953125, |
| "learning_rate": 7.359452060205245e-06, |
| "loss": 0.18039216995239257, |
| "mean_token_accuracy": 0.9467289201915264, |
| "num_tokens": 80566532.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 0.17256247820332646, |
| "epoch": 6.031746031746032, |
| "grad_norm": 1.7421875, |
| "learning_rate": 7.194629964919989e-06, |
| "loss": 0.15564732551574706, |
| "mean_token_accuracy": 0.954721450060606, |
| "num_tokens": 81283415.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.16467317258939146, |
| "epoch": 6.084656084656085, |
| "grad_norm": 1.5859375, |
| "learning_rate": 7.030631022732781e-06, |
| "loss": 0.14938194751739503, |
| "mean_token_accuracy": 0.9563332803547382, |
| "num_tokens": 81995607.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.1660136022605002, |
| "epoch": 6.137566137566138, |
| "grad_norm": 1.65625, |
| "learning_rate": 6.867503354296392e-06, |
| "loss": 0.15061533451080322, |
| "mean_token_accuracy": 0.9557233087718486, |
| "num_tokens": 82712148.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 0.16843323688954115, |
| "epoch": 6.190476190476191, |
| "grad_norm": 1.59375, |
| "learning_rate": 6.705294824614004e-06, |
| "loss": 0.15027358531951904, |
| "mean_token_accuracy": 0.9563688673079014, |
| "num_tokens": 83425506.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.1694221578538418, |
| "epoch": 6.243386243386244, |
| "grad_norm": 1.7578125, |
| "learning_rate": 6.54405302899463e-06, |
| "loss": 0.15375804901123047, |
| "mean_token_accuracy": 0.9546328760683537, |
| "num_tokens": 84137018.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.16813772525638343, |
| "epoch": 6.296296296296296, |
| "grad_norm": 1.71875, |
| "learning_rate": 6.383825279087699e-06, |
| "loss": 0.15173449516296386, |
| "mean_token_accuracy": 0.9560045033693314, |
| "num_tokens": 84849598.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.16589505458250642, |
| "epoch": 6.349206349206349, |
| "grad_norm": 1.5390625, |
| "learning_rate": 6.2246585890008245e-06, |
| "loss": 0.15113791227340698, |
| "mean_token_accuracy": 0.9562842510640621, |
| "num_tokens": 85561962.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.16350512765347958, |
| "epoch": 6.402116402116402, |
| "grad_norm": 1.8203125, |
| "learning_rate": 6.06659966150491e-06, |
| "loss": 0.1486595630645752, |
| "mean_token_accuracy": 0.9566722363233566, |
| "num_tokens": 86286338.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.17174449795857072, |
| "epoch": 6.455026455026455, |
| "grad_norm": 1.7265625, |
| "learning_rate": 5.909694874330601e-06, |
| "loss": 0.15474398136138917, |
| "mean_token_accuracy": 0.954923465847969, |
| "num_tokens": 87001911.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.16522456351667641, |
| "epoch": 6.507936507936508, |
| "grad_norm": 1.6953125, |
| "learning_rate": 5.753990266560105e-06, |
| "loss": 0.14906713962554932, |
| "mean_token_accuracy": 0.9566663041710853, |
| "num_tokens": 87714571.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 0.16697804098948837, |
| "epoch": 6.560846560846561, |
| "grad_norm": 1.5703125, |
| "learning_rate": 5.599531525118373e-06, |
| "loss": 0.15133001804351806, |
| "mean_token_accuracy": 0.9559323705732823, |
| "num_tokens": 88417674.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.16594976373016834, |
| "epoch": 6.613756613756614, |
| "grad_norm": 1.6796875, |
| "learning_rate": 5.446363971367638e-06, |
| "loss": 0.1497833251953125, |
| "mean_token_accuracy": 0.9561318881809712, |
| "num_tokens": 89123522.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.16131719732657074, |
| "epoch": 6.666666666666667, |
| "grad_norm": 1.5546875, |
| "learning_rate": 5.29453254780917e-06, |
| "loss": 0.14488465785980226, |
| "mean_token_accuracy": 0.9581532798707485, |
| "num_tokens": 89838831.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 0.16528327343985438, |
| "epoch": 6.71957671957672, |
| "grad_norm": 1.5390625, |
| "learning_rate": 5.1440818048962575e-06, |
| "loss": 0.1491045355796814, |
| "mean_token_accuracy": 0.9563969649374485, |
| "num_tokens": 90546066.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 0.16403559185564517, |
| "epoch": 6.772486772486772, |
| "grad_norm": 1.7109375, |
| "learning_rate": 4.995055887962167e-06, |
| "loss": 0.14752378463745117, |
| "mean_token_accuracy": 0.957070492208004, |
| "num_tokens": 91257301.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 0.16546227606013417, |
| "epoch": 6.825396825396825, |
| "grad_norm": 1.6953125, |
| "learning_rate": 4.8474985242669965e-06, |
| "loss": 0.15117653608322143, |
| "mean_token_accuracy": 0.9560656279325486, |
| "num_tokens": 91961075.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 0.17248945254832507, |
| "epoch": 6.878306878306878, |
| "grad_norm": 1.796875, |
| "learning_rate": 4.701453010167217e-06, |
| "loss": 0.15554436445236205, |
| "mean_token_accuracy": 0.9547501035034657, |
| "num_tokens": 92677769.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.15598487379029394, |
| "epoch": 6.931216931216931, |
| "grad_norm": 1.515625, |
| "learning_rate": 4.556962198411631e-06, |
| "loss": 0.13962192535400392, |
| "mean_token_accuracy": 0.9600816458463669, |
| "num_tokens": 93390418.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 0.1598960392177105, |
| "epoch": 6.984126984126984, |
| "grad_norm": 1.6640625, |
| "learning_rate": 4.414068485567473e-06, |
| "loss": 0.14374563694000245, |
| "mean_token_accuracy": 0.958079669624567, |
| "num_tokens": 94107638.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 0.155168966576457, |
| "epoch": 7.037037037037037, |
| "grad_norm": 1.53125, |
| "learning_rate": 4.272813799580417e-06, |
| "loss": 0.13499114513397217, |
| "mean_token_accuracy": 0.9609342411160469, |
| "num_tokens": 94829458.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 0.1519143793731928, |
| "epoch": 7.08994708994709, |
| "grad_norm": 1.53125, |
| "learning_rate": 4.1332395874720474e-06, |
| "loss": 0.135742449760437, |
| "mean_token_accuracy": 0.9605026729404926, |
| "num_tokens": 95540140.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.14949759403243662, |
| "epoch": 7.142857142857143, |
| "grad_norm": 1.4296875, |
| "learning_rate": 3.9953868031784415e-06, |
| "loss": 0.13058431148529054, |
| "mean_token_accuracy": 0.962359694391489, |
| "num_tokens": 96254434.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.15626662448048592, |
| "epoch": 7.195767195767195, |
| "grad_norm": 1.6015625, |
| "learning_rate": 3.8592958955334636e-06, |
| "loss": 0.13887736797332764, |
| "mean_token_accuracy": 0.9602727100253106, |
| "num_tokens": 96966014.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.15087712686508895, |
| "epoch": 7.248677248677248, |
| "grad_norm": 1.6171875, |
| "learning_rate": 3.725006796400231e-06, |
| "loss": 0.13402986526489258, |
| "mean_token_accuracy": 0.9616384491324425, |
| "num_tokens": 97677188.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.15504114823415877, |
| "epoch": 7.301587301587301, |
| "grad_norm": 1.6875, |
| "learning_rate": 3.5925589089542954e-06, |
| "loss": 0.1371316432952881, |
| "mean_token_accuracy": 0.9602581821382046, |
| "num_tokens": 98391677.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 0.15244807954877615, |
| "epoch": 7.354497354497354, |
| "grad_norm": 1.5625, |
| "learning_rate": 3.461991096121955e-06, |
| "loss": 0.1345362424850464, |
| "mean_token_accuracy": 0.9612249001860619, |
| "num_tokens": 99105789.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 0.1552990710362792, |
| "epoch": 7.407407407407407, |
| "grad_norm": 1.6640625, |
| "learning_rate": 3.3333416691770402e-06, |
| "loss": 0.13869800567626953, |
| "mean_token_accuracy": 0.9595564372837544, |
| "num_tokens": 99819774.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.15235155755653979, |
| "epoch": 7.4603174603174605, |
| "grad_norm": 1.5078125, |
| "learning_rate": 3.206648376499637e-06, |
| "loss": 0.13434691429138185, |
| "mean_token_accuracy": 0.961196530610323, |
| "num_tokens": 100540345.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.15812121173366905, |
| "epoch": 7.5132275132275135, |
| "grad_norm": 1.5, |
| "learning_rate": 3.0819483924999183e-06, |
| "loss": 0.14151489734649658, |
| "mean_token_accuracy": 0.9588122457265854, |
| "num_tokens": 101261007.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 0.1553444412536919, |
| "epoch": 7.5661375661375665, |
| "grad_norm": 1.46875, |
| "learning_rate": 2.959278306710409e-06, |
| "loss": 0.13649454116821289, |
| "mean_token_accuracy": 0.9604852601885796, |
| "num_tokens": 101972692.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 0.15734316296875478, |
| "epoch": 7.619047619047619, |
| "grad_norm": 1.6171875, |
| "learning_rate": 2.838674113049884e-06, |
| "loss": 0.13888288736343385, |
| "mean_token_accuracy": 0.95976832062006, |
| "num_tokens": 102684836.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 0.16042319955304266, |
| "epoch": 7.671957671957672, |
| "grad_norm": 1.5703125, |
| "learning_rate": 2.7201711992619873e-06, |
| "loss": 0.1441219449043274, |
| "mean_token_accuracy": 0.9588080503046512, |
| "num_tokens": 103404477.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.1534231398254633, |
| "epoch": 7.724867724867725, |
| "grad_norm": 1.5859375, |
| "learning_rate": 2.6038043365317643e-06, |
| "loss": 0.13559118509292603, |
| "mean_token_accuracy": 0.9607343763113022, |
| "num_tokens": 104111178.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 0.14997789431363345, |
| "epoch": 7.777777777777778, |
| "grad_norm": 1.625, |
| "learning_rate": 2.489607669283088e-06, |
| "loss": 0.13227237462997438, |
| "mean_token_accuracy": 0.9615601144731045, |
| "num_tokens": 104820924.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.1600227179005742, |
| "epoch": 7.830687830687831, |
| "grad_norm": 1.6171875, |
| "learning_rate": 2.3776147051599518e-06, |
| "loss": 0.14188143014907836, |
| "mean_token_accuracy": 0.9592055462300777, |
| "num_tokens": 105525661.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.15876007685437799, |
| "epoch": 7.883597883597884, |
| "grad_norm": 1.5546875, |
| "learning_rate": 2.2678583051946733e-06, |
| "loss": 0.1410558819770813, |
| "mean_token_accuracy": 0.9590302243828773, |
| "num_tokens": 106244487.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.1540695504285395, |
| "epoch": 7.936507936507937, |
| "grad_norm": 1.6484375, |
| "learning_rate": 2.1603706741657803e-06, |
| "loss": 0.13452168703079223, |
| "mean_token_accuracy": 0.9611428380012512, |
| "num_tokens": 106950025.0, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.1625247542746365, |
| "epoch": 7.98941798941799, |
| "grad_norm": 1.5078125, |
| "learning_rate": 2.055183351148463e-06, |
| "loss": 0.1474345326423645, |
| "mean_token_accuracy": 0.9569125808775425, |
| "num_tokens": 107659233.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 0.15517827067524195, |
| "epoch": 8.042328042328043, |
| "grad_norm": 1.703125, |
| "learning_rate": 1.9523272002603745e-06, |
| "loss": 0.1366629719734192, |
| "mean_token_accuracy": 0.9605339720845223, |
| "num_tokens": 108374173.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 0.14798865001648664, |
| "epoch": 8.095238095238095, |
| "grad_norm": 1.484375, |
| "learning_rate": 1.8518324016054601e-06, |
| "loss": 0.12860276699066162, |
| "mean_token_accuracy": 0.9632845722138882, |
| "num_tokens": 109085583.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.15172817055135965, |
| "epoch": 8.148148148148149, |
| "grad_norm": 1.609375, |
| "learning_rate": 1.7537284424184852e-06, |
| "loss": 0.1327933669090271, |
| "mean_token_accuracy": 0.9615824081003665, |
| "num_tokens": 109801232.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 0.14800362922251226, |
| "epoch": 8.201058201058201, |
| "grad_norm": 1.625, |
| "learning_rate": 1.658044108412905e-06, |
| "loss": 0.13004300594329835, |
| "mean_token_accuracy": 0.9628023765981197, |
| "num_tokens": 110516150.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.1480136933736503, |
| "epoch": 8.253968253968253, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.564807475334509e-06, |
| "loss": 0.12993026971817018, |
| "mean_token_accuracy": 0.9628436036407948, |
| "num_tokens": 111232078.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.14930932661518453, |
| "epoch": 8.306878306878307, |
| "grad_norm": 1.6484375, |
| "learning_rate": 1.4740459007234498e-06, |
| "loss": 0.1299981117248535, |
| "mean_token_accuracy": 0.9624004021286965, |
| "num_tokens": 111950711.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 0.15494557972997428, |
| "epoch": 8.359788359788359, |
| "grad_norm": 1.6953125, |
| "learning_rate": 1.3857860158869696e-06, |
| "loss": 0.13706895112991332, |
| "mean_token_accuracy": 0.9603550881147385, |
| "num_tokens": 112658277.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 0.15331083843484522, |
| "epoch": 8.412698412698413, |
| "grad_norm": 1.46875, |
| "learning_rate": 1.3000537180852213e-06, |
| "loss": 0.1342510461807251, |
| "mean_token_accuracy": 0.9613098680973053, |
| "num_tokens": 113378303.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 0.15126598635688424, |
| "epoch": 8.465608465608465, |
| "grad_norm": 1.4375, |
| "learning_rate": 1.2168741629324944e-06, |
| "loss": 0.13197760581970214, |
| "mean_token_accuracy": 0.961933346837759, |
| "num_tokens": 114087597.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.14689233647659422, |
| "epoch": 8.518518518518519, |
| "grad_norm": 1.53125, |
| "learning_rate": 1.1362717570160387e-06, |
| "loss": 0.1281040906906128, |
| "mean_token_accuracy": 0.9627489626407624, |
| "num_tokens": 114803563.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 0.1479676628485322, |
| "epoch": 8.571428571428571, |
| "grad_norm": 1.578125, |
| "learning_rate": 1.0582701507346582e-06, |
| "loss": 0.12949721813201903, |
| "mean_token_accuracy": 0.9626893244683743, |
| "num_tokens": 115519221.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 0.1561860882677138, |
| "epoch": 8.624338624338625, |
| "grad_norm": 1.5390625, |
| "learning_rate": 9.828922313592227e-07, |
| "loss": 0.13759052753448486, |
| "mean_token_accuracy": 0.9600970260798931, |
| "num_tokens": 116238487.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.14839244754984976, |
| "epoch": 8.677248677248677, |
| "grad_norm": 1.4921875, |
| "learning_rate": 9.10160116317047e-07, |
| "loss": 0.12978951930999755, |
| "mean_token_accuracy": 0.9626202285289764, |
| "num_tokens": 116956616.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 0.1546482781879604, |
| "epoch": 8.73015873015873, |
| "grad_norm": 1.625, |
| "learning_rate": 8.400951467022067e-07, |
| "loss": 0.13703587055206298, |
| "mean_token_accuracy": 0.9604786932468414, |
| "num_tokens": 117662184.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.14911084109917283, |
| "epoch": 8.783068783068783, |
| "grad_norm": 1.6953125, |
| "learning_rate": 7.727178810136094e-07, |
| "loss": 0.1300983190536499, |
| "mean_token_accuracy": 0.9623136028647423, |
| "num_tokens": 118365440.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 0.15351321259513498, |
| "epoch": 8.835978835978835, |
| "grad_norm": 1.4765625, |
| "learning_rate": 7.080480891227126e-07, |
| "loss": 0.1333402395248413, |
| "mean_token_accuracy": 0.9615973889827728, |
| "num_tokens": 119078464.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 0.1598691301420331, |
| "epoch": 8.88888888888889, |
| "grad_norm": 1.5859375, |
| "learning_rate": 6.461047464726433e-07, |
| "loss": 0.14236483573913575, |
| "mean_token_accuracy": 0.9591965340077877, |
| "num_tokens": 119792226.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 0.14664519103243948, |
| "epoch": 8.941798941798941, |
| "grad_norm": 1.5234375, |
| "learning_rate": 5.869060285104089e-07, |
| "loss": 0.12819191217422485, |
| "mean_token_accuracy": 0.9630496680736542, |
| "num_tokens": 120500468.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.1479152231477201, |
| "epoch": 8.994708994708995, |
| "grad_norm": 1.3984375, |
| "learning_rate": 5.304693053538434e-07, |
| "loss": 0.12968764305114747, |
| "mean_token_accuracy": 0.9625036440789699, |
| "num_tokens": 121205368.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.14528713496401907, |
| "epoch": 9.047619047619047, |
| "grad_norm": 1.5390625, |
| "learning_rate": 4.768111366948736e-07, |
| "loss": 0.12741949558258056, |
| "mean_token_accuracy": 0.9634793661534786, |
| "num_tokens": 121917493.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 0.1503755608573556, |
| "epoch": 9.100529100529101, |
| "grad_norm": 1.6875, |
| "learning_rate": 4.2594726694055397e-07, |
| "loss": 0.13262239694595337, |
| "mean_token_accuracy": 0.9618236005306244, |
| "num_tokens": 122643710.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.15070006167516112, |
| "epoch": 9.153439153439153, |
| "grad_norm": 1.5546875, |
| "learning_rate": 3.778926205933342e-07, |
| "loss": 0.1313535213470459, |
| "mean_token_accuracy": 0.9620834827423096, |
| "num_tokens": 123357455.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.15359645066782832, |
| "epoch": 9.206349206349206, |
| "grad_norm": 1.6015625, |
| "learning_rate": 3.3266129787191015e-07, |
| "loss": 0.13509082794189453, |
| "mean_token_accuracy": 0.9614561274647713, |
| "num_tokens": 124071350.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.15185305522754788, |
| "epoch": 9.25925925925926, |
| "grad_norm": 1.5390625, |
| "learning_rate": 2.9026657057392715e-07, |
| "loss": 0.13305494785308838, |
| "mean_token_accuracy": 0.9617955818772316, |
| "num_tokens": 124788814.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.14987680977210402, |
| "epoch": 9.312169312169312, |
| "grad_norm": 1.6953125, |
| "learning_rate": 2.507208781817638e-07, |
| "loss": 0.1299354076385498, |
| "mean_token_accuracy": 0.9621578969061375, |
| "num_tokens": 125504162.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.1526137980632484, |
| "epoch": 9.365079365079366, |
| "grad_norm": 1.484375, |
| "learning_rate": 2.1403582421253398e-07, |
| "loss": 0.1334925889968872, |
| "mean_token_accuracy": 0.9612173721194267, |
| "num_tokens": 126213099.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.1515716938301921, |
| "epoch": 9.417989417989418, |
| "grad_norm": 1.671875, |
| "learning_rate": 1.8022217281336908e-07, |
| "loss": 0.13385677337646484, |
| "mean_token_accuracy": 0.9614432126283645, |
| "num_tokens": 126934123.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 0.15292979842051863, |
| "epoch": 9.470899470899472, |
| "grad_norm": 1.5390625, |
| "learning_rate": 1.4928984560301006e-07, |
| "loss": 0.13438937664031983, |
| "mean_token_accuracy": 0.9609661236405372, |
| "num_tokens": 127647228.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.1540503304451704, |
| "epoch": 9.523809523809524, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.212479187605897e-07, |
| "loss": 0.13354144096374512, |
| "mean_token_accuracy": 0.9617787852883339, |
| "num_tokens": 128350815.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.1485958187840879, |
| "epoch": 9.576719576719576, |
| "grad_norm": 1.4765625, |
| "learning_rate": 9.610462036249823e-08, |
| "loss": 0.12973830699920655, |
| "mean_token_accuracy": 0.9625511176884174, |
| "num_tokens": 129070776.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.15611315574496984, |
| "epoch": 9.62962962962963, |
| "grad_norm": 1.5546875, |
| "learning_rate": 7.386732796810014e-08, |
| "loss": 0.13796322345733641, |
| "mean_token_accuracy": 0.959777320176363, |
| "num_tokens": 129784905.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 0.14790695123374462, |
| "epoch": 9.682539682539682, |
| "grad_norm": 1.4765625, |
| "learning_rate": 5.454256645500477e-08, |
| "loss": 0.1278387188911438, |
| "mean_token_accuracy": 0.9629337847232818, |
| "num_tokens": 130492958.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.14842531718313695, |
| "epoch": 9.735449735449736, |
| "grad_norm": 1.5859375, |
| "learning_rate": 3.8136006104533276e-08, |
| "loss": 0.12838966846466066, |
| "mean_token_accuracy": 0.9630717404186726, |
| "num_tokens": 131214260.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 0.1488181633874774, |
| "epoch": 9.788359788359788, |
| "grad_norm": 1.6953125, |
| "learning_rate": 2.4652460937941714e-08, |
| "loss": 0.13013782501220703, |
| "mean_token_accuracy": 0.9623884275555611, |
| "num_tokens": 131927722.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.15232085334137083, |
| "epoch": 9.841269841269842, |
| "grad_norm": 1.5390625, |
| "learning_rate": 1.4095887303888778e-08, |
| "loss": 0.13350577354431153, |
| "mean_token_accuracy": 0.9614293269813061, |
| "num_tokens": 132622563.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 0.14972581919282674, |
| "epoch": 9.894179894179894, |
| "grad_norm": 1.46875, |
| "learning_rate": 6.469382717563255e-09, |
| "loss": 0.13146800994873048, |
| "mean_token_accuracy": 0.9620610632002353, |
| "num_tokens": 133332199.0, |
| "step": 1870 |
| }, |
| { |
| "entropy": 0.1426906729117036, |
| "epoch": 9.947089947089948, |
| "grad_norm": 1.46875, |
| "learning_rate": 1.775184951806619e-09, |
| "loss": 0.12295315265655518, |
| "mean_token_accuracy": 0.9648364841938019, |
| "num_tokens": 134041278.0, |
| "step": 1880 |
| }, |
| { |
| "entropy": 0.15274320617318154, |
| "epoch": 10.0, |
| "grad_norm": 1.6015625, |
| "learning_rate": 1.4671380509367893e-11, |
| "loss": 0.1341402292251587, |
| "mean_token_accuracy": 0.9616317078471184, |
| "num_tokens": 134755992.0, |
| "step": 1890 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1890, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 2.4340057784875745e+18, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
