| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.0, |
| "eval_steps": 500, |
| "global_step": 1890, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "entropy": 1.2536803647875785, |
| "epoch": 0.05291005291005291, |
| "grad_norm": 8.8125, |
| "learning_rate": 3.2142857142857147e-06, |
| "loss": 1.1722795486450195, |
| "mean_token_accuracy": 0.7258133076131343, |
| "num_tokens": 710093.0, |
| "step": 10 |
| }, |
| { |
| "entropy": 0.9760579235851765, |
| "epoch": 0.10582010582010581, |
| "grad_norm": 2.65625, |
| "learning_rate": 6.785714285714287e-06, |
| "loss": 0.9763177871704102, |
| "mean_token_accuracy": 0.7567757397890091, |
| "num_tokens": 1429271.0, |
| "step": 20 |
| }, |
| { |
| "entropy": 0.794147289544344, |
| "epoch": 0.15873015873015872, |
| "grad_norm": 2.46875, |
| "learning_rate": 1.0357142857142859e-05, |
| "loss": 0.8046525001525879, |
| "mean_token_accuracy": 0.7873034350574016, |
| "num_tokens": 2149735.0, |
| "step": 30 |
| }, |
| { |
| "entropy": 0.7088500954210758, |
| "epoch": 0.21164021164021163, |
| "grad_norm": 1.4609375, |
| "learning_rate": 1.3928571428571429e-05, |
| "loss": 0.7121071815490723, |
| "mean_token_accuracy": 0.8070594437420369, |
| "num_tokens": 2868867.0, |
| "step": 40 |
| }, |
| { |
| "entropy": 0.683020292967558, |
| "epoch": 0.26455026455026454, |
| "grad_norm": 1.46875, |
| "learning_rate": 1.7500000000000002e-05, |
| "loss": 0.68443603515625, |
| "mean_token_accuracy": 0.8138657048344612, |
| "num_tokens": 3582476.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 0.6564815938472748, |
| "epoch": 0.31746031746031744, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.999986795783372e-05, |
| "loss": 0.6578450202941895, |
| "mean_token_accuracy": 0.8192978233098984, |
| "num_tokens": 4287689.0, |
| "step": 60 |
| }, |
| { |
| "entropy": 0.6446133196353913, |
| "epoch": 0.37037037037037035, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.9997520638548385e-05, |
| "loss": 0.651158332824707, |
| "mean_token_accuracy": 0.8207244381308556, |
| "num_tokens": 5011191.0, |
| "step": 70 |
| }, |
| { |
| "entropy": 0.6360202118754387, |
| "epoch": 0.42328042328042326, |
| "grad_norm": 1.3984375, |
| "learning_rate": 1.9992239841688418e-05, |
| "loss": 0.6350794792175293, |
| "mean_token_accuracy": 0.8251068785786628, |
| "num_tokens": 5714842.0, |
| "step": 80 |
| }, |
| { |
| "entropy": 0.6189912840723991, |
| "epoch": 0.47619047619047616, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.9984027116747914e-05, |
| "loss": 0.622182559967041, |
| "mean_token_accuracy": 0.8279523886740208, |
| "num_tokens": 6424614.0, |
| "step": 90 |
| }, |
| { |
| "entropy": 0.6159785229712724, |
| "epoch": 0.5291005291005291, |
| "grad_norm": 1.3671875, |
| "learning_rate": 1.997288487350878e-05, |
| "loss": 0.617555046081543, |
| "mean_token_accuracy": 0.8284852005541324, |
| "num_tokens": 7139778.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 0.6129939660429955, |
| "epoch": 0.582010582010582, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.9958816381333684e-05, |
| "loss": 0.6128347873687744, |
| "mean_token_accuracy": 0.8294584318995476, |
| "num_tokens": 7852540.0, |
| "step": 110 |
| }, |
| { |
| "entropy": 0.6052542962133884, |
| "epoch": 0.6349206349206349, |
| "grad_norm": 1.3359375, |
| "learning_rate": 1.994182576820673e-05, |
| "loss": 0.6046356201171875, |
| "mean_token_accuracy": 0.831394524872303, |
| "num_tokens": 8568056.0, |
| "step": 120 |
| }, |
| { |
| "entropy": 0.6013224326074124, |
| "epoch": 0.6878306878306878, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.9921918019522238e-05, |
| "loss": 0.6063410758972168, |
| "mean_token_accuracy": 0.8313738636672496, |
| "num_tokens": 9276742.0, |
| "step": 130 |
| }, |
| { |
| "entropy": 0.5983927462249994, |
| "epoch": 0.7407407407407407, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.9899098976621922e-05, |
| "loss": 0.5977112770080566, |
| "mean_token_accuracy": 0.8335464030504227, |
| "num_tokens": 9990163.0, |
| "step": 140 |
| }, |
| { |
| "entropy": 0.5922725383192301, |
| "epoch": 0.7936507936507936, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.9873375335080915e-05, |
| "loss": 0.5917873382568359, |
| "mean_token_accuracy": 0.8347524181008339, |
| "num_tokens": 10710006.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 0.5920293003320694, |
| "epoch": 0.8465608465608465, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.9844754642743164e-05, |
| "loss": 0.5962053298950195, |
| "mean_token_accuracy": 0.8335559628903866, |
| "num_tokens": 11416822.0, |
| "step": 160 |
| }, |
| { |
| "entropy": 0.5966673269867897, |
| "epoch": 0.8994708994708994, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.981324529750672e-05, |
| "loss": 0.6019061088562012, |
| "mean_token_accuracy": 0.83258865326643, |
| "num_tokens": 12123118.0, |
| "step": 170 |
| }, |
| { |
| "entropy": 0.5854565776884556, |
| "epoch": 0.9523809523809523, |
| "grad_norm": 1.359375, |
| "learning_rate": 1.9778856544859647e-05, |
| "loss": 0.5845203399658203, |
| "mean_token_accuracy": 0.8360438510775566, |
| "num_tokens": 12831058.0, |
| "step": 180 |
| }, |
| { |
| "entropy": 0.5759805988520383, |
| "epoch": 1.0052910052910053, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.9741598475167173e-05, |
| "loss": 0.5696067810058594, |
| "mean_token_accuracy": 0.839682175219059, |
| "num_tokens": 13547943.0, |
| "step": 190 |
| }, |
| { |
| "entropy": 0.5275494448840619, |
| "epoch": 1.0582010582010581, |
| "grad_norm": 1.3984375, |
| "learning_rate": 1.9701482020711004e-05, |
| "loss": 0.5299928665161133, |
| "mean_token_accuracy": 0.8484233327209949, |
| "num_tokens": 14264772.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 0.5196751751005649, |
| "epoch": 1.1111111111111112, |
| "grad_norm": 1.2109375, |
| "learning_rate": 1.9658518952481543e-05, |
| "loss": 0.5208037853240967, |
| "mean_token_accuracy": 0.8509362854063511, |
| "num_tokens": 14985565.0, |
| "step": 210 |
| }, |
| { |
| "entropy": 0.5303940393030644, |
| "epoch": 1.164021164021164, |
| "grad_norm": 1.3359375, |
| "learning_rate": 1.9612721876724054e-05, |
| "loss": 0.5318955421447754, |
| "mean_token_accuracy": 0.8480072498321534, |
| "num_tokens": 15701796.0, |
| "step": 220 |
| }, |
| { |
| "entropy": 0.5232694145292044, |
| "epoch": 1.216931216931217, |
| "grad_norm": 1.359375, |
| "learning_rate": 1.9564104231239726e-05, |
| "loss": 0.5231654167175293, |
| "mean_token_accuracy": 0.8490844376385212, |
| "num_tokens": 16412522.0, |
| "step": 230 |
| }, |
| { |
| "entropy": 0.529466836899519, |
| "epoch": 1.2698412698412698, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.9512680281442745e-05, |
| "loss": 0.5309067726135254, |
| "mean_token_accuracy": 0.8485323786735535, |
| "num_tokens": 17126523.0, |
| "step": 240 |
| }, |
| { |
| "entropy": 0.527790617197752, |
| "epoch": 1.3227513227513228, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.9458465116174534e-05, |
| "loss": 0.5262090682983398, |
| "mean_token_accuracy": 0.8493455089628696, |
| "num_tokens": 17834934.0, |
| "step": 250 |
| }, |
| { |
| "entropy": 0.528411227837205, |
| "epoch": 1.3756613756613756, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.940147464327637e-05, |
| "loss": 0.529714298248291, |
| "mean_token_accuracy": 0.8480914957821369, |
| "num_tokens": 18553413.0, |
| "step": 260 |
| }, |
| { |
| "entropy": 0.516933124884963, |
| "epoch": 1.4285714285714286, |
| "grad_norm": 1.296875, |
| "learning_rate": 1.9341725584921712e-05, |
| "loss": 0.5172879219055175, |
| "mean_token_accuracy": 0.8517736725509166, |
| "num_tokens": 19270191.0, |
| "step": 270 |
| }, |
| { |
| "entropy": 0.5223661050200462, |
| "epoch": 1.4814814814814814, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.9279235472709554e-05, |
| "loss": 0.5231949329376221, |
| "mean_token_accuracy": 0.8496842503547668, |
| "num_tokens": 19976552.0, |
| "step": 280 |
| }, |
| { |
| "entropy": 0.5293067336082459, |
| "epoch": 1.5343915343915344, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.92140226425203e-05, |
| "loss": 0.5295753479003906, |
| "mean_token_accuracy": 0.8479710258543491, |
| "num_tokens": 20676613.0, |
| "step": 290 |
| }, |
| { |
| "entropy": 0.5302139949053526, |
| "epoch": 1.5873015873015874, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.914610622913566e-05, |
| "loss": 0.530374813079834, |
| "mean_token_accuracy": 0.8484688423573971, |
| "num_tokens": 21385978.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 0.5183596257120371, |
| "epoch": 1.6402116402116402, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.9075506160624068e-05, |
| "loss": 0.519154167175293, |
| "mean_token_accuracy": 0.8509341597557067, |
| "num_tokens": 22094336.0, |
| "step": 310 |
| }, |
| { |
| "entropy": 0.5201449614018202, |
| "epoch": 1.693121693121693, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.900224315249343e-05, |
| "loss": 0.5171724796295166, |
| "mean_token_accuracy": 0.8508495308458806, |
| "num_tokens": 22798906.0, |
| "step": 320 |
| }, |
| { |
| "entropy": 0.5111908499151468, |
| "epoch": 1.746031746031746, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.8926338701612738e-05, |
| "loss": 0.511273717880249, |
| "mean_token_accuracy": 0.8524932019412518, |
| "num_tokens": 23511239.0, |
| "step": 330 |
| }, |
| { |
| "entropy": 0.5265866108238697, |
| "epoch": 1.798941798941799, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.884781507990445e-05, |
| "loss": 0.5289421558380127, |
| "mean_token_accuracy": 0.8486531592905522, |
| "num_tokens": 24231185.0, |
| "step": 340 |
| }, |
| { |
| "entropy": 0.5189403794705868, |
| "epoch": 1.8518518518518519, |
| "grad_norm": 1.2578125, |
| "learning_rate": 1.8766695327809463e-05, |
| "loss": 0.5158460140228271, |
| "mean_token_accuracy": 0.8517816573381424, |
| "num_tokens": 24944285.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 0.5055529039353133, |
| "epoch": 1.9047619047619047, |
| "grad_norm": 1.1953125, |
| "learning_rate": 1.8683003247526567e-05, |
| "loss": 0.5064782619476318, |
| "mean_token_accuracy": 0.8533773042261601, |
| "num_tokens": 25670441.0, |
| "step": 360 |
| }, |
| { |
| "entropy": 0.5245815236121416, |
| "epoch": 1.9576719576719577, |
| "grad_norm": 1.2421875, |
| "learning_rate": 1.8596763396028384e-05, |
| "loss": 0.5262872219085694, |
| "mean_token_accuracy": 0.8492341041564941, |
| "num_tokens": 26379786.0, |
| "step": 370 |
| }, |
| { |
| "entropy": 0.5117530789226293, |
| "epoch": 2.0105820105820107, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.8508001077855876e-05, |
| "loss": 0.5020773887634278, |
| "mean_token_accuracy": 0.854968623816967, |
| "num_tokens": 27097166.0, |
| "step": 380 |
| }, |
| { |
| "entropy": 0.44686204604804514, |
| "epoch": 2.0634920634920633, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.8416742337693436e-05, |
| "loss": 0.4472813606262207, |
| "mean_token_accuracy": 0.8682501375675201, |
| "num_tokens": 27818991.0, |
| "step": 390 |
| }, |
| { |
| "entropy": 0.4516366470605135, |
| "epoch": 2.1164021164021163, |
| "grad_norm": 1.484375, |
| "learning_rate": 1.8323013952726877e-05, |
| "loss": 0.45279760360717775, |
| "mean_token_accuracy": 0.8667486943304539, |
| "num_tokens": 28524822.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 0.44368173032999036, |
| "epoch": 2.1693121693121693, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.8226843424786434e-05, |
| "loss": 0.4443831443786621, |
| "mean_token_accuracy": 0.8681288480758667, |
| "num_tokens": 29245634.0, |
| "step": 410 |
| }, |
| { |
| "entropy": 0.44680118896067145, |
| "epoch": 2.2222222222222223, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.8128258972277176e-05, |
| "loss": 0.4440959930419922, |
| "mean_token_accuracy": 0.8689466655254364, |
| "num_tokens": 29967135.0, |
| "step": 420 |
| }, |
| { |
| "entropy": 0.4494645904749632, |
| "epoch": 2.2751322751322753, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.8027289521899143e-05, |
| "loss": 0.45146980285644533, |
| "mean_token_accuracy": 0.8666525632143021, |
| "num_tokens": 30680274.0, |
| "step": 430 |
| }, |
| { |
| "entropy": 0.4557589769363403, |
| "epoch": 2.328042328042328, |
| "grad_norm": 1.2734375, |
| "learning_rate": 1.7923964700159658e-05, |
| "loss": 0.45585927963256834, |
| "mean_token_accuracy": 0.8663338787853718, |
| "num_tokens": 31386859.0, |
| "step": 440 |
| }, |
| { |
| "entropy": 0.46223078556358815, |
| "epoch": 2.380952380952381, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.78183148246803e-05, |
| "loss": 0.4587599754333496, |
| "mean_token_accuracy": 0.86525354757905, |
| "num_tokens": 32111007.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 0.4548660650849342, |
| "epoch": 2.433862433862434, |
| "grad_norm": 1.265625, |
| "learning_rate": 1.77103708953011e-05, |
| "loss": 0.4531528949737549, |
| "mean_token_accuracy": 0.8663033962249755, |
| "num_tokens": 32819180.0, |
| "step": 460 |
| }, |
| { |
| "entropy": 0.4630272053182125, |
| "epoch": 2.4867724867724865, |
| "grad_norm": 1.28125, |
| "learning_rate": 1.7600164584984548e-05, |
| "loss": 0.4610149383544922, |
| "mean_token_accuracy": 0.8639790855348111, |
| "num_tokens": 33523113.0, |
| "step": 470 |
| }, |
| { |
| "entropy": 0.4509055320173502, |
| "epoch": 2.5396825396825395, |
| "grad_norm": 1.34375, |
| "learning_rate": 1.7487728230522085e-05, |
| "loss": 0.44989833831787107, |
| "mean_token_accuracy": 0.8673267938196659, |
| "num_tokens": 34240357.0, |
| "step": 480 |
| }, |
| { |
| "entropy": 0.455976203456521, |
| "epoch": 2.5925925925925926, |
| "grad_norm": 1.2265625, |
| "learning_rate": 1.737309482304585e-05, |
| "loss": 0.4550386905670166, |
| "mean_token_accuracy": 0.8660441562533379, |
| "num_tokens": 34958032.0, |
| "step": 490 |
| }, |
| { |
| "entropy": 0.4540476597845554, |
| "epoch": 2.6455026455026456, |
| "grad_norm": 1.328125, |
| "learning_rate": 1.7256297998348388e-05, |
| "loss": 0.45427827835083007, |
| "mean_token_accuracy": 0.866205920279026, |
| "num_tokens": 35658658.0, |
| "step": 500 |
| }, |
| { |
| "entropy": 0.45439887307584287, |
| "epoch": 2.6984126984126986, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.7137372027013212e-05, |
| "loss": 0.45563058853149413, |
| "mean_token_accuracy": 0.865429712831974, |
| "num_tokens": 36364754.0, |
| "step": 510 |
| }, |
| { |
| "entropy": 0.46197318099439144, |
| "epoch": 2.751322751322751, |
| "grad_norm": 1.296875, |
| "learning_rate": 1.7016351804359134e-05, |
| "loss": 0.46036925315856936, |
| "mean_token_accuracy": 0.8651507623493672, |
| "num_tokens": 37074678.0, |
| "step": 520 |
| }, |
| { |
| "entropy": 0.47125925347208975, |
| "epoch": 2.804232804232804, |
| "grad_norm": 1.296875, |
| "learning_rate": 1.6893272840201243e-05, |
| "loss": 0.4696796417236328, |
| "mean_token_accuracy": 0.862788101285696, |
| "num_tokens": 37787128.0, |
| "step": 530 |
| }, |
| { |
| "entropy": 0.4597961250692606, |
| "epoch": 2.857142857142857, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.6768171248431603e-05, |
| "loss": 0.45906572341918944, |
| "mean_token_accuracy": 0.8650128476321697, |
| "num_tokens": 38504013.0, |
| "step": 540 |
| }, |
| { |
| "entropy": 0.4489801775664091, |
| "epoch": 2.91005291005291, |
| "grad_norm": 1.3046875, |
| "learning_rate": 1.66410837364227e-05, |
| "loss": 0.4477699279785156, |
| "mean_token_accuracy": 0.8676867328584195, |
| "num_tokens": 39212782.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 0.4513770941644907, |
| "epoch": 2.962962962962963, |
| "grad_norm": 1.2734375, |
| "learning_rate": 1.651204759425674e-05, |
| "loss": 0.4503082275390625, |
| "mean_token_accuracy": 0.8672237955033779, |
| "num_tokens": 39931947.0, |
| "step": 560 |
| }, |
| { |
| "entropy": 0.4483271919190884, |
| "epoch": 3.015873015873016, |
| "grad_norm": 1.34375, |
| "learning_rate": 1.6381100683783984e-05, |
| "loss": 0.4379873275756836, |
| "mean_token_accuracy": 0.8702801637351513, |
| "num_tokens": 40643225.0, |
| "step": 570 |
| }, |
| { |
| "entropy": 0.3994087837636471, |
| "epoch": 3.068783068783069, |
| "grad_norm": 1.375, |
| "learning_rate": 1.6248281427513293e-05, |
| "loss": 0.40430545806884766, |
| "mean_token_accuracy": 0.8792076490819454, |
| "num_tokens": 41359577.0, |
| "step": 580 |
| }, |
| { |
| "entropy": 0.40569392777979374, |
| "epoch": 3.121693121693122, |
| "grad_norm": 1.46875, |
| "learning_rate": 1.6113628797338182e-05, |
| "loss": 0.40410671234130857, |
| "mean_token_accuracy": 0.8795830741524696, |
| "num_tokens": 42068599.0, |
| "step": 590 |
| }, |
| { |
| "entropy": 0.39706668704748155, |
| "epoch": 3.1746031746031744, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.597718230310167e-05, |
| "loss": 0.39309277534484866, |
| "mean_token_accuracy": 0.8820179499685764, |
| "num_tokens": 42785175.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 0.3929748997092247, |
| "epoch": 3.2275132275132274, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.5838981981003273e-05, |
| "loss": 0.391530179977417, |
| "mean_token_accuracy": 0.8828775212168694, |
| "num_tokens": 43504295.0, |
| "step": 610 |
| }, |
| { |
| "entropy": 0.4006485942751169, |
| "epoch": 3.2804232804232805, |
| "grad_norm": 1.3359375, |
| "learning_rate": 1.5699068381851576e-05, |
| "loss": 0.39961411952972414, |
| "mean_token_accuracy": 0.8799484111368656, |
| "num_tokens": 44210262.0, |
| "step": 620 |
| }, |
| { |
| "entropy": 0.40273183323442935, |
| "epoch": 3.3333333333333335, |
| "grad_norm": 1.296875, |
| "learning_rate": 1.5557482559165777e-05, |
| "loss": 0.3995194435119629, |
| "mean_token_accuracy": 0.8803919807076455, |
| "num_tokens": 44921899.0, |
| "step": 630 |
| }, |
| { |
| "entropy": 0.39858098365366457, |
| "epoch": 3.386243386243386, |
| "grad_norm": 1.359375, |
| "learning_rate": 1.541426605712975e-05, |
| "loss": 0.39831721782684326, |
| "mean_token_accuracy": 0.8801213204860687, |
| "num_tokens": 45631735.0, |
| "step": 640 |
| }, |
| { |
| "entropy": 0.40417286232113836, |
| "epoch": 3.439153439153439, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.526946089840214e-05, |
| "loss": 0.4012503147125244, |
| "mean_token_accuracy": 0.8794990263879299, |
| "num_tokens": 46339604.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 0.4001857254654169, |
| "epoch": 3.492063492063492, |
| "grad_norm": 1.359375, |
| "learning_rate": 1.5123109571786034e-05, |
| "loss": 0.3985880374908447, |
| "mean_token_accuracy": 0.8811507351696491, |
| "num_tokens": 47060193.0, |
| "step": 660 |
| }, |
| { |
| "entropy": 0.40479574501514437, |
| "epoch": 3.544973544973545, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.4975255019761896e-05, |
| "loss": 0.40291218757629393, |
| "mean_token_accuracy": 0.8791408479213715, |
| "num_tokens": 47768090.0, |
| "step": 670 |
| }, |
| { |
| "entropy": 0.4033591251820326, |
| "epoch": 3.597883597883598, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.4825940625887343e-05, |
| "loss": 0.4013660430908203, |
| "mean_token_accuracy": 0.8803223237395287, |
| "num_tokens": 48486016.0, |
| "step": 680 |
| }, |
| { |
| "entropy": 0.40111338496208193, |
| "epoch": 3.6507936507936507, |
| "grad_norm": 1.3203125, |
| "learning_rate": 1.467521020206753e-05, |
| "loss": 0.4004556655883789, |
| "mean_token_accuracy": 0.8803155072033405, |
| "num_tokens": 49196144.0, |
| "step": 690 |
| }, |
| { |
| "entropy": 0.4051408216357231, |
| "epoch": 3.7037037037037037, |
| "grad_norm": 1.4453125, |
| "learning_rate": 1.4523107975699833e-05, |
| "loss": 0.40234980583190916, |
| "mean_token_accuracy": 0.8797906585037708, |
| "num_tokens": 49912021.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 0.40589384697377684, |
| "epoch": 3.7566137566137567, |
| "grad_norm": 1.3828125, |
| "learning_rate": 1.4369678576696633e-05, |
| "loss": 0.40387759208679197, |
| "mean_token_accuracy": 0.8787553466856479, |
| "num_tokens": 50634262.0, |
| "step": 710 |
| }, |
| { |
| "entropy": 0.40110721029341223, |
| "epoch": 3.8095238095238093, |
| "grad_norm": 1.375, |
| "learning_rate": 1.4214967024389978e-05, |
| "loss": 0.3980360269546509, |
| "mean_token_accuracy": 0.8805328562855721, |
| "num_tokens": 51343452.0, |
| "step": 720 |
| }, |
| { |
| "entropy": 0.40014385208487513, |
| "epoch": 3.8624338624338623, |
| "grad_norm": 1.3671875, |
| "learning_rate": 1.4059018714321986e-05, |
| "loss": 0.39777488708496095, |
| "mean_token_accuracy": 0.8803488783538341, |
| "num_tokens": 52046363.0, |
| "step": 730 |
| }, |
| { |
| "entropy": 0.39626855254173277, |
| "epoch": 3.9153439153439153, |
| "grad_norm": 1.3671875, |
| "learning_rate": 1.390187940492486e-05, |
| "loss": 0.39633784294128416, |
| "mean_token_accuracy": 0.8808404296636582, |
| "num_tokens": 52755135.0, |
| "step": 740 |
| }, |
| { |
| "entropy": 0.39399982392787936, |
| "epoch": 3.9682539682539684, |
| "grad_norm": 1.3125, |
| "learning_rate": 1.3743595204094442e-05, |
| "loss": 0.3921464443206787, |
| "mean_token_accuracy": 0.8814935088157654, |
| "num_tokens": 53473566.0, |
| "step": 750 |
| }, |
| { |
| "entropy": 0.38536344952881335, |
| "epoch": 4.021164021164021, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.3584212555661187e-05, |
| "loss": 0.37671327590942383, |
| "mean_token_accuracy": 0.8863395161926746, |
| "num_tokens": 54190492.0, |
| "step": 760 |
| }, |
| { |
| "entropy": 0.353043582290411, |
| "epoch": 4.074074074074074, |
| "grad_norm": 1.375, |
| "learning_rate": 1.3423778225762609e-05, |
| "loss": 0.35420970916748046, |
| "mean_token_accuracy": 0.8928504519164562, |
| "num_tokens": 54897936.0, |
| "step": 770 |
| }, |
| { |
| "entropy": 0.36162243895232676, |
| "epoch": 4.1269841269841265, |
| "grad_norm": 1.5234375, |
| "learning_rate": 1.3262339289121129e-05, |
| "loss": 0.3595908164978027, |
| "mean_token_accuracy": 0.8911875270307064, |
| "num_tokens": 55609388.0, |
| "step": 780 |
| }, |
| { |
| "entropy": 0.35968837030231954, |
| "epoch": 4.1798941798941796, |
| "grad_norm": 1.5546875, |
| "learning_rate": 1.3099943115231386e-05, |
| "loss": 0.3570686101913452, |
| "mean_token_accuracy": 0.8919634193181991, |
| "num_tokens": 56329299.0, |
| "step": 790 |
| }, |
| { |
| "entropy": 0.3573555305600166, |
| "epoch": 4.232804232804233, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.2936637354461064e-05, |
| "loss": 0.35538694858551023, |
| "mean_token_accuracy": 0.891864238679409, |
| "num_tokens": 57042704.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 0.3631866227835417, |
| "epoch": 4.285714285714286, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.2772469924069285e-05, |
| "loss": 0.36122586727142336, |
| "mean_token_accuracy": 0.8907185509800911, |
| "num_tokens": 57755769.0, |
| "step": 810 |
| }, |
| { |
| "entropy": 0.35537443719804285, |
| "epoch": 4.338624338624339, |
| "grad_norm": 1.3515625, |
| "learning_rate": 1.2607488994146703e-05, |
| "loss": 0.3507373809814453, |
| "mean_token_accuracy": 0.8932205140590668, |
| "num_tokens": 58467967.0, |
| "step": 820 |
| }, |
| { |
| "entropy": 0.36577242650091646, |
| "epoch": 4.391534391534392, |
| "grad_norm": 1.421875, |
| "learning_rate": 1.2441742973481432e-05, |
| "loss": 0.36283602714538576, |
| "mean_token_accuracy": 0.8900898195803165, |
| "num_tokens": 59188579.0, |
| "step": 830 |
| }, |
| { |
| "entropy": 0.3536319576203823, |
| "epoch": 4.444444444444445, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.2275280495354889e-05, |
| "loss": 0.3507254123687744, |
| "mean_token_accuracy": 0.8936197653412818, |
| "num_tokens": 59899589.0, |
| "step": 840 |
| }, |
| { |
| "entropy": 0.35393124222755434, |
| "epoch": 4.497354497354498, |
| "grad_norm": 1.390625, |
| "learning_rate": 1.21081504032718e-05, |
| "loss": 0.35221099853515625, |
| "mean_token_accuracy": 0.8937442168593407, |
| "num_tokens": 60620913.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 0.3644785340875387, |
| "epoch": 4.550264550264551, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.1940401736628524e-05, |
| "loss": 0.36148934364318847, |
| "mean_token_accuracy": 0.8908700153231621, |
| "num_tokens": 61330675.0, |
| "step": 860 |
| }, |
| { |
| "entropy": 0.36559525281190874, |
| "epoch": 4.603174603174603, |
| "grad_norm": 1.546875, |
| "learning_rate": 1.1772083716323878e-05, |
| "loss": 0.36388137340545657, |
| "mean_token_accuracy": 0.8894597724080086, |
| "num_tokens": 62041645.0, |
| "step": 870 |
| }, |
| { |
| "entropy": 0.3636305950582027, |
| "epoch": 4.656084656084656, |
| "grad_norm": 1.3671875, |
| "learning_rate": 1.1603245730316724e-05, |
| "loss": 0.35996088981628416, |
| "mean_token_accuracy": 0.8900512047111988, |
| "num_tokens": 62750446.0, |
| "step": 880 |
| }, |
| { |
| "entropy": 0.35667852535843847, |
| "epoch": 4.708994708994709, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.1433937319134512e-05, |
| "loss": 0.3551082372665405, |
| "mean_token_accuracy": 0.8923696964979172, |
| "num_tokens": 63460008.0, |
| "step": 890 |
| }, |
| { |
| "entropy": 0.35801570676267147, |
| "epoch": 4.761904761904762, |
| "grad_norm": 1.4375, |
| "learning_rate": 1.1264208161337091e-05, |
| "loss": 0.3563329458236694, |
| "mean_token_accuracy": 0.8912324517965317, |
| "num_tokens": 64164803.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 0.36424682177603246, |
| "epoch": 4.814814814814815, |
| "grad_norm": 1.4140625, |
| "learning_rate": 1.1094108058940005e-05, |
| "loss": 0.3606156826019287, |
| "mean_token_accuracy": 0.8901903286576272, |
| "num_tokens": 64878163.0, |
| "step": 910 |
| }, |
| { |
| "entropy": 0.3541768804192543, |
| "epoch": 4.867724867724868, |
| "grad_norm": 1.4296875, |
| "learning_rate": 1.0923686922801537e-05, |
| "loss": 0.34981212615966795, |
| "mean_token_accuracy": 0.893631624430418, |
| "num_tokens": 65599878.0, |
| "step": 920 |
| }, |
| { |
| "entropy": 0.35314388051629064, |
| "epoch": 4.920634920634921, |
| "grad_norm": 1.40625, |
| "learning_rate": 1.0752994757977862e-05, |
| "loss": 0.3507479190826416, |
| "mean_token_accuracy": 0.8934470795094966, |
| "num_tokens": 66312478.0, |
| "step": 930 |
| }, |
| { |
| "entropy": 0.36552518904209136, |
| "epoch": 4.973544973544973, |
| "grad_norm": 1.40625, |
| "learning_rate": 1.058208164905052e-05, |
| "loss": 0.364410400390625, |
| "mean_token_accuracy": 0.8898321248590946, |
| "num_tokens": 67026652.0, |
| "step": 940 |
| }, |
| { |
| "entropy": 0.34920971691608427, |
| "epoch": 5.026455026455026, |
| "grad_norm": 1.4921875, |
| "learning_rate": 1.041099774543059e-05, |
| "loss": 0.33951549530029296, |
| "mean_token_accuracy": 0.896391049772501, |
| "num_tokens": 67743837.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 0.3251554422080517, |
| "epoch": 5.079365079365079, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.023979324664382e-05, |
| "loss": 0.3226634979248047, |
| "mean_token_accuracy": 0.9009568735957145, |
| "num_tokens": 68459077.0, |
| "step": 960 |
| }, |
| { |
| "entropy": 0.3228011641651392, |
| "epoch": 5.132275132275132, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.0068518387601038e-05, |
| "loss": 0.31957221031188965, |
| "mean_token_accuracy": 0.9022335946559906, |
| "num_tokens": 69175886.0, |
| "step": 970 |
| }, |
| { |
| "entropy": 0.3227011114358902, |
| "epoch": 5.185185185185185, |
| "grad_norm": 1.4765625, |
| "learning_rate": 9.897223423858226e-06, |
| "loss": 0.31845569610595703, |
| "mean_token_accuracy": 0.9026259168982506, |
| "num_tokens": 69888377.0, |
| "step": 980 |
| }, |
| { |
| "entropy": 0.3288699559867382, |
| "epoch": 5.238095238095238, |
| "grad_norm": 1.453125, |
| "learning_rate": 9.725958616870495e-06, |
| "loss": 0.3244814872741699, |
| "mean_token_accuracy": 0.9008886761963367, |
| "num_tokens": 70604584.0, |
| "step": 990 |
| }, |
| { |
| "entropy": 0.33571923822164534, |
| "epoch": 5.291005291005291, |
| "grad_norm": 1.546875, |
| "learning_rate": 9.554774219244344e-06, |
| "loss": 0.3316959857940674, |
| "mean_token_accuracy": 0.899080365896225, |
| "num_tokens": 71321169.0, |
| "step": 1000 |
| }, |
| { |
| "entropy": 0.3336105212569237, |
| "epoch": 5.343915343915344, |
| "grad_norm": 1.4296875, |
| "learning_rate": 9.38372045999252e-06, |
| "loss": 0.32957212924957274, |
| "mean_token_accuracy": 0.8995683066546917, |
| "num_tokens": 72041175.0, |
| "step": 1010 |
| }, |
| { |
| "entropy": 0.3264105889946222, |
| "epoch": 5.396825396825397, |
| "grad_norm": 1.5390625, |
| "learning_rate": 9.212847529795784e-06, |
| "loss": 0.32186915874481203, |
| "mean_token_accuracy": 0.9013952367007733, |
| "num_tokens": 72749079.0, |
| "step": 1020 |
| }, |
| { |
| "entropy": 0.3262810740619898, |
| "epoch": 5.449735449735449, |
| "grad_norm": 1.4375, |
| "learning_rate": 9.042205566275952e-06, |
| "loss": 0.32430481910705566, |
| "mean_token_accuracy": 0.9012646913528443, |
| "num_tokens": 73462893.0, |
| "step": 1030 |
| }, |
| { |
| "entropy": 0.3334294486790895, |
| "epoch": 5.502645502645502, |
| "grad_norm": 1.46875, |
| "learning_rate": 8.871844639284479e-06, |
| "loss": 0.3293310165405273, |
| "mean_token_accuracy": 0.8999485336244106, |
| "num_tokens": 74167133.0, |
| "step": 1040 |
| }, |
| { |
| "entropy": 0.3317366179078817, |
| "epoch": 5.555555555555555, |
| "grad_norm": 1.4609375, |
| "learning_rate": 8.701814736210933e-06, |
| "loss": 0.3299627065658569, |
| "mean_token_accuracy": 0.8994868867099285, |
| "num_tokens": 74870399.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 0.3392947752028704, |
| "epoch": 5.608465608465608, |
| "grad_norm": 1.46875, |
| "learning_rate": 8.532165747315686e-06, |
| "loss": 0.33389506340026853, |
| "mean_token_accuracy": 0.8977861106395721, |
| "num_tokens": 75590782.0, |
| "step": 1060 |
| }, |
| { |
| "entropy": 0.33543253857642413, |
| "epoch": 5.661375661375661, |
| "grad_norm": 1.4140625, |
| "learning_rate": 8.362947451091072e-06, |
| "loss": 0.33173198699951173, |
| "mean_token_accuracy": 0.8986442036926746, |
| "num_tokens": 76305788.0, |
| "step": 1070 |
| }, |
| { |
| "entropy": 0.3333395093679428, |
| "epoch": 5.714285714285714, |
| "grad_norm": 1.421875, |
| "learning_rate": 8.194209499655364e-06, |
| "loss": 0.33096561431884763, |
| "mean_token_accuracy": 0.8996573880314827, |
| "num_tokens": 77023579.0, |
| "step": 1080 |
| }, |
| { |
| "entropy": 0.33497946709394455, |
| "epoch": 5.767195767195767, |
| "grad_norm": 1.5625, |
| "learning_rate": 8.026001404183831e-06, |
| "loss": 0.3307910919189453, |
| "mean_token_accuracy": 0.8994414336979389, |
| "num_tokens": 77720821.0, |
| "step": 1090 |
| }, |
| { |
| "entropy": 0.3360676342621446, |
| "epoch": 5.8201058201058204, |
| "grad_norm": 1.453125, |
| "learning_rate": 7.85837252038112e-06, |
| "loss": 0.33355400562286375, |
| "mean_token_accuracy": 0.8985835999250412, |
| "num_tokens": 78431671.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 0.33601767774671315, |
| "epoch": 5.8730158730158735, |
| "grad_norm": 1.4609375, |
| "learning_rate": 7.69137203399931e-06, |
| "loss": 0.33111412525177003, |
| "mean_token_accuracy": 0.8989483065903187, |
| "num_tokens": 79141393.0, |
| "step": 1110 |
| }, |
| { |
| "entropy": 0.33282165341079234, |
| "epoch": 5.925925925925926, |
| "grad_norm": 1.4140625, |
| "learning_rate": 7.525048946405791e-06, |
| "loss": 0.33011479377746583, |
| "mean_token_accuracy": 0.8995157510042191, |
| "num_tokens": 79858356.0, |
| "step": 1120 |
| }, |
| { |
| "entropy": 0.3331317029893398, |
| "epoch": 5.978835978835979, |
| "grad_norm": 1.4296875, |
| "learning_rate": 7.359452060205245e-06, |
| "loss": 0.3283017873764038, |
| "mean_token_accuracy": 0.9000704519450664, |
| "num_tokens": 80566532.0, |
| "step": 1130 |
| }, |
| { |
| "entropy": 0.3162786137312651, |
| "epoch": 6.031746031746032, |
| "grad_norm": 1.46875, |
| "learning_rate": 7.194629964919989e-06, |
| "loss": 0.30768542289733886, |
| "mean_token_accuracy": 0.9056257985532283, |
| "num_tokens": 81283415.0, |
| "step": 1140 |
| }, |
| { |
| "entropy": 0.31019794568419456, |
| "epoch": 6.084656084656085, |
| "grad_norm": 1.5078125, |
| "learning_rate": 7.030631022732781e-06, |
| "loss": 0.3101642370223999, |
| "mean_token_accuracy": 0.9054678924381733, |
| "num_tokens": 81995607.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 0.3164690284058452, |
| "epoch": 6.137566137566138, |
| "grad_norm": 1.6015625, |
| "learning_rate": 6.867503354296392e-06, |
| "loss": 0.3106247425079346, |
| "mean_token_accuracy": 0.9047419615089893, |
| "num_tokens": 82712148.0, |
| "step": 1160 |
| }, |
| { |
| "entropy": 0.32531814724206926, |
| "epoch": 6.190476190476191, |
| "grad_norm": 1.5, |
| "learning_rate": 6.705294824614004e-06, |
| "loss": 0.3201704978942871, |
| "mean_token_accuracy": 0.9021763414144516, |
| "num_tokens": 83425506.0, |
| "step": 1170 |
| }, |
| { |
| "entropy": 0.319172390550375, |
| "epoch": 6.243386243386244, |
| "grad_norm": 1.5, |
| "learning_rate": 6.54405302899463e-06, |
| "loss": 0.31355197429656984, |
| "mean_token_accuracy": 0.9040127523243427, |
| "num_tokens": 84137018.0, |
| "step": 1180 |
| }, |
| { |
| "entropy": 0.32360128555446865, |
| "epoch": 6.296296296296296, |
| "grad_norm": 1.4765625, |
| "learning_rate": 6.383825279087699e-06, |
| "loss": 0.31948628425598147, |
| "mean_token_accuracy": 0.9021987453103065, |
| "num_tokens": 84849598.0, |
| "step": 1190 |
| }, |
| { |
| "entropy": 0.3194924917072058, |
| "epoch": 6.349206349206349, |
| "grad_norm": 1.5, |
| "learning_rate": 6.2246585890008245e-06, |
| "loss": 0.31670660972595216, |
| "mean_token_accuracy": 0.9040497578680515, |
| "num_tokens": 85561962.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 0.3129683220759034, |
| "epoch": 6.402116402116402, |
| "grad_norm": 1.5859375, |
| "learning_rate": 6.06659966150491e-06, |
| "loss": 0.3079230785369873, |
| "mean_token_accuracy": 0.9053744673728943, |
| "num_tokens": 86286338.0, |
| "step": 1210 |
| }, |
| { |
| "entropy": 0.3165762163698673, |
| "epoch": 6.455026455026455, |
| "grad_norm": 1.4765625, |
| "learning_rate": 5.909694874330601e-06, |
| "loss": 0.3120215654373169, |
| "mean_token_accuracy": 0.9052867114543914, |
| "num_tokens": 87001911.0, |
| "step": 1220 |
| }, |
| { |
| "entropy": 0.31230680998414756, |
| "epoch": 6.507936507936508, |
| "grad_norm": 1.5078125, |
| "learning_rate": 5.753990266560105e-06, |
| "loss": 0.30913171768188474, |
| "mean_token_accuracy": 0.9060923457145691, |
| "num_tokens": 87714571.0, |
| "step": 1230 |
| }, |
| { |
| "entropy": 0.3162264319136739, |
| "epoch": 6.560846560846561, |
| "grad_norm": 1.5546875, |
| "learning_rate": 5.599531525118373e-06, |
| "loss": 0.3127452850341797, |
| "mean_token_accuracy": 0.9043003864586353, |
| "num_tokens": 88417674.0, |
| "step": 1240 |
| }, |
| { |
| "entropy": 0.3162810893729329, |
| "epoch": 6.613756613756614, |
| "grad_norm": 1.46875, |
| "learning_rate": 5.446363971367638e-06, |
| "loss": 0.30910825729370117, |
| "mean_token_accuracy": 0.9054918497800827, |
| "num_tokens": 89123522.0, |
| "step": 1250 |
| }, |
| { |
| "entropy": 0.31419141963124275, |
| "epoch": 6.666666666666667, |
| "grad_norm": 1.5, |
| "learning_rate": 5.29453254780917e-06, |
| "loss": 0.31052522659301757, |
| "mean_token_accuracy": 0.9056126736104488, |
| "num_tokens": 89838831.0, |
| "step": 1260 |
| }, |
| { |
| "entropy": 0.3123618332669139, |
| "epoch": 6.71957671957672, |
| "grad_norm": 1.4453125, |
| "learning_rate": 5.1440818048962575e-06, |
| "loss": 0.3084896087646484, |
| "mean_token_accuracy": 0.905338567495346, |
| "num_tokens": 90546066.0, |
| "step": 1270 |
| }, |
| { |
| "entropy": 0.31750516016036273, |
| "epoch": 6.772486772486772, |
| "grad_norm": 1.5390625, |
| "learning_rate": 4.995055887962167e-06, |
| "loss": 0.31345534324645996, |
| "mean_token_accuracy": 0.9047295741736889, |
| "num_tokens": 91257301.0, |
| "step": 1280 |
| }, |
| { |
| "entropy": 0.321336280182004, |
| "epoch": 6.825396825396825, |
| "grad_norm": 1.5703125, |
| "learning_rate": 4.8474985242669965e-06, |
| "loss": 0.31727027893066406, |
| "mean_token_accuracy": 0.9030331708490849, |
| "num_tokens": 91961075.0, |
| "step": 1290 |
| }, |
| { |
| "entropy": 0.32664828822016717, |
| "epoch": 6.878306878306878, |
| "grad_norm": 1.515625, |
| "learning_rate": 4.701453010167217e-06, |
| "loss": 0.3214175462722778, |
| "mean_token_accuracy": 0.902243759483099, |
| "num_tokens": 92677769.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 0.31388139445334673, |
| "epoch": 6.931216931216931, |
| "grad_norm": 1.5078125, |
| "learning_rate": 4.556962198411631e-06, |
| "loss": 0.309248685836792, |
| "mean_token_accuracy": 0.9059560939669609, |
| "num_tokens": 93390418.0, |
| "step": 1310 |
| }, |
| { |
| "entropy": 0.3076509850099683, |
| "epoch": 6.984126984126984, |
| "grad_norm": 1.515625, |
| "learning_rate": 4.414068485567473e-06, |
| "loss": 0.3037854194641113, |
| "mean_token_accuracy": 0.9067774422466754, |
| "num_tokens": 94107638.0, |
| "step": 1320 |
| }, |
| { |
| "entropy": 0.309505027346313, |
| "epoch": 7.037037037037037, |
| "grad_norm": 1.4453125, |
| "learning_rate": 4.272813799580417e-06, |
| "loss": 0.29927496910095214, |
| "mean_token_accuracy": 0.9081789322197438, |
| "num_tokens": 94829458.0, |
| "step": 1330 |
| }, |
| { |
| "entropy": 0.3077107332646847, |
| "epoch": 7.08994708994709, |
| "grad_norm": 1.546875, |
| "learning_rate": 4.1332395874720474e-06, |
| "loss": 0.3050470590591431, |
| "mean_token_accuracy": 0.9063494049012661, |
| "num_tokens": 95540140.0, |
| "step": 1340 |
| }, |
| { |
| "entropy": 0.3082722883671522, |
| "epoch": 7.142857142857143, |
| "grad_norm": 1.4375, |
| "learning_rate": 3.9953868031784415e-06, |
| "loss": 0.3043971538543701, |
| "mean_token_accuracy": 0.9069416023790836, |
| "num_tokens": 96254434.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 0.3132112707942724, |
| "epoch": 7.195767195767195, |
| "grad_norm": 1.5, |
| "learning_rate": 3.8592958955334636e-06, |
| "loss": 0.30689363479614257, |
| "mean_token_accuracy": 0.9063246354460717, |
| "num_tokens": 96966014.0, |
| "step": 1360 |
| }, |
| { |
| "entropy": 0.3025448817759752, |
| "epoch": 7.248677248677248, |
| "grad_norm": 1.53125, |
| "learning_rate": 3.725006796400231e-06, |
| "loss": 0.2965035915374756, |
| "mean_token_accuracy": 0.909070585668087, |
| "num_tokens": 97677188.0, |
| "step": 1370 |
| }, |
| { |
| "entropy": 0.30754552241414784, |
| "epoch": 7.301587301587301, |
| "grad_norm": 1.609375, |
| "learning_rate": 3.5925589089542954e-06, |
| "loss": 0.3027654647827148, |
| "mean_token_accuracy": 0.9068837359547615, |
| "num_tokens": 98391677.0, |
| "step": 1380 |
| }, |
| { |
| "entropy": 0.30708651468157766, |
| "epoch": 7.354497354497354, |
| "grad_norm": 1.5078125, |
| "learning_rate": 3.461991096121955e-06, |
| "loss": 0.30066862106323244, |
| "mean_token_accuracy": 0.9076349563896656, |
| "num_tokens": 99105789.0, |
| "step": 1390 |
| }, |
| { |
| "entropy": 0.3103673914447427, |
| "epoch": 7.407407407407407, |
| "grad_norm": 1.484375, |
| "learning_rate": 3.3333416691770402e-06, |
| "loss": 0.3048877716064453, |
| "mean_token_accuracy": 0.9061001695692539, |
| "num_tokens": 99819774.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 0.30695827305316925, |
| "epoch": 7.4603174603174605, |
| "grad_norm": 1.46875, |
| "learning_rate": 3.206648376499637e-06, |
| "loss": 0.30250701904296873, |
| "mean_token_accuracy": 0.90780760794878, |
| "num_tokens": 100540345.0, |
| "step": 1410 |
| }, |
| { |
| "entropy": 0.310471523180604, |
| "epoch": 7.5132275132275135, |
| "grad_norm": 1.5, |
| "learning_rate": 3.0819483924999183e-06, |
| "loss": 0.3093590497970581, |
| "mean_token_accuracy": 0.9053481601178646, |
| "num_tokens": 101261007.0, |
| "step": 1420 |
| }, |
| { |
| "entropy": 0.31176430340856315, |
| "epoch": 7.5661375661375665, |
| "grad_norm": 1.453125, |
| "learning_rate": 2.959278306710409e-06, |
| "loss": 0.30368719100952146, |
| "mean_token_accuracy": 0.9074827641248703, |
| "num_tokens": 101972692.0, |
| "step": 1430 |
| }, |
| { |
| "entropy": 0.30677727833390234, |
| "epoch": 7.619047619047619, |
| "grad_norm": 1.5078125, |
| "learning_rate": 2.838674113049884e-06, |
| "loss": 0.3007056713104248, |
| "mean_token_accuracy": 0.9077389024198055, |
| "num_tokens": 102684836.0, |
| "step": 1440 |
| }, |
| { |
| "entropy": 0.3181039709597826, |
| "epoch": 7.671957671957672, |
| "grad_norm": 1.5234375, |
| "learning_rate": 2.7201711992619873e-06, |
| "loss": 0.3143282890319824, |
| "mean_token_accuracy": 0.9046376876533031, |
| "num_tokens": 103404477.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 0.31714251451194286, |
| "epoch": 7.724867724867725, |
| "grad_norm": 1.5390625, |
| "learning_rate": 2.6038043365317643e-06, |
| "loss": 0.3124920129776001, |
| "mean_token_accuracy": 0.90488907918334, |
| "num_tokens": 104111178.0, |
| "step": 1460 |
| }, |
| { |
| "entropy": 0.3034475488588214, |
| "epoch": 7.777777777777778, |
| "grad_norm": 1.5234375, |
| "learning_rate": 2.489607669283088e-06, |
| "loss": 0.2986010551452637, |
| "mean_token_accuracy": 0.9083267234265804, |
| "num_tokens": 104820924.0, |
| "step": 1470 |
| }, |
| { |
| "entropy": 0.3176258482038975, |
| "epoch": 7.830687830687831, |
| "grad_norm": 1.4921875, |
| "learning_rate": 2.3776147051599518e-06, |
| "loss": 0.313706636428833, |
| "mean_token_accuracy": 0.904422540217638, |
| "num_tokens": 105525661.0, |
| "step": 1480 |
| }, |
| { |
| "entropy": 0.31959378309547903, |
| "epoch": 7.883597883597884, |
| "grad_norm": 1.4921875, |
| "learning_rate": 2.2678583051946733e-06, |
| "loss": 0.3164098262786865, |
| "mean_token_accuracy": 0.9037438824772834, |
| "num_tokens": 106244487.0, |
| "step": 1490 |
| }, |
| { |
| "entropy": 0.3161809345707297, |
| "epoch": 7.936507936507937, |
| "grad_norm": 1.5703125, |
| "learning_rate": 2.1603706741657803e-06, |
| "loss": 0.30962021350860597, |
| "mean_token_accuracy": 0.905376186221838, |
| "num_tokens": 106950025.0, |
| "step": 1500 |
| }, |
| { |
| "entropy": 0.3143223883584142, |
| "epoch": 7.98941798941799, |
| "grad_norm": 1.4609375, |
| "learning_rate": 2.055183351148463e-06, |
| "loss": 0.3079953193664551, |
| "mean_token_accuracy": 0.9055778756737709, |
| "num_tokens": 107659233.0, |
| "step": 1510 |
| }, |
| { |
| "entropy": 0.30422661881893875, |
| "epoch": 8.042328042328043, |
| "grad_norm": 1.625, |
| "learning_rate": 1.9523272002603745e-06, |
| "loss": 0.29883790016174316, |
| "mean_token_accuracy": 0.9084436170756817, |
| "num_tokens": 108374173.0, |
| "step": 1520 |
| }, |
| { |
| "entropy": 0.30016559790819886, |
| "epoch": 8.095238095238095, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.8518324016054601e-06, |
| "loss": 0.2931604146957397, |
| "mean_token_accuracy": 0.910134493559599, |
| "num_tokens": 109085583.0, |
| "step": 1530 |
| }, |
| { |
| "entropy": 0.3010789893567562, |
| "epoch": 8.148148148148149, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.7537284424184852e-06, |
| "loss": 0.2946164131164551, |
| "mean_token_accuracy": 0.9093765616416931, |
| "num_tokens": 109801232.0, |
| "step": 1540 |
| }, |
| { |
| "entropy": 0.30833523981273175, |
| "epoch": 8.201058201058201, |
| "grad_norm": 1.5859375, |
| "learning_rate": 1.658044108412905e-06, |
| "loss": 0.3037881374359131, |
| "mean_token_accuracy": 0.9076551206409931, |
| "num_tokens": 110516150.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 0.3026609459891915, |
| "epoch": 8.253968253968253, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.564807475334509e-06, |
| "loss": 0.29924802780151366, |
| "mean_token_accuracy": 0.9084347561001778, |
| "num_tokens": 111232078.0, |
| "step": 1560 |
| }, |
| { |
| "entropy": 0.30380540397018196, |
| "epoch": 8.306878306878307, |
| "grad_norm": 1.5, |
| "learning_rate": 1.4740459007234498e-06, |
| "loss": 0.2977043390274048, |
| "mean_token_accuracy": 0.9087416157126427, |
| "num_tokens": 111950711.0, |
| "step": 1570 |
| }, |
| { |
| "entropy": 0.30840193424373863, |
| "epoch": 8.359788359788359, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.3857860158869696e-06, |
| "loss": 0.30392866134643554, |
| "mean_token_accuracy": 0.9073121502995491, |
| "num_tokens": 112658277.0, |
| "step": 1580 |
| }, |
| { |
| "entropy": 0.30752763357013463, |
| "epoch": 8.412698412698413, |
| "grad_norm": 1.46875, |
| "learning_rate": 1.3000537180852213e-06, |
| "loss": 0.3043984413146973, |
| "mean_token_accuracy": 0.9070875540375709, |
| "num_tokens": 113378303.0, |
| "step": 1590 |
| }, |
| { |
| "entropy": 0.30953508615493774, |
| "epoch": 8.465608465608465, |
| "grad_norm": 1.5390625, |
| "learning_rate": 1.2168741629324944e-06, |
| "loss": 0.30271248817443847, |
| "mean_token_accuracy": 0.9074668884277344, |
| "num_tokens": 114087597.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 0.29960060995072124, |
| "epoch": 8.518518518518519, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.1362717570160387e-06, |
| "loss": 0.29501566886901853, |
| "mean_token_accuracy": 0.9095637299120426, |
| "num_tokens": 114803563.0, |
| "step": 1610 |
| }, |
| { |
| "entropy": 0.3040910255163908, |
| "epoch": 8.571428571428571, |
| "grad_norm": 1.4765625, |
| "learning_rate": 1.0582701507346582e-06, |
| "loss": 0.2978386402130127, |
| "mean_token_accuracy": 0.9087345205247402, |
| "num_tokens": 115519221.0, |
| "step": 1620 |
| }, |
| { |
| "entropy": 0.3037101987749338, |
| "epoch": 8.624338624338625, |
| "grad_norm": 1.375, |
| "learning_rate": 9.828922313592227e-07, |
| "loss": 0.29636406898498535, |
| "mean_token_accuracy": 0.9094856515526771, |
| "num_tokens": 116238487.0, |
| "step": 1630 |
| }, |
| { |
| "entropy": 0.305535570718348, |
| "epoch": 8.677248677248677, |
| "grad_norm": 1.5703125, |
| "learning_rate": 9.10160116317047e-07, |
| "loss": 0.3010878086090088, |
| "mean_token_accuracy": 0.9079214721918106, |
| "num_tokens": 116956616.0, |
| "step": 1640 |
| }, |
| { |
| "entropy": 0.3080721700564027, |
| "epoch": 8.73015873015873, |
| "grad_norm": 1.5703125, |
| "learning_rate": 8.400951467022067e-07, |
| "loss": 0.30458664894104004, |
| "mean_token_accuracy": 0.9070508010685444, |
| "num_tokens": 117662184.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 0.3044055197387934, |
| "epoch": 8.783068783068783, |
| "grad_norm": 1.5, |
| "learning_rate": 7.727178810136094e-07, |
| "loss": 0.30016112327575684, |
| "mean_token_accuracy": 0.9080108553171158, |
| "num_tokens": 118365440.0, |
| "step": 1660 |
| }, |
| { |
| "entropy": 0.3029957629740238, |
| "epoch": 8.835978835978835, |
| "grad_norm": 1.4765625, |
| "learning_rate": 7.080480891227126e-07, |
| "loss": 0.2962267637252808, |
| "mean_token_accuracy": 0.9091366939246655, |
| "num_tokens": 119078464.0, |
| "step": 1670 |
| }, |
| { |
| "entropy": 0.31025845259428025, |
| "epoch": 8.88888888888889, |
| "grad_norm": 1.4609375, |
| "learning_rate": 6.461047464726433e-07, |
| "loss": 0.30565040111541747, |
| "mean_token_accuracy": 0.907152583450079, |
| "num_tokens": 119792226.0, |
| "step": 1680 |
| }, |
| { |
| "entropy": 0.30560624208301307, |
| "epoch": 8.941798941798941, |
| "grad_norm": 1.53125, |
| "learning_rate": 5.869060285104089e-07, |
| "loss": 0.30106794834136963, |
| "mean_token_accuracy": 0.9083184324204922, |
| "num_tokens": 120500468.0, |
| "step": 1690 |
| }, |
| { |
| "entropy": 0.30569964554160833, |
| "epoch": 8.994708994708995, |
| "grad_norm": 1.4375, |
| "learning_rate": 5.304693053538434e-07, |
| "loss": 0.29906630516052246, |
| "mean_token_accuracy": 0.908292219787836, |
| "num_tokens": 121205368.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 0.29638308342546227, |
| "epoch": 9.047619047619047, |
| "grad_norm": 1.46875, |
| "learning_rate": 4.768111366948736e-07, |
| "loss": 0.2910614013671875, |
| "mean_token_accuracy": 0.9109223008155822, |
| "num_tokens": 121917493.0, |
| "step": 1710 |
| }, |
| { |
| "entropy": 0.3074631987139583, |
| "epoch": 9.100529100529101, |
| "grad_norm": 1.5234375, |
| "learning_rate": 4.2594726694055397e-07, |
| "loss": 0.30144467353820803, |
| "mean_token_accuracy": 0.9081278949975967, |
| "num_tokens": 122643710.0, |
| "step": 1720 |
| }, |
| { |
| "entropy": 0.30852508898824454, |
| "epoch": 9.153439153439153, |
| "grad_norm": 1.5234375, |
| "learning_rate": 3.778926205933342e-07, |
| "loss": 0.3022093534469604, |
| "mean_token_accuracy": 0.9076375000178813, |
| "num_tokens": 123357455.0, |
| "step": 1730 |
| }, |
| { |
| "entropy": 0.3028355214744806, |
| "epoch": 9.206349206349206, |
| "grad_norm": 1.4921875, |
| "learning_rate": 3.3266129787191015e-07, |
| "loss": 0.29703240394592284, |
| "mean_token_accuracy": 0.9097487904131413, |
| "num_tokens": 124071350.0, |
| "step": 1740 |
| }, |
| { |
| "entropy": 0.30439889822155236, |
| "epoch": 9.25925925925926, |
| "grad_norm": 1.5, |
| "learning_rate": 2.9026657057392715e-07, |
| "loss": 0.2984210968017578, |
| "mean_token_accuracy": 0.9092232577502728, |
| "num_tokens": 124788814.0, |
| "step": 1750 |
| }, |
| { |
| "entropy": 0.3000444371253252, |
| "epoch": 9.312169312169312, |
| "grad_norm": 1.5390625, |
| "learning_rate": 2.507208781817638e-07, |
| "loss": 0.294765043258667, |
| "mean_token_accuracy": 0.9093967579305172, |
| "num_tokens": 125504162.0, |
| "step": 1760 |
| }, |
| { |
| "entropy": 0.3023909794166684, |
| "epoch": 9.365079365079366, |
| "grad_norm": 1.5234375, |
| "learning_rate": 2.1403582421253398e-07, |
| "loss": 0.29541583061218263, |
| "mean_token_accuracy": 0.909448866546154, |
| "num_tokens": 126213099.0, |
| "step": 1770 |
| }, |
| { |
| "entropy": 0.307912852615118, |
| "epoch": 9.417989417989418, |
| "grad_norm": 1.5, |
| "learning_rate": 1.8022217281336908e-07, |
| "loss": 0.30301742553710936, |
| "mean_token_accuracy": 0.9073327742516994, |
| "num_tokens": 126934123.0, |
| "step": 1780 |
| }, |
| { |
| "entropy": 0.3068047560751438, |
| "epoch": 9.470899470899472, |
| "grad_norm": 1.5, |
| "learning_rate": 1.4928984560301006e-07, |
| "loss": 0.30068130493164064, |
| "mean_token_accuracy": 0.9078795284032821, |
| "num_tokens": 127647228.0, |
| "step": 1790 |
| }, |
| { |
| "entropy": 0.3161488797515631, |
| "epoch": 9.523809523809524, |
| "grad_norm": 1.5078125, |
| "learning_rate": 1.212479187605897e-07, |
| "loss": 0.31014142036437986, |
| "mean_token_accuracy": 0.9057340927422046, |
| "num_tokens": 128350815.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 0.30816534254699945, |
| "epoch": 9.576719576719576, |
| "grad_norm": 1.5078125, |
| "learning_rate": 9.610462036249823e-08, |
| "loss": 0.3027071952819824, |
| "mean_token_accuracy": 0.9073677562177181, |
| "num_tokens": 129070776.0, |
| "step": 1810 |
| }, |
| { |
| "entropy": 0.3056219911202788, |
| "epoch": 9.62962962962963, |
| "grad_norm": 1.46875, |
| "learning_rate": 7.386732796810014e-08, |
| "loss": 0.3010460615158081, |
| "mean_token_accuracy": 0.9079062268137932, |
| "num_tokens": 129784905.0, |
| "step": 1820 |
| }, |
| { |
| "entropy": 0.30222475472837684, |
| "epoch": 9.682539682539682, |
| "grad_norm": 1.4921875, |
| "learning_rate": 5.454256645500477e-08, |
| "loss": 0.29520158767700194, |
| "mean_token_accuracy": 0.9089621223509312, |
| "num_tokens": 130492958.0, |
| "step": 1830 |
| }, |
| { |
| "entropy": 0.3073990933597088, |
| "epoch": 9.735449735449736, |
| "grad_norm": 1.5078125, |
| "learning_rate": 3.8136006104533276e-08, |
| "loss": 0.3022012710571289, |
| "mean_token_accuracy": 0.9076414979994297, |
| "num_tokens": 131214260.0, |
| "step": 1840 |
| }, |
| { |
| "entropy": 0.30419554468244314, |
| "epoch": 9.788359788359788, |
| "grad_norm": 1.59375, |
| "learning_rate": 2.4652460937941714e-08, |
| "loss": 0.2982198476791382, |
| "mean_token_accuracy": 0.9086012624204158, |
| "num_tokens": 131927722.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 0.3068025728687644, |
| "epoch": 9.841269841269842, |
| "grad_norm": 1.5, |
| "learning_rate": 1.4095887303888778e-08, |
| "loss": 0.3004434108734131, |
| "mean_token_accuracy": 0.9073408909142018, |
| "num_tokens": 132622563.0, |
| "step": 1860 |
| }, |
| { |
| "entropy": 0.3025443097576499, |
| "epoch": 9.894179894179894, |
| "grad_norm": 1.4921875, |
| "learning_rate": 6.469382717563255e-09, |
| "loss": 0.29805679321289064, |
| "mean_token_accuracy": 0.908904866874218, |
| "num_tokens": 133332199.0, |
| "step": 1870 |
| }, |
| { |
| "entropy": 0.3039748823270202, |
| "epoch": 9.947089947089948, |
| "grad_norm": 1.4453125, |
| "learning_rate": 1.775184951806619e-09, |
| "loss": 0.29894199371337893, |
| "mean_token_accuracy": 0.9083068743348122, |
| "num_tokens": 134041278.0, |
| "step": 1880 |
| }, |
| { |
| "entropy": 0.3088474523276091, |
| "epoch": 10.0, |
| "grad_norm": 1.453125, |
| "learning_rate": 1.4671380509367893e-11, |
| "loss": 0.3046253204345703, |
| "mean_token_accuracy": 0.9069877013564109, |
| "num_tokens": 134755992.0, |
| "step": 1890 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 1890, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 2.4340057784875745e+18, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
