| { |
| "best_global_step": 2500, |
| "best_metric": 1.9019721597678847, |
| "best_model_checkpoint": "/content/cot-split3/checkpoint-2500", |
| "epoch": 1.0, |
| "eval_steps": 250, |
| "global_step": 2554, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0, |
| "eval_bpd": 5.410557520272195, |
| "eval_entropy": 3.805819726348172, |
| "eval_loss": 3.754805564880371, |
| "eval_mean_token_accuracy": 0.3303780525619185, |
| "eval_nll": 3.7503126904340807, |
| "eval_num_tokens": 0.0, |
| "eval_ppl": 42.534380014623586, |
| "eval_runtime": 319.2646, |
| "eval_samples_per_second": 568.835, |
| "eval_steps_per_second": 8.889, |
| "step": 0 |
| }, |
| { |
| "entropy": 3.939532442367398, |
| "epoch": 0.00039155800931908063, |
| "grad_norm": 2.921875, |
| "learning_rate": 0.0, |
| "loss": 3.6805007457733154, |
| "mean_token_accuracy": 0.33797585886722376, |
| "num_tokens": 159295.0, |
| "step": 1 |
| }, |
| { |
| "entropy": 4.131688260781753, |
| "epoch": 0.009788950232977016, |
| "grad_norm": 1.828125, |
| "learning_rate": 9.375e-05, |
| "loss": 3.7089433670043945, |
| "mean_token_accuracy": 0.32488122625965454, |
| "num_tokens": 3875056.0, |
| "step": 25 |
| }, |
| { |
| "entropy": 4.03907456745432, |
| "epoch": 0.01957790046595403, |
| "grad_norm": 0.87109375, |
| "learning_rate": 0.00019140625, |
| "loss": 3.2840447998046876, |
| "mean_token_accuracy": 0.3501161635947849, |
| "num_tokens": 7796334.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 3.718109237212473, |
| "epoch": 0.02936685069893105, |
| "grad_norm": 0.875, |
| "learning_rate": 0.0002890625, |
| "loss": 2.907643127441406, |
| "mean_token_accuracy": 0.37667007823156323, |
| "num_tokens": 11688138.0, |
| "step": 75 |
| }, |
| { |
| "entropy": 3.4646144502832286, |
| "epoch": 0.03915580093190806, |
| "grad_norm": 0.71484375, |
| "learning_rate": 0.00038671875, |
| "loss": 2.5448974609375, |
| "mean_token_accuracy": 0.41452789283904995, |
| "num_tokens": 15554877.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 3.2713017840492147, |
| "epoch": 0.04894475116488508, |
| "grad_norm": 0.71484375, |
| "learning_rate": 0.000484375, |
| "loss": 2.4108335876464846, |
| "mean_token_accuracy": 0.4332291316341289, |
| "num_tokens": 19437565.0, |
| "step": 125 |
| }, |
| { |
| "entropy": 3.177977246980637, |
| "epoch": 0.0587337013978621, |
| "grad_norm": 0.6796875, |
| "learning_rate": 0.0004999075642455791, |
| "loss": 2.35259765625, |
| "mean_token_accuracy": 0.4402555277869966, |
| "num_tokens": 23324888.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 3.1149309732260146, |
| "epoch": 0.06852265163083911, |
| "grad_norm": 0.6171875, |
| "learning_rate": 0.0004995565798713207, |
| "loss": 2.2896893310546873, |
| "mean_token_accuracy": 0.44857320203325, |
| "num_tokens": 27225978.0, |
| "step": 175 |
| }, |
| { |
| "entropy": 3.0429316923380134, |
| "epoch": 0.07831160186381612, |
| "grad_norm": 0.7109375, |
| "learning_rate": 0.0004989440608306114, |
| "loss": 2.2477577209472654, |
| "mean_token_accuracy": 0.45537929991850945, |
| "num_tokens": 31124891.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 3.000580134712211, |
| "epoch": 0.08810055209679314, |
| "grad_norm": 0.73046875, |
| "learning_rate": 0.0004980706490418613, |
| "loss": 2.2050234985351564, |
| "mean_token_accuracy": 0.4608995247476681, |
| "num_tokens": 35005557.0, |
| "step": 225 |
| }, |
| { |
| "entropy": 2.972177586040443, |
| "epoch": 0.09788950232977016, |
| "grad_norm": 0.64453125, |
| "learning_rate": 0.0004969372598384225, |
| "loss": 2.1969384765625, |
| "mean_token_accuracy": 0.46284280083339074, |
| "num_tokens": 38902413.0, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.09788950232977016, |
| "eval_bpd": 3.072745569254064, |
| "eval_entropy": 2.8243270517127983, |
| "eval_loss": 2.126833915710449, |
| "eval_mean_token_accuracy": 0.4728344847516112, |
| "eval_nll": 2.1298649279065187, |
| "eval_num_tokens": 38902413.0, |
| "eval_ppl": 8.41373027452248, |
| "eval_runtime": 318.4023, |
| "eval_samples_per_second": 570.376, |
| "eval_steps_per_second": 8.913, |
| "step": 250 |
| }, |
| { |
| "entropy": 2.9943911642269376, |
| "epoch": 0.10767845256274718, |
| "grad_norm": 0.6328125, |
| "learning_rate": 0.0004955450810093227, |
| "loss": 2.189352722167969, |
| "mean_token_accuracy": 0.4613144874232056, |
| "num_tokens": 42787954.0, |
| "step": 275 |
| }, |
| { |
| "entropy": 2.9854789787582336, |
| "epoch": 0.1174674027957242, |
| "grad_norm": 0.64453125, |
| "learning_rate": 0.0004938955715544644, |
| "loss": 2.197265625, |
| "mean_token_accuracy": 0.4586064115776618, |
| "num_tokens": 46649640.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 2.967871895537734, |
| "epoch": 0.1272563530287012, |
| "grad_norm": 0.77734375, |
| "learning_rate": 0.0004919904601555969, |
| "loss": 2.1636979675292967, |
| "mean_token_accuracy": 0.46279846221708726, |
| "num_tokens": 50562181.0, |
| "step": 325 |
| }, |
| { |
| "entropy": 2.9346317600010297, |
| "epoch": 0.13704530326167821, |
| "grad_norm": 0.66796875, |
| "learning_rate": 0.0004898317433646626, |
| "loss": 2.1643003845214843, |
| "mean_token_accuracy": 0.4652492077421426, |
| "num_tokens": 54435712.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 2.8769926588271595, |
| "epoch": 0.14683425349465523, |
| "grad_norm": 0.81640625, |
| "learning_rate": 0.00048742168351141525, |
| "loss": 2.1070077514648435, |
| "mean_token_accuracy": 0.4735955714097171, |
| "num_tokens": 58331515.0, |
| "step": 375 |
| }, |
| { |
| "entropy": 2.9047266886544443, |
| "epoch": 0.15662320372763225, |
| "grad_norm": 0.58984375, |
| "learning_rate": 0.00048476280633250404, |
| "loss": 2.1295965576171874, |
| "mean_token_accuracy": 0.4706730742865917, |
| "num_tokens": 62245696.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 2.8752667624718105, |
| "epoch": 0.16641215396060927, |
| "grad_norm": 0.58203125, |
| "learning_rate": 0.00048185789832450813, |
| "loss": 2.11253173828125, |
| "mean_token_accuracy": 0.4743365019853278, |
| "num_tokens": 66145585.0, |
| "step": 425 |
| }, |
| { |
| "entropy": 2.8552429066577076, |
| "epoch": 0.17620110419358628, |
| "grad_norm": 0.59375, |
| "learning_rate": 0.00047871000382369524, |
| "loss": 2.0846556091308592, |
| "mean_token_accuracy": 0.47697299871989524, |
| "num_tokens": 70063214.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 2.8341120851783472, |
| "epoch": 0.1859900544265633, |
| "grad_norm": 0.58203125, |
| "learning_rate": 0.0004753224218155648, |
| "loss": 2.068045349121094, |
| "mean_token_accuracy": 0.47787787454415265, |
| "num_tokens": 73984544.0, |
| "step": 475 |
| }, |
| { |
| "entropy": 2.8375192761847376, |
| "epoch": 0.19577900465954032, |
| "grad_norm": 0.61328125, |
| "learning_rate": 0.0004716987024775197, |
| "loss": 2.0847515869140625, |
| "mean_token_accuracy": 0.47784874690977935, |
| "num_tokens": 77862885.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.19577900465954032, |
| "eval_bpd": 2.9090075702135, |
| "eval_entropy": 2.6186967416048157, |
| "eval_loss": 2.013190507888794, |
| "eval_mean_token_accuracy": 0.48935668935428794, |
| "eval_nll": 2.0163703955210246, |
| "eval_num_tokens": 77862885.0, |
| "eval_ppl": 7.5110133907515495, |
| "eval_runtime": 318.4358, |
| "eval_samples_per_second": 570.316, |
| "eval_steps_per_second": 8.912, |
| "step": 500 |
| }, |
| { |
| "entropy": 2.86442423287629, |
| "epoch": 0.20556795489251733, |
| "grad_norm": 0.6796875, |
| "learning_rate": 0.000467842643458289, |
| "loss": 2.097323455810547, |
| "mean_token_accuracy": 0.4766681852583933, |
| "num_tokens": 81738952.0, |
| "step": 525 |
| }, |
| { |
| "entropy": 2.8243706708841483, |
| "epoch": 0.21535690512549435, |
| "grad_norm": 0.6328125, |
| "learning_rate": 0.0004637582858980017, |
| "loss": 2.050118865966797, |
| "mean_token_accuracy": 0.48122489244307076, |
| "num_tokens": 85626497.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 2.848133797467131, |
| "epoch": 0.22514585535847137, |
| "grad_norm": 0.84375, |
| "learning_rate": 0.0004594499101930825, |
| "loss": 2.0830853271484373, |
| "mean_token_accuracy": 0.47580645161290325, |
| "num_tokens": 89484956.0, |
| "step": 575 |
| }, |
| { |
| "entropy": 2.8410350923085876, |
| "epoch": 0.2349348055914484, |
| "grad_norm": 0.52734375, |
| "learning_rate": 0.00045492203151040655, |
| "loss": 2.0800726318359377, |
| "mean_token_accuracy": 0.47742178446222894, |
| "num_tokens": 93413171.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 2.7831898042795857, |
| "epoch": 0.24472375582442538, |
| "grad_norm": 0.5859375, |
| "learning_rate": 0.00045017939505541626, |
| "loss": 2.0370738220214846, |
| "mean_token_accuracy": 0.4864190419562228, |
| "num_tokens": 97298042.0, |
| "step": 625 |
| }, |
| { |
| "entropy": 2.8064169968349897, |
| "epoch": 0.2545127060574024, |
| "grad_norm": 0.5546875, |
| "learning_rate": 0.00044522697109915813, |
| "loss": 2.061403503417969, |
| "mean_token_accuracy": 0.48147006407999693, |
| "num_tokens": 101194607.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 2.7893201624619977, |
| "epoch": 0.26430165629037944, |
| "grad_norm": 0.5234375, |
| "learning_rate": 0.0004400699497694506, |
| "loss": 2.0294932556152343, |
| "mean_token_accuracy": 0.48583452555693873, |
| "num_tokens": 105105248.0, |
| "step": 675 |
| }, |
| { |
| "entropy": 2.766137535649265, |
| "epoch": 0.27409060652335643, |
| "grad_norm": 0.61328125, |
| "learning_rate": 0.00043471373561164396, |
| "loss": 2.0185002136230468, |
| "mean_token_accuracy": 0.48713530665141636, |
| "num_tokens": 108992058.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 2.7472840214584604, |
| "epoch": 0.2838795567563335, |
| "grad_norm": 0.61328125, |
| "learning_rate": 0.00042916394192466997, |
| "loss": 2.007948150634766, |
| "mean_token_accuracy": 0.49140476584755016, |
| "num_tokens": 112863096.0, |
| "step": 725 |
| }, |
| { |
| "entropy": 2.8039981412212533, |
| "epoch": 0.29366850698931046, |
| "grad_norm": 0.60546875, |
| "learning_rate": 0.00042342638487831874, |
| "loss": 2.0519680786132812, |
| "mean_token_accuracy": 0.4848400037316914, |
| "num_tokens": 116741714.0, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.29366850698931046, |
| "eval_bpd": 2.8368191124508524, |
| "eval_entropy": 2.5129475370871655, |
| "eval_loss": 1.963122010231018, |
| "eval_mean_token_accuracy": 0.49759134340714123, |
| "eval_nll": 1.9663331695538748, |
| "eval_num_tokens": 116741714.0, |
| "eval_ppl": 7.144430981798381, |
| "eval_runtime": 318.4485, |
| "eval_samples_per_second": 570.293, |
| "eval_steps_per_second": 8.912, |
| "step": 750 |
| }, |
| { |
| "entropy": 2.7818116819998244, |
| "epoch": 0.3034574572222875, |
| "grad_norm": 0.60546875, |
| "learning_rate": 0.0004175070774179078, |
| "loss": 2.0355569458007814, |
| "mean_token_accuracy": 0.4845890820401234, |
| "num_tokens": 120630485.0, |
| "step": 775 |
| }, |
| { |
| "entropy": 2.754836296553474, |
| "epoch": 0.3132464074552645, |
| "grad_norm": 0.6328125, |
| "learning_rate": 0.0004114122229627302, |
| "loss": 2.026649627685547, |
| "mean_token_accuracy": 0.4883731104937326, |
| "num_tokens": 124539577.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 2.7313264609464514, |
| "epoch": 0.3230353576882415, |
| "grad_norm": 0.5546875, |
| "learning_rate": 0.00040514820890488664, |
| "loss": 1.9894279479980468, |
| "mean_token_accuracy": 0.493762371510729, |
| "num_tokens": 128444796.0, |
| "step": 825 |
| }, |
| { |
| "entropy": 2.799315380985199, |
| "epoch": 0.33282430792121853, |
| "grad_norm": 0.6171875, |
| "learning_rate": 0.00039872159991531533, |
| "loss": 2.04507568359375, |
| "mean_token_accuracy": 0.48317747648488935, |
| "num_tokens": 132371500.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 2.7418567548344273, |
| "epoch": 0.3426132581541955, |
| "grad_norm": 0.64453125, |
| "learning_rate": 0.0003921391310640325, |
| "loss": 2.0068865966796876, |
| "mean_token_accuracy": 0.4890909259070856, |
| "num_tokens": 136267287.0, |
| "step": 875 |
| }, |
| { |
| "entropy": 2.7740001842357076, |
| "epoch": 0.35240220838717257, |
| "grad_norm": 0.62109375, |
| "learning_rate": 0.0003854077007617969, |
| "loss": 2.022200164794922, |
| "mean_token_accuracy": 0.4868092625998601, |
| "num_tokens": 140138103.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 2.7364819494297907, |
| "epoch": 0.36219115862014956, |
| "grad_norm": 1.65625, |
| "learning_rate": 0.0003785343635305919, |
| "loss": 2.0004840087890625, |
| "mean_token_accuracy": 0.49300860279730147, |
| "num_tokens": 144053086.0, |
| "step": 925 |
| }, |
| { |
| "entropy": 2.739402972362386, |
| "epoch": 0.3719801088531266, |
| "grad_norm": 0.51953125, |
| "learning_rate": 0.00037152632261050483, |
| "loss": 1.9922245788574218, |
| "mean_token_accuracy": 0.4915943380171221, |
| "num_tokens": 147953420.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 2.7438382548220077, |
| "epoch": 0.3817690590861036, |
| "grad_norm": 0.7265625, |
| "learning_rate": 0.0003643909224107492, |
| "loss": 2.0027291870117185, |
| "mean_token_accuracy": 0.48921148393722236, |
| "num_tokens": 151836411.0, |
| "step": 975 |
| }, |
| { |
| "entropy": 2.7921177392917085, |
| "epoch": 0.39155800931908064, |
| "grad_norm": 0.6171875, |
| "learning_rate": 0.00035713564081274257, |
| "loss": 2.0273634338378907, |
| "mean_token_accuracy": 0.48521556448666425, |
| "num_tokens": 155738536.0, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.39155800931908064, |
| "eval_bpd": 2.798010089627336, |
| "eval_entropy": 2.536048009293489, |
| "eval_loss": 1.9362056255340576, |
| "eval_mean_token_accuracy": 0.5017364167555982, |
| "eval_nll": 1.9394328048034677, |
| "eval_num_tokens": 155738536.0, |
| "eval_ppl": 6.95480511965251, |
| "eval_runtime": 318.4763, |
| "eval_samples_per_second": 570.243, |
| "eval_steps_per_second": 8.911, |
| "step": 1000 |
| }, |
| { |
| "entropy": 2.6852813685402612, |
| "epoch": 0.4013469595520576, |
| "grad_norm": 0.66015625, |
| "learning_rate": 0.0003497680813333055, |
| "loss": 1.966862030029297, |
| "mean_token_accuracy": 0.49858123107216257, |
| "num_tokens": 159635561.0, |
| "step": 1025 |
| }, |
| { |
| "entropy": 2.7182392248732024, |
| "epoch": 0.41113590978503467, |
| "grad_norm": 0.61328125, |
| "learning_rate": 0.0003422959651561952, |
| "loss": 1.9684019470214844, |
| "mean_token_accuracy": 0.4934097538623164, |
| "num_tokens": 163523845.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 2.723624222111781, |
| "epoch": 0.42092486001801166, |
| "grad_norm": 0.5546875, |
| "learning_rate": 0.0003347271230403245, |
| "loss": 1.9813336181640624, |
| "mean_token_accuracy": 0.4927086445159934, |
| "num_tokens": 167422317.0, |
| "step": 1075 |
| }, |
| { |
| "entropy": 2.692510651594633, |
| "epoch": 0.4307138102509887, |
| "grad_norm": 0.60546875, |
| "learning_rate": 0.00032706948711314597, |
| "loss": 1.9508421325683594, |
| "mean_token_accuracy": 0.4979150942691247, |
| "num_tokens": 171329965.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 2.7370277544203048, |
| "epoch": 0.4405027604839657, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.0003193310825578028, |
| "loss": 2.004888458251953, |
| "mean_token_accuracy": 0.4905988301989983, |
| "num_tokens": 175222947.0, |
| "step": 1125 |
| }, |
| { |
| "entropy": 2.7431375323996496, |
| "epoch": 0.45029171071694274, |
| "grad_norm": 0.51953125, |
| "learning_rate": 0.0003115200192027566, |
| "loss": 2.0024537658691406, |
| "mean_token_accuracy": 0.4898783212043249, |
| "num_tokens": 179148056.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 2.7380168134624645, |
| "epoch": 0.46008066094991973, |
| "grad_norm": 0.53515625, |
| "learning_rate": 0.00030364448302270787, |
| "loss": 1.981593017578125, |
| "mean_token_accuracy": 0.49313135497616134, |
| "num_tokens": 183043792.0, |
| "step": 1175 |
| }, |
| { |
| "entropy": 2.7328614630792543, |
| "epoch": 0.4698696111828968, |
| "grad_norm": 0.54296875, |
| "learning_rate": 0.0002957127275597154, |
| "loss": 1.999188690185547, |
| "mean_token_accuracy": 0.49086318694960057, |
| "num_tokens": 186926264.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 2.7342380331600005, |
| "epoch": 0.47965856141587376, |
| "grad_norm": 0.55078125, |
| "learning_rate": 0.0002877330652735044, |
| "loss": 1.9958016967773438, |
| "mean_token_accuracy": 0.491330924542103, |
| "num_tokens": 190854133.0, |
| "step": 1225 |
| }, |
| { |
| "entropy": 2.729595283099829, |
| "epoch": 0.48944751164885075, |
| "grad_norm": 0.56640625, |
| "learning_rate": 0.0002797138588300303, |
| "loss": 1.9835462951660157, |
| "mean_token_accuracy": 0.4924394890660959, |
| "num_tokens": 194753804.0, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.48944751164885075, |
| "eval_bpd": 2.7761820901665817, |
| "eval_entropy": 2.496259016098447, |
| "eval_loss": 1.9210537672042847, |
| "eval_mean_token_accuracy": 0.5045342520232504, |
| "eval_nll": 1.924302788519982, |
| "eval_num_tokens": 194753804.0, |
| "eval_ppl": 6.8503708433220005, |
| "eval_runtime": 318.4425, |
| "eval_samples_per_second": 570.304, |
| "eval_steps_per_second": 8.912, |
| "step": 1250 |
| }, |
| { |
| "entropy": 2.716212966863494, |
| "epoch": 0.4992364618818278, |
| "grad_norm": 0.53515625, |
| "learning_rate": 0.0002716635123374255, |
| "loss": 1.9872402954101562, |
| "mean_token_accuracy": 0.4946632478001808, |
| "num_tokens": 198626081.0, |
| "step": 1275 |
| }, |
| { |
| "entropy": 2.710371494440595, |
| "epoch": 0.5090254121148048, |
| "grad_norm": 0.578125, |
| "learning_rate": 0.00026359046253851537, |
| "loss": 1.9923269653320312, |
| "mean_token_accuracy": 0.4937079610850272, |
| "num_tokens": 202510698.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 2.7343069936045317, |
| "epoch": 0.5188143623477818, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.00025550316996913314, |
| "loss": 1.994044647216797, |
| "mean_token_accuracy": 0.49244318068188636, |
| "num_tokens": 206412904.0, |
| "step": 1325 |
| }, |
| { |
| "entropy": 2.742348145241906, |
| "epoch": 0.5286033125807589, |
| "grad_norm": 0.5390625, |
| "learning_rate": 0.00024741011009149944, |
| "loss": 1.9998208618164062, |
| "mean_token_accuracy": 0.4908574945170533, |
| "num_tokens": 210328167.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 2.724076816799444, |
| "epoch": 0.5383922628137359, |
| "grad_norm": 0.828125, |
| "learning_rate": 0.0002393197644119594, |
| "loss": 1.9833689880371095, |
| "mean_token_accuracy": 0.4942413058436947, |
| "num_tokens": 214229484.0, |
| "step": 1375 |
| }, |
| { |
| "entropy": 2.703643675804064, |
| "epoch": 0.5481812130467129, |
| "grad_norm": 0.53515625, |
| "learning_rate": 0.00023124061159238553, |
| "loss": 1.9649853515625, |
| "mean_token_accuracy": 0.49494697581620767, |
| "num_tokens": 218112413.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 2.676096756869268, |
| "epoch": 0.5579701632796898, |
| "grad_norm": 0.5546875, |
| "learning_rate": 0.0002231811185645613, |
| "loss": 1.9395463562011719, |
| "mean_token_accuracy": 0.49995920531962634, |
| "num_tokens": 222029177.0, |
| "step": 1425 |
| }, |
| { |
| "entropy": 2.7083076951825085, |
| "epoch": 0.567759113512667, |
| "grad_norm": 0.54296875, |
| "learning_rate": 0.00021514973165685783, |
| "loss": 1.9707827758789063, |
| "mean_token_accuracy": 0.49421519689406546, |
| "num_tokens": 225922154.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 2.6908386128611452, |
| "epoch": 0.5775480637456439, |
| "grad_norm": 0.5625, |
| "learning_rate": 0.00020715486774250342, |
| "loss": 1.9708702087402343, |
| "mean_token_accuracy": 0.49710921564991656, |
| "num_tokens": 229803415.0, |
| "step": 1475 |
| }, |
| { |
| "entropy": 2.6993993439695094, |
| "epoch": 0.5873370139786209, |
| "grad_norm": 0.5703125, |
| "learning_rate": 0.0001992049054187214, |
| "loss": 1.9545468139648436, |
| "mean_token_accuracy": 0.4976514485983912, |
| "num_tokens": 233716105.0, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.5873370139786209, |
| "eval_bpd": 2.7561543934923574, |
| "eval_entropy": 2.5443466803816968, |
| "eval_loss": 1.9071590900421143, |
| "eval_mean_token_accuracy": 0.5068238914979266, |
| "eval_nll": 1.9104206470371334, |
| "eval_num_tokens": 233716105.0, |
| "eval_ppl": 6.755930062868705, |
| "eval_runtime": 318.4228, |
| "eval_samples_per_second": 570.339, |
| "eval_steps_per_second": 8.913, |
| "step": 1500 |
| }, |
| { |
| "entropy": 2.6966697504184745, |
| "epoch": 0.5971259642115979, |
| "grad_norm": 0.578125, |
| "learning_rate": 0.00019130817622598124, |
| "loss": 1.9578515625, |
| "mean_token_accuracy": 0.4964109645637419, |
| "num_tokens": 237600620.0, |
| "step": 1525 |
| }, |
| { |
| "entropy": 2.7033514475924334, |
| "epoch": 0.606914914444575, |
| "grad_norm": 0.56640625, |
| "learning_rate": 0.0001834729559165655, |
| "loss": 1.9643101501464844, |
| "mean_token_accuracy": 0.49634019273442664, |
| "num_tokens": 241485071.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 2.717692422407934, |
| "epoch": 0.616703864677552, |
| "grad_norm": 0.52734375, |
| "learning_rate": 0.00017570745578160198, |
| "loss": 1.9810302734375, |
| "mean_token_accuracy": 0.49517153960338245, |
| "num_tokens": 245377062.0, |
| "step": 1575 |
| }, |
| { |
| "entropy": 2.725204793410659, |
| "epoch": 0.626492814910529, |
| "grad_norm": 0.515625, |
| "learning_rate": 0.0001680198140456516, |
| "loss": 1.9905001831054687, |
| "mean_token_accuracy": 0.4927978377032189, |
| "num_tokens": 249268102.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 2.7151895718309267, |
| "epoch": 0.636281765143506, |
| "grad_norm": 0.5703125, |
| "learning_rate": 0.00016041808733786915, |
| "loss": 1.9757655334472657, |
| "mean_token_accuracy": 0.49435293814632136, |
| "num_tokens": 253169575.0, |
| "step": 1625 |
| }, |
| { |
| "entropy": 2.684670246552436, |
| "epoch": 0.646070715376483, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.0001529102422486769, |
| "loss": 1.95475830078125, |
| "mean_token_accuracy": 0.4991906837479353, |
| "num_tokens": 257058288.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 2.7151397740572563, |
| "epoch": 0.6558596656094601, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.00014550414698079727, |
| "loss": 1.9787599182128905, |
| "mean_token_accuracy": 0.4948471062821136, |
| "num_tokens": 260965925.0, |
| "step": 1675 |
| }, |
| { |
| "entropy": 2.6747805638197337, |
| "epoch": 0.6656486158424371, |
| "grad_norm": 0.52734375, |
| "learning_rate": 0.00013820756310339665, |
| "loss": 1.945904998779297, |
| "mean_token_accuracy": 0.5007789977758781, |
| "num_tokens": 264875335.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 2.6830354690355294, |
| "epoch": 0.675437566075414, |
| "grad_norm": 0.53125, |
| "learning_rate": 0.00013102813741798025, |
| "loss": 1.9539076232910155, |
| "mean_token_accuracy": 0.49903066554834535, |
| "num_tokens": 268791461.0, |
| "step": 1725 |
| }, |
| { |
| "entropy": 2.662765131871894, |
| "epoch": 0.685226516308391, |
| "grad_norm": 0.56640625, |
| "learning_rate": 0.00012397339394456356, |
| "loss": 1.950017852783203, |
| "mean_token_accuracy": 0.5014596502385913, |
| "num_tokens": 272652226.0, |
| "step": 1750 |
| }, |
| { |
| "epoch": 0.685226516308391, |
| "eval_bpd": 2.7488492664222206, |
| "eval_entropy": 2.520750623280978, |
| "eval_loss": 1.902093529701233, |
| "eval_mean_token_accuracy": 0.5076032224026725, |
| "eval_nll": 1.9053571188048362, |
| "eval_num_tokens": 272652226.0, |
| "eval_ppl": 6.721807682983077, |
| "eval_runtime": 318.3902, |
| "eval_samples_per_second": 570.398, |
| "eval_steps_per_second": 8.914, |
| "step": 1750 |
| }, |
| { |
| "entropy": 2.7304805326112556, |
| "epoch": 0.6950154665413681, |
| "grad_norm": 0.5703125, |
| "learning_rate": 0.00011705072603651829, |
| "loss": 1.9920945739746094, |
| "mean_token_accuracy": 0.4919112391490569, |
| "num_tokens": 276555213.0, |
| "step": 1775 |
| }, |
| { |
| "entropy": 2.6686812855730433, |
| "epoch": 0.7048044167743451, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.00011026738863235674, |
| "loss": 1.9485256958007813, |
| "mean_token_accuracy": 0.5003510564778191, |
| "num_tokens": 280483168.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 2.739967521815523, |
| "epoch": 0.7145933670073221, |
| "grad_norm": 0.5625, |
| "learning_rate": 0.00010363049065257416, |
| "loss": 1.9830743408203124, |
| "mean_token_accuracy": 0.4915797393574036, |
| "num_tokens": 284377873.0, |
| "step": 1825 |
| }, |
| { |
| "entropy": 2.698982752705625, |
| "epoch": 0.7243823172402991, |
| "grad_norm": 0.625, |
| "learning_rate": 9.71469875495186e-05, |
| "loss": 1.964124755859375, |
| "mean_token_accuracy": 0.496017001010034, |
| "num_tokens": 288260608.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 2.683373723573216, |
| "epoch": 0.7341712674732762, |
| "grad_norm": 0.5390625, |
| "learning_rate": 9.082367401809396e-05, |
| "loss": 1.9577156066894532, |
| "mean_token_accuracy": 0.4987992949988984, |
| "num_tokens": 292198541.0, |
| "step": 1875 |
| }, |
| { |
| "entropy": 2.706842296094011, |
| "epoch": 0.7439602177062532, |
| "grad_norm": 0.5546875, |
| "learning_rate": 8.466717687493703e-05, |
| "loss": 1.9568612670898438, |
| "mean_token_accuracy": 0.4959953935881719, |
| "num_tokens": 296110198.0, |
| "step": 1900 |
| }, |
| { |
| "entropy": 2.7134012857924437, |
| "epoch": 0.7537491679392302, |
| "grad_norm": 0.53515625, |
| "learning_rate": 7.868394811353039e-05, |
| "loss": 1.989944305419922, |
| "mean_token_accuracy": 0.49300837097963085, |
| "num_tokens": 299993061.0, |
| "step": 1925 |
| }, |
| { |
| "entropy": 2.6834044473523524, |
| "epoch": 0.7635381181722072, |
| "grad_norm": 0.5546875, |
| "learning_rate": 7.288025814252918e-05, |
| "loss": 1.952486572265625, |
| "mean_token_accuracy": 0.4983726410717931, |
| "num_tokens": 303894183.0, |
| "step": 1950 |
| }, |
| { |
| "entropy": 2.6719225988585014, |
| "epoch": 0.7733270684051843, |
| "grad_norm": 0.51953125, |
| "learning_rate": 6.726218921438915e-05, |
| "loss": 1.93992919921875, |
| "mean_token_accuracy": 0.4986390935453886, |
| "num_tokens": 307806569.0, |
| "step": 1975 |
| }, |
| { |
| "entropy": 2.6906488245972353, |
| "epoch": 0.7831160186381613, |
| "grad_norm": 0.57421875, |
| "learning_rate": 6.183562905118107e-05, |
| "loss": 1.958197021484375, |
| "mean_token_accuracy": 0.4967591651106469, |
| "num_tokens": 311727930.0, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.7831160186381613, |
| "eval_bpd": 2.7447081438067773, |
| "eval_entropy": 2.5224358858056153, |
| "eval_loss": 1.8992265462875366, |
| "eval_mean_token_accuracy": 0.5077641835222869, |
| "eval_nll": 1.9024867113395885, |
| "eval_num_tokens": 311727930.0, |
| "eval_ppl": 6.702541020843852, |
| "eval_runtime": 318.3972, |
| "eval_samples_per_second": 570.385, |
| "eval_steps_per_second": 8.913, |
| "step": 2000 |
| }, |
| { |
| "entropy": 2.692847571895764, |
| "epoch": 0.7929049688711383, |
| "grad_norm": 0.54296875, |
| "learning_rate": 5.660626467427307e-05, |
| "loss": 1.9653746032714843, |
| "mean_token_accuracy": 0.4960952429953253, |
| "num_tokens": 315619973.0, |
| "step": 2025 |
| }, |
| { |
| "entropy": 2.6994904209462374, |
| "epoch": 0.8026939191041152, |
| "grad_norm": 0.546875, |
| "learning_rate": 5.157957644434627e-05, |
| "loss": 1.9680723571777343, |
| "mean_token_accuracy": 0.49756041665126527, |
| "num_tokens": 319484828.0, |
| "step": 2050 |
| }, |
| { |
| "entropy": 2.68640734189637, |
| "epoch": 0.8124828693370922, |
| "grad_norm": 0.52734375, |
| "learning_rate": 4.676083231799086e-05, |
| "loss": 1.9520834350585938, |
| "mean_token_accuracy": 0.49859836666433366, |
| "num_tokens": 323423122.0, |
| "step": 2075 |
| }, |
| { |
| "entropy": 2.6634044602616287, |
| "epoch": 0.8222718195700693, |
| "grad_norm": 0.49609375, |
| "learning_rate": 4.2155082326900877e-05, |
| "loss": 1.9399433898925782, |
| "mean_token_accuracy": 0.5020899480781442, |
| "num_tokens": 327302804.0, |
| "step": 2100 |
| }, |
| { |
| "entropy": 2.6912057347946354, |
| "epoch": 0.8320607698030463, |
| "grad_norm": 0.68359375, |
| "learning_rate": 3.7767153285453686e-05, |
| "loss": 1.9653933715820313, |
| "mean_token_accuracy": 0.49717872825327036, |
| "num_tokens": 331218187.0, |
| "step": 2125 |
| }, |
| { |
| "entropy": 2.7046046517021276, |
| "epoch": 0.8418497200360233, |
| "grad_norm": 0.5390625, |
| "learning_rate": 3.360164373222063e-05, |
| "loss": 1.982100067138672, |
| "mean_token_accuracy": 0.4947454706720563, |
| "num_tokens": 335143921.0, |
| "step": 2150 |
| }, |
| { |
| "entropy": 2.6821387397394765, |
| "epoch": 0.8516386702690003, |
| "grad_norm": 0.494140625, |
| "learning_rate": 2.966291911071012e-05, |
| "loss": 1.9384609985351562, |
| "mean_token_accuracy": 0.4992011088945707, |
| "num_tokens": 338985633.0, |
| "step": 2175 |
| }, |
| { |
| "entropy": 2.714641275012099, |
| "epoch": 0.8614276205019774, |
| "grad_norm": 0.53515625, |
| "learning_rate": 2.5955107194394062e-05, |
| "loss": 1.9649354553222655, |
| "mean_token_accuracy": 0.4940774985352163, |
| "num_tokens": 342892576.0, |
| "step": 2200 |
| }, |
| { |
| "entropy": 2.6872711382347774, |
| "epoch": 0.8712165707349544, |
| "grad_norm": 0.53125, |
| "learning_rate": 2.2482093760811617e-05, |
| "loss": 1.962772674560547, |
| "mean_token_accuracy": 0.4993625307712092, |
| "num_tokens": 346835933.0, |
| "step": 2225 |
| }, |
| { |
| "entropy": 2.676342596904866, |
| "epoch": 0.8810055209679314, |
| "grad_norm": 0.6875, |
| "learning_rate": 1.9247518519283968e-05, |
| "loss": 1.9479315185546875, |
| "mean_token_accuracy": 0.4994813839214091, |
| "num_tokens": 350719859.0, |
| "step": 2250 |
| }, |
| { |
| "epoch": 0.8810055209679314, |
| "eval_bpd": 2.7441256628978783, |
| "eval_entropy": 2.5175174792400643, |
| "eval_loss": 1.8988220691680908, |
| "eval_mean_token_accuracy": 0.5081831367590252, |
| "eval_nll": 1.902082966339855, |
| "eval_num_tokens": 350719859.0, |
| "eval_ppl": 6.699835449638354, |
| "eval_runtime": 318.406, |
| "eval_samples_per_second": 570.369, |
| "eval_steps_per_second": 8.913, |
| "step": 2250 |
| }, |
| { |
| "entropy": 2.7144406457761727, |
| "epoch": 0.8907944712009084, |
| "grad_norm": 0.546875, |
| "learning_rate": 1.625477129650835e-05, |
| "loss": 1.9875895690917968, |
| "mean_token_accuracy": 0.49512781017346974, |
| "num_tokens": 354612028.0, |
| "step": 2275 |
| }, |
| { |
| "entropy": 2.7052265682160703, |
| "epoch": 0.9005834214338855, |
| "grad_norm": 0.51953125, |
| "learning_rate": 1.3506988484027926e-05, |
| "loss": 1.98323486328125, |
| "mean_token_accuracy": 0.4958426919140038, |
| "num_tokens": 358517938.0, |
| "step": 2300 |
| }, |
| { |
| "entropy": 2.683134757355532, |
| "epoch": 0.9103723716668625, |
| "grad_norm": 0.52734375, |
| "learning_rate": 1.1007049751301789e-05, |
| "loss": 1.9486451721191407, |
| "mean_token_accuracy": 0.4983769791012847, |
| "num_tokens": 362392490.0, |
| "step": 2325 |
| }, |
| { |
| "entropy": 2.6868362478560734, |
| "epoch": 0.9201613218998395, |
| "grad_norm": 0.498046875, |
| "learning_rate": 8.757575027818238e-06, |
| "loss": 1.9728414916992187, |
| "mean_token_accuracy": 0.4980474413039569, |
| "num_tokens": 366305769.0, |
| "step": 2350 |
| }, |
| { |
| "entropy": 2.7114968257715675, |
| "epoch": 0.9299502721328164, |
| "grad_norm": 0.53515625, |
| "learning_rate": 6.760921757415439e-06, |
| "loss": 1.9739515686035156, |
| "mean_token_accuracy": 0.495761617308943, |
| "num_tokens": 370210294.0, |
| "step": 2375 |
| }, |
| { |
| "entropy": 2.723082096025959, |
| "epoch": 0.9397392223657935, |
| "grad_norm": 0.55859375, |
| "learning_rate": 5.0191824276862585e-06, |
| "loss": 1.9711668395996094, |
| "mean_token_accuracy": 0.4947962255092054, |
| "num_tokens": 374129359.0, |
| "step": 2400 |
| }, |
| { |
| "entropy": 2.7243931290128818, |
| "epoch": 0.9495281725987705, |
| "grad_norm": 0.5703125, |
| "learning_rate": 3.5341823770564385e-06, |
| "loss": 1.9800309753417968, |
| "mean_token_accuracy": 0.494186617936668, |
| "num_tokens": 378038492.0, |
| "step": 2425 |
| }, |
| { |
| "entropy": 2.6882893116541458, |
| "epoch": 0.9593171228317475, |
| "grad_norm": 0.5703125, |
| "learning_rate": 2.307477881834663e-06, |
| "loss": 1.960792236328125, |
| "mean_token_accuracy": 0.49821483996751215, |
| "num_tokens": 381921956.0, |
| "step": 2450 |
| }, |
| { |
| "entropy": 2.7129936127462098, |
| "epoch": 0.9691060730647245, |
| "grad_norm": 0.51953125, |
| "learning_rate": 1.340354525238785e-06, |
| "loss": 1.9839578247070313, |
| "mean_token_accuracy": 0.4944347484579711, |
| "num_tokens": 385800276.0, |
| "step": 2475 |
| }, |
| { |
| "entropy": 2.687453053126722, |
| "epoch": 0.9788950232977015, |
| "grad_norm": 0.54296875, |
| "learning_rate": 6.33825850107933e-07, |
| "loss": 1.9629275512695312, |
| "mean_token_accuracy": 0.49751757115833734, |
| "num_tokens": 389694012.0, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.9788950232977015, |
| "eval_bpd": 2.7439658028059983, |
| "eval_entropy": 2.518428092528893, |
| "eval_loss": 1.8987137079238892, |
| "eval_mean_token_accuracy": 0.5082029323805907, |
| "eval_nll": 1.9019721597678847, |
| "eval_num_tokens": 389694012.0, |
| "eval_ppl": 6.699093104968507, |
| "eval_runtime": 318.3733, |
| "eval_samples_per_second": 570.428, |
| "eval_steps_per_second": 8.914, |
| "step": 2500 |
| }, |
| { |
| "entropy": 2.699357015282978, |
| "epoch": 0.9886839735306786, |
| "grad_norm": 0.5546875, |
| "learning_rate": 1.88632296712149e-07, |
| "loss": 1.9804957580566407, |
| "mean_token_accuracy": 0.495278594895215, |
| "num_tokens": 393590178.0, |
| "step": 2525 |
| }, |
| { |
| "entropy": 2.7129794216862817, |
| "epoch": 0.9984729237636556, |
| "grad_norm": 0.5546875, |
| "learning_rate": 5.240426772828988e-09, |
| "loss": 1.9630178833007812, |
| "mean_token_accuracy": 0.4952611821264031, |
| "num_tokens": 397490330.0, |
| "step": 2550 |
| }, |
| { |
| "epoch": 1.0, |
| "eval_bpd": 2.7439666464510006, |
| "eval_entropy": 2.5184869402462216, |
| "eval_loss": 1.8987144231796265, |
| "eval_mean_token_accuracy": 0.5081928723106147, |
| "eval_nll": 1.9019727445380394, |
| "eval_num_tokens": 398097939.0, |
| "eval_ppl": 6.699097022399364, |
| "eval_runtime": 318.2646, |
| "eval_samples_per_second": 570.623, |
| "eval_steps_per_second": 8.917, |
| "step": 2554 |
| } |
| ], |
| "logging_steps": 25, |
| "max_steps": 2554, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.2931484417361754e+18, |
| "train_batch_size": 64, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
