| { |
| "best_global_step": 2250, |
| "best_metric": 1.8998193594895059, |
| "best_model_checkpoint": "/content/cot-split2/checkpoint-2000", |
| "epoch": 2.0, |
| "eval_steps": 250, |
| "global_step": 2554, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0, |
| "eval_bpd": 5.449506205976702, |
| "eval_entropy": 3.8259646681880883, |
| "eval_loss": 3.7819511890411377, |
| "eval_mean_token_accuracy": 0.3298281561563004, |
| "eval_nll": 3.777309862116675, |
| "eval_num_tokens": 0.0, |
| "eval_ppl": 43.69832894461198, |
| "eval_runtime": 318.4675, |
| "eval_samples_per_second": 570.259, |
| "eval_steps_per_second": 8.911, |
| "step": 0 |
| }, |
| { |
| "entropy": 4.035161377492861, |
| "epoch": 0.0007831466833737959, |
| "grad_norm": 2.9375, |
| "learning_rate": 0.0, |
| "loss": 3.6375203132629395, |
| "mean_token_accuracy": 0.3242907148120855, |
| "num_tokens": 156431.0, |
| "step": 1 |
| }, |
| { |
| "entropy": 4.157279884560115, |
| "epoch": 0.019578667084344897, |
| "grad_norm": 1.8515625, |
| "learning_rate": 9.375e-05, |
| "loss": 3.7293694814046225, |
| "mean_token_accuracy": 0.3223868495477843, |
| "num_tokens": 3882398.0, |
| "step": 25 |
| }, |
| { |
| "entropy": 4.069613211207327, |
| "epoch": 0.03915733416868979, |
| "grad_norm": 6.65625, |
| "learning_rate": 0.00019140625, |
| "loss": 3.286188659667969, |
| "mean_token_accuracy": 0.3470086962011918, |
| "num_tokens": 7771579.0, |
| "step": 50 |
| }, |
| { |
| "entropy": 3.712915897718112, |
| "epoch": 0.05873600125303469, |
| "grad_norm": 1.078125, |
| "learning_rate": 0.0002890625, |
| "loss": 2.9255511474609377, |
| "mean_token_accuracy": 0.37689020381328076, |
| "num_tokens": 11677887.0, |
| "step": 75 |
| }, |
| { |
| "entropy": 3.4658741959475385, |
| "epoch": 0.07831466833737959, |
| "grad_norm": 1.671875, |
| "learning_rate": 0.00038671875, |
| "loss": 2.5379095458984375, |
| "mean_token_accuracy": 0.41490777290923164, |
| "num_tokens": 15584019.0, |
| "step": 100 |
| }, |
| { |
| "entropy": 3.295554994185651, |
| "epoch": 0.09789333542172449, |
| "grad_norm": 0.59765625, |
| "learning_rate": 0.000484375, |
| "loss": 2.405628204345703, |
| "mean_token_accuracy": 0.4313112262648087, |
| "num_tokens": 19484337.0, |
| "step": 125 |
| }, |
| { |
| "entropy": 3.1617948360015533, |
| "epoch": 0.11747200250606939, |
| "grad_norm": 0.80078125, |
| "learning_rate": 0.0004999075642455791, |
| "loss": 2.3207965087890625, |
| "mean_token_accuracy": 0.4436994497174724, |
| "num_tokens": 23377294.0, |
| "step": 150 |
| }, |
| { |
| "entropy": 3.0947463825081942, |
| "epoch": 0.1370506695904143, |
| "grad_norm": 0.6640625, |
| "learning_rate": 0.0004995565798713207, |
| "loss": 2.2583859252929686, |
| "mean_token_accuracy": 0.45017372579314874, |
| "num_tokens": 27287783.0, |
| "step": 175 |
| }, |
| { |
| "entropy": 3.045135160417696, |
| "epoch": 0.15662933667475917, |
| "grad_norm": 0.65625, |
| "learning_rate": 0.0004989440608306114, |
| "loss": 2.230653076171875, |
| "mean_token_accuracy": 0.4551180636124264, |
| "num_tokens": 31205249.0, |
| "step": 200 |
| }, |
| { |
| "entropy": 3.0288844077617805, |
| "epoch": 0.1762080037591041, |
| "grad_norm": 0.625, |
| "learning_rate": 0.0004980706490418613, |
| "loss": 2.214619140625, |
| "mean_token_accuracy": 0.4571791829436394, |
| "num_tokens": 35081817.0, |
| "step": 225 |
| }, |
| { |
| "entropy": 2.9857863094922594, |
| "epoch": 0.19578667084344897, |
| "grad_norm": 0.6015625, |
| "learning_rate": 0.0004969372598384225, |
| "loss": 2.183477783203125, |
| "mean_token_accuracy": 0.4623076980301137, |
| "num_tokens": 38961791.0, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.19578667084344897, |
| "eval_bpd": 3.0542670212451286, |
| "eval_entropy": 2.831413693293467, |
| "eval_loss": 2.114036798477173, |
| "eval_mean_token_accuracy": 0.471896302096762, |
| "eval_nll": 2.1170565744532834, |
| "eval_num_tokens": 38961791.0, |
| "eval_ppl": 8.306651458853036, |
| "eval_runtime": 317.7783, |
| "eval_samples_per_second": 571.496, |
| "eval_steps_per_second": 8.931, |
| "step": 250 |
| }, |
| { |
| "entropy": 2.960300336004756, |
| "epoch": 0.2153653379277939, |
| "grad_norm": 0.59765625, |
| "learning_rate": 0.0004955450810093227, |
| "loss": 2.1587040710449217, |
| "mean_token_accuracy": 0.46429396638467557, |
| "num_tokens": 42868157.0, |
| "step": 275 |
| }, |
| { |
| "entropy": 2.9496937473585083, |
| "epoch": 0.23494400501213877, |
| "grad_norm": 0.6953125, |
| "learning_rate": 0.0004938955715544644, |
| "loss": 2.1702507019042967, |
| "mean_token_accuracy": 0.4644734879386833, |
| "num_tokens": 46774567.0, |
| "step": 300 |
| }, |
| { |
| "entropy": 2.925756508647974, |
| "epoch": 0.25452267209648366, |
| "grad_norm": 0.61328125, |
| "learning_rate": 0.0004919904601555969, |
| "loss": 2.1320841979980467, |
| "mean_token_accuracy": 0.4678374269200783, |
| "num_tokens": 50651509.0, |
| "step": 325 |
| }, |
| { |
| "entropy": 2.916845726644516, |
| "epoch": 0.2741013391808286, |
| "grad_norm": 0.60546875, |
| "learning_rate": 0.0004898317433646626, |
| "loss": 2.149468994140625, |
| "mean_token_accuracy": 0.46948018886239345, |
| "num_tokens": 54545113.0, |
| "step": 350 |
| }, |
| { |
| "entropy": 2.8672960236041836, |
| "epoch": 0.2936800062651735, |
| "grad_norm": 0.59765625, |
| "learning_rate": 0.00048742168351141525, |
| "loss": 2.095373077392578, |
| "mean_token_accuracy": 0.47492304637997923, |
| "num_tokens": 58450259.0, |
| "step": 375 |
| }, |
| { |
| "entropy": 2.866450867223391, |
| "epoch": 0.31325867334951835, |
| "grad_norm": 0.609375, |
| "learning_rate": 0.00048476280633250404, |
| "loss": 2.093819122314453, |
| "mean_token_accuracy": 0.4762016071737522, |
| "num_tokens": 62334830.0, |
| "step": 400 |
| }, |
| { |
| "entropy": 2.8444264384733913, |
| "epoch": 0.33283734043386326, |
| "grad_norm": 0.671875, |
| "learning_rate": 0.00048185789832450813, |
| "loss": 2.07142822265625, |
| "mean_token_accuracy": 0.47914267030296726, |
| "num_tokens": 66254709.0, |
| "step": 425 |
| }, |
| { |
| "entropy": 2.8643093723566215, |
| "epoch": 0.3524160075182082, |
| "grad_norm": 0.578125, |
| "learning_rate": 0.00047871000382369524, |
| "loss": 2.0907044982910157, |
| "mean_token_accuracy": 0.475262880935016, |
| "num_tokens": 70149264.0, |
| "step": 450 |
| }, |
| { |
| "entropy": 2.8425666356253885, |
| "epoch": 0.37199467460255303, |
| "grad_norm": 1.453125, |
| "learning_rate": 0.0004753224218155648, |
| "loss": 2.0714340209960938, |
| "mean_token_accuracy": 0.47639239522332366, |
| "num_tokens": 74036195.0, |
| "step": 475 |
| }, |
| { |
| "entropy": 2.8495917513303857, |
| "epoch": 0.39157334168689795, |
| "grad_norm": 0.58203125, |
| "learning_rate": 0.0004716987024775197, |
| "loss": 2.0957228088378907, |
| "mean_token_accuracy": 0.4775797151536097, |
| "num_tokens": 77964744.0, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.39157334168689795, |
| "eval_bpd": 2.9013035889897933, |
| "eval_entropy": 2.6662824092598547, |
| "eval_loss": 2.007856607437134, |
| "eval_mean_token_accuracy": 0.48950937525505117, |
| "eval_nll": 2.0110304026567256, |
| "eval_num_tokens": 77964744.0, |
| "eval_ppl": 7.471011532715633, |
| "eval_runtime": 317.7246, |
| "eval_samples_per_second": 571.593, |
| "eval_steps_per_second": 8.932, |
| "step": 500 |
| }, |
| { |
| "entropy": 2.8462999362059764, |
| "epoch": 0.41115200877124286, |
| "grad_norm": 0.59765625, |
| "learning_rate": 0.000467842643458289, |
| "loss": 2.0840892028808593, |
| "mean_token_accuracy": 0.47706405281698494, |
| "num_tokens": 81861726.0, |
| "step": 525 |
| }, |
| { |
| "entropy": 2.8082084615239284, |
| "epoch": 0.4307306758555878, |
| "grad_norm": 0.62109375, |
| "learning_rate": 0.0004637582858980017, |
| "loss": 2.041808624267578, |
| "mean_token_accuracy": 0.4826572669515401, |
| "num_tokens": 85760943.0, |
| "step": 550 |
| }, |
| { |
| "entropy": 2.8388380295514866, |
| "epoch": 0.45030934293993263, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.0004594499101930825, |
| "loss": 2.078563385009766, |
| "mean_token_accuracy": 0.4769605679393806, |
| "num_tokens": 89653726.0, |
| "step": 575 |
| }, |
| { |
| "entropy": 2.8302025972271743, |
| "epoch": 0.46988801002427755, |
| "grad_norm": 0.578125, |
| "learning_rate": 0.00045492203151040655, |
| "loss": 2.0540911865234377, |
| "mean_token_accuracy": 0.478844059310469, |
| "num_tokens": 93570215.0, |
| "step": 600 |
| }, |
| { |
| "entropy": 2.792682902628425, |
| "epoch": 0.48946667710862246, |
| "grad_norm": 0.69140625, |
| "learning_rate": 0.00045017939505541626, |
| "loss": 2.0480902099609377, |
| "mean_token_accuracy": 0.4848035732412953, |
| "num_tokens": 97477693.0, |
| "step": 625 |
| }, |
| { |
| "entropy": 2.7777595483840716, |
| "epoch": 0.5090453441929673, |
| "grad_norm": 0.66796875, |
| "learning_rate": 0.00044522697109915813, |
| "loss": 2.0343661499023438, |
| "mean_token_accuracy": 0.48608020488521336, |
| "num_tokens": 101394752.0, |
| "step": 650 |
| }, |
| { |
| "entropy": 2.742625050496606, |
| "epoch": 0.5286240112773123, |
| "grad_norm": 0.65625, |
| "learning_rate": 0.0004400699497694506, |
| "loss": 2.0052694702148437, |
| "mean_token_accuracy": 0.4903364669038516, |
| "num_tokens": 105270418.0, |
| "step": 675 |
| }, |
| { |
| "entropy": 2.8160673456556586, |
| "epoch": 0.5482026783616571, |
| "grad_norm": 0.65234375, |
| "learning_rate": 0.00043471373561164396, |
| "loss": 2.0440855407714844, |
| "mean_token_accuracy": 0.4817835272308287, |
| "num_tokens": 109173756.0, |
| "step": 700 |
| }, |
| { |
| "entropy": 2.731675431270922, |
| "epoch": 0.567781345446002, |
| "grad_norm": 0.625, |
| "learning_rate": 0.00042916394192466997, |
| "loss": 1.9994474792480468, |
| "mean_token_accuracy": 0.49231067122440925, |
| "num_tokens": 113075567.0, |
| "step": 725 |
| }, |
| { |
| "entropy": 2.761901786283376, |
| "epoch": 0.587360012530347, |
| "grad_norm": 0.5859375, |
| "learning_rate": 0.00042342638487831874, |
| "loss": 2.0215072631835938, |
| "mean_token_accuracy": 0.48973995306839846, |
| "num_tokens": 116976712.0, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.587360012530347, |
| "eval_bpd": 2.8346400005221386, |
| "eval_entropy": 2.564649322232512, |
| "eval_loss": 1.9616204500198364, |
| "eval_mean_token_accuracy": 0.4976015657363103, |
| "eval_nll": 1.964822724264362, |
| "eval_num_tokens": 116976712.0, |
| "eval_ppl": 7.133647855387014, |
| "eval_runtime": 317.7915, |
| "eval_samples_per_second": 571.472, |
| "eval_steps_per_second": 8.93, |
| "step": 750 |
| }, |
| { |
| "entropy": 2.772676199080625, |
| "epoch": 0.6069386796146918, |
| "grad_norm": 0.6484375, |
| "learning_rate": 0.0004175070774179078, |
| "loss": 2.0193243408203125, |
| "mean_token_accuracy": 0.4871398110581337, |
| "num_tokens": 120856970.0, |
| "step": 775 |
| }, |
| { |
| "entropy": 2.7522809489804128, |
| "epoch": 0.6265173466990367, |
| "grad_norm": 0.51953125, |
| "learning_rate": 0.0004114122229627302, |
| "loss": 2.0201119995117187, |
| "mean_token_accuracy": 0.4884969423881558, |
| "num_tokens": 124747012.0, |
| "step": 800 |
| }, |
| { |
| "entropy": 2.728323356258852, |
| "epoch": 0.6460960137833817, |
| "grad_norm": 0.59765625, |
| "learning_rate": 0.00040514820890488664, |
| "loss": 2.000582427978516, |
| "mean_token_accuracy": 0.49145337661979344, |
| "num_tokens": 128633902.0, |
| "step": 825 |
| }, |
| { |
| "entropy": 2.7372595225009375, |
| "epoch": 0.6656746808677265, |
| "grad_norm": 0.59765625, |
| "learning_rate": 0.00039872159991531533, |
| "loss": 2.00209716796875, |
| "mean_token_accuracy": 0.4912540972003919, |
| "num_tokens": 132518194.0, |
| "step": 850 |
| }, |
| { |
| "entropy": 2.738841588033162, |
| "epoch": 0.6852533479520714, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.0003921391310640325, |
| "loss": 1.994276580810547, |
| "mean_token_accuracy": 0.491252567341933, |
| "num_tokens": 136385199.0, |
| "step": 875 |
| }, |
| { |
| "entropy": 2.7295696415657966, |
| "epoch": 0.7048320150364163, |
| "grad_norm": 0.53515625, |
| "learning_rate": 0.0003854077007617969, |
| "loss": 1.9865512084960937, |
| "mean_token_accuracy": 0.49203635280876573, |
| "num_tokens": 140293961.0, |
| "step": 900 |
| }, |
| { |
| "entropy": 2.7351411749253978, |
| "epoch": 0.7244106821207612, |
| "grad_norm": 0.5703125, |
| "learning_rate": 0.0003785343635305919, |
| "loss": 1.9895498657226562, |
| "mean_token_accuracy": 0.49209919251085, |
| "num_tokens": 144186204.0, |
| "step": 925 |
| }, |
| { |
| "entropy": 2.7418933621077652, |
| "epoch": 0.7439893492051061, |
| "grad_norm": 0.63671875, |
| "learning_rate": 0.00037152632261050483, |
| "loss": 2.0013661193847656, |
| "mean_token_accuracy": 0.49214851600044945, |
| "num_tokens": 148108105.0, |
| "step": 950 |
| }, |
| { |
| "entropy": 2.7243121079421706, |
| "epoch": 0.763568016289451, |
| "grad_norm": 0.640625, |
| "learning_rate": 0.0003643909224107492, |
| "loss": 1.991259307861328, |
| "mean_token_accuracy": 0.4933121377369521, |
| "num_tokens": 152015615.0, |
| "step": 975 |
| }, |
| { |
| "entropy": 2.778145840960037, |
| "epoch": 0.7831466833737959, |
| "grad_norm": 0.58984375, |
| "learning_rate": 0.00035713564081274257, |
| "loss": 2.0278318786621092, |
| "mean_token_accuracy": 0.48574800621666386, |
| "num_tokens": 155927990.0, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.7831466833737959, |
| "eval_bpd": 2.789343226908039, |
| "eval_entropy": 2.525867688934465, |
| "eval_loss": 1.9301713705062866, |
| "eval_mean_token_accuracy": 0.502722952650009, |
| "eval_nll": 1.9334253933452872, |
| "eval_num_tokens": 155927990.0, |
| "eval_ppl": 6.913149988716967, |
| "eval_runtime": 317.7734, |
| "eval_samples_per_second": 571.505, |
| "eval_steps_per_second": 8.931, |
| "step": 1000 |
| }, |
| { |
| "entropy": 2.6940799960317015, |
| "epoch": 0.8027253504581409, |
| "grad_norm": 0.73046875, |
| "learning_rate": 0.0003497680813333055, |
| "loss": 1.9498268127441407, |
| "mean_token_accuracy": 0.4984518737087283, |
| "num_tokens": 159830176.0, |
| "step": 1025 |
| }, |
| { |
| "entropy": 2.7343128778140633, |
| "epoch": 0.8223040175424857, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.0003422959651561952, |
| "loss": 1.9870970153808594, |
| "mean_token_accuracy": 0.4924939781360015, |
| "num_tokens": 163728360.0, |
| "step": 1050 |
| }, |
| { |
| "entropy": 2.7091974552364038, |
| "epoch": 0.8418826846268306, |
| "grad_norm": 0.546875, |
| "learning_rate": 0.0003347271230403245, |
| "loss": 1.9727362060546876, |
| "mean_token_accuracy": 0.49556485851019944, |
| "num_tokens": 167642672.0, |
| "step": 1075 |
| }, |
| { |
| "entropy": 2.7602979798373886, |
| "epoch": 0.8614613517111755, |
| "grad_norm": 0.6640625, |
| "learning_rate": 0.00032706948711314597, |
| "loss": 1.992530059814453, |
| "mean_token_accuracy": 0.490016912306627, |
| "num_tokens": 171525383.0, |
| "step": 1100 |
| }, |
| { |
| "entropy": 2.721992457947341, |
| "epoch": 0.8810400187955204, |
| "grad_norm": 0.57421875, |
| "learning_rate": 0.0003193310825578028, |
| "loss": 1.984290771484375, |
| "mean_token_accuracy": 0.4936226868736485, |
| "num_tokens": 175403307.0, |
| "step": 1125 |
| }, |
| { |
| "entropy": 2.739525592921455, |
| "epoch": 0.9006186858798653, |
| "grad_norm": 0.5078125, |
| "learning_rate": 0.0003115200192027566, |
| "loss": 2.005537261962891, |
| "mean_token_accuracy": 0.4910348970524133, |
| "num_tokens": 179313014.0, |
| "step": 1150 |
| }, |
| { |
| "entropy": 2.7139533455144047, |
| "epoch": 0.9201973529642102, |
| "grad_norm": 0.7578125, |
| "learning_rate": 0.00030364448302270787, |
| "loss": 1.9704147338867188, |
| "mean_token_accuracy": 0.4949843998242547, |
| "num_tokens": 183213621.0, |
| "step": 1175 |
| }, |
| { |
| "entropy": 2.7396354848939612, |
| "epoch": 0.9397760200485551, |
| "grad_norm": 0.5859375, |
| "learning_rate": 0.0002957127275597154, |
| "loss": 2.010960998535156, |
| "mean_token_accuracy": 0.4896595800467586, |
| "num_tokens": 187081439.0, |
| "step": 1200 |
| }, |
| { |
| "entropy": 2.768980867176604, |
| "epoch": 0.9593546871329, |
| "grad_norm": 0.5546875, |
| "learning_rate": 0.0002877330652735044, |
| "loss": 2.016462554931641, |
| "mean_token_accuracy": 0.48757137446833304, |
| "num_tokens": 190967002.0, |
| "step": 1225 |
| }, |
| { |
| "entropy": 2.704207567026802, |
| "epoch": 0.9789333542172449, |
| "grad_norm": 0.74609375, |
| "learning_rate": 0.0002797138588300303, |
| "loss": 1.9740122985839843, |
| "mean_token_accuracy": 0.4958090399193563, |
| "num_tokens": 194892883.0, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.9789333542172449, |
| "eval_bpd": 2.767466912988666, |
| "eval_entropy": 2.5821667777041397, |
| "eval_loss": 1.9150066375732422, |
| "eval_mean_token_accuracy": 0.5045345765416368, |
| "eval_nll": 1.9182618880310294, |
| "eval_num_tokens": 194892883.0, |
| "eval_ppl": 6.809113176939714, |
| "eval_runtime": 317.7422, |
| "eval_samples_per_second": 571.561, |
| "eval_steps_per_second": 8.932, |
| "step": 1250 |
| }, |
| { |
| "entropy": 2.6901174929114173, |
| "epoch": 0.9985120213015898, |
| "grad_norm": 0.609375, |
| "learning_rate": 0.0002716635123374255, |
| "loss": 1.9728475952148437, |
| "mean_token_accuracy": 0.49665255697471494, |
| "num_tokens": 198787951.0, |
| "step": 1275 |
| }, |
| { |
| "entropy": 2.7251224381449632, |
| "epoch": 1.0180123737175972, |
| "grad_norm": 0.56640625, |
| "learning_rate": 0.00026359046253851537, |
| "loss": 1.9876043701171875, |
| "mean_token_accuracy": 0.49355816180166034, |
| "num_tokens": 202648704.0, |
| "step": 1300 |
| }, |
| { |
| "entropy": 2.7282011647258444, |
| "epoch": 1.0375910408019422, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.00025550316996913314, |
| "loss": 1.9952455139160157, |
| "mean_token_accuracy": 0.4923797138191662, |
| "num_tokens": 206542240.0, |
| "step": 1325 |
| }, |
| { |
| "entropy": 2.7167434371623265, |
| "epoch": 1.0571697078862872, |
| "grad_norm": 0.6640625, |
| "learning_rate": 0.00024741011009149944, |
| "loss": 1.985081329345703, |
| "mean_token_accuracy": 0.49359039342358696, |
| "num_tokens": 210432789.0, |
| "step": 1350 |
| }, |
| { |
| "entropy": 2.71175808717535, |
| "epoch": 1.076748374970632, |
| "grad_norm": 0.51171875, |
| "learning_rate": 0.0002393197644119594, |
| "loss": 1.96485595703125, |
| "mean_token_accuracy": 0.4930601012845071, |
| "num_tokens": 214328522.0, |
| "step": 1375 |
| }, |
| { |
| "entropy": 2.692236351652126, |
| "epoch": 1.0963270420549769, |
| "grad_norm": 0.5859375, |
| "learning_rate": 0.00023124061159238553, |
| "loss": 1.9634054565429688, |
| "mean_token_accuracy": 0.49771587887169566, |
| "num_tokens": 218240312.0, |
| "step": 1400 |
| }, |
| { |
| "entropy": 2.6694742179626036, |
| "epoch": 1.1159057091393219, |
| "grad_norm": 0.5625, |
| "learning_rate": 0.0002231811185645613, |
| "loss": 1.92927978515625, |
| "mean_token_accuracy": 0.5001408948135679, |
| "num_tokens": 222131527.0, |
| "step": 1425 |
| }, |
| { |
| "entropy": 2.698059647261731, |
| "epoch": 1.1354843762236666, |
| "grad_norm": 0.5390625, |
| "learning_rate": 0.00021514973165685783, |
| "loss": 1.9672230529785155, |
| "mean_token_accuracy": 0.49608284242814266, |
| "num_tokens": 226067443.0, |
| "step": 1450 |
| }, |
| { |
| "entropy": 2.6789358364283884, |
| "epoch": 1.1550630433080116, |
| "grad_norm": 0.60546875, |
| "learning_rate": 0.00020715486774250342, |
| "loss": 1.956893310546875, |
| "mean_token_accuracy": 0.4982129548229775, |
| "num_tokens": 229991406.0, |
| "step": 1475 |
| }, |
| { |
| "entropy": 2.7062208250785704, |
| "epoch": 1.1746417103923565, |
| "grad_norm": 0.5546875, |
| "learning_rate": 0.0001992049054187214, |
| "loss": 1.9547114562988281, |
| "mean_token_accuracy": 0.49589620364152964, |
| "num_tokens": 233890134.0, |
| "step": 1500 |
| }, |
| { |
| "epoch": 1.1746417103923565, |
| "eval_bpd": 2.7522503005513013, |
| "eval_entropy": 2.5481301208510145, |
| "eval_loss": 1.9044488668441772, |
| "eval_mean_token_accuracy": 0.5064063985939268, |
| "eval_nll": 1.9077145360223966, |
| "eval_num_tokens": 233890134.0, |
| "eval_ppl": 6.737672480774532, |
| "eval_runtime": 317.8596, |
| "eval_samples_per_second": 571.35, |
| "eval_steps_per_second": 8.928, |
| "step": 1500 |
| }, |
| { |
| "entropy": 2.7129361564886767, |
| "epoch": 1.1942203774767013, |
| "grad_norm": 0.58203125, |
| "learning_rate": 0.00019130817622598124, |
| "loss": 1.9719258117675782, |
| "mean_token_accuracy": 0.49460158206696975, |
| "num_tokens": 237758654.0, |
| "step": 1525 |
| }, |
| { |
| "entropy": 2.6747650715948335, |
| "epoch": 1.2137990445610463, |
| "grad_norm": 0.5234375, |
| "learning_rate": 0.0001834729559165655, |
| "loss": 1.9511233520507814, |
| "mean_token_accuracy": 0.5000184974177605, |
| "num_tokens": 241628710.0, |
| "step": 1550 |
| }, |
| { |
| "entropy": 2.71732418091414, |
| "epoch": 1.2333777116453912, |
| "grad_norm": 0.65234375, |
| "learning_rate": 0.00017570745578160198, |
| "loss": 1.970428466796875, |
| "mean_token_accuracy": 0.4950899709651767, |
| "num_tokens": 245505115.0, |
| "step": 1575 |
| }, |
| { |
| "entropy": 2.7108077335151752, |
| "epoch": 1.2529563787297362, |
| "grad_norm": 1.4140625, |
| "learning_rate": 0.0001680198140456516, |
| "loss": 1.9686761474609376, |
| "mean_token_accuracy": 0.4953383364533806, |
| "num_tokens": 249424811.0, |
| "step": 1600 |
| }, |
| { |
| "entropy": 2.703162872704107, |
| "epoch": 1.272535045814081, |
| "grad_norm": 0.546875, |
| "learning_rate": 0.00016041808733786915, |
| "loss": 1.964592742919922, |
| "mean_token_accuracy": 0.49645553220386396, |
| "num_tokens": 253343499.0, |
| "step": 1625 |
| }, |
| { |
| "entropy": 2.66976916182448, |
| "epoch": 1.292113712898426, |
| "grad_norm": 0.6328125, |
| "learning_rate": 0.0001529102422486769, |
| "loss": 1.9394204711914063, |
| "mean_token_accuracy": 0.5004235959752784, |
| "num_tokens": 257234610.0, |
| "step": 1650 |
| }, |
| { |
| "entropy": 2.721756610187284, |
| "epoch": 1.3116923799827709, |
| "grad_norm": 0.55859375, |
| "learning_rate": 0.00014550414698079727, |
| "loss": 1.973395538330078, |
| "mean_token_accuracy": 0.4934210284984832, |
| "num_tokens": 261125376.0, |
| "step": 1675 |
| }, |
| { |
| "entropy": 2.675206924819273, |
| "epoch": 1.3312710470671156, |
| "grad_norm": 0.6015625, |
| "learning_rate": 0.00013820756310339665, |
| "loss": 1.9494314575195313, |
| "mean_token_accuracy": 0.5002018785718743, |
| "num_tokens": 265045653.0, |
| "step": 1700 |
| }, |
| { |
| "entropy": 2.6791174841836285, |
| "epoch": 1.3508497141514606, |
| "grad_norm": 0.6015625, |
| "learning_rate": 0.00013102813741798025, |
| "loss": 1.9530244445800782, |
| "mean_token_accuracy": 0.4984120513628194, |
| "num_tokens": 268938718.0, |
| "step": 1725 |
| }, |
| { |
| "entropy": 2.6993336161214603, |
| "epoch": 1.3704283812358056, |
| "grad_norm": 0.53125, |
| "learning_rate": 0.00012397339394456356, |
| "loss": 1.9576156616210938, |
| "mean_token_accuracy": 0.4969041736325842, |
| "num_tokens": 272852464.0, |
| "step": 1750 |
| }, |
| { |
| "epoch": 1.3704283812358056, |
| "eval_bpd": 2.745460742423444, |
| "eval_entropy": 2.540863456538099, |
| "eval_loss": 1.8997492790222168, |
| "eval_mean_token_accuracy": 0.5073028806361339, |
| "eval_nll": 1.9030083729488245, |
| "eval_num_tokens": 272852464.0, |
| "eval_ppl": 6.7060383913214, |
| "eval_runtime": 317.8678, |
| "eval_samples_per_second": 571.335, |
| "eval_steps_per_second": 8.928, |
| "step": 1750 |
| }, |
| { |
| "entropy": 2.6804251901539, |
| "epoch": 1.3900070483201503, |
| "grad_norm": 1.1328125, |
| "learning_rate": 0.00011705072603651829, |
| "loss": 1.966283721923828, |
| "mean_token_accuracy": 0.499262063001487, |
| "num_tokens": 276760333.0, |
| "step": 1775 |
| }, |
| { |
| "entropy": 2.6867454461529983, |
| "epoch": 1.4095857154044953, |
| "grad_norm": 0.58984375, |
| "learning_rate": 0.00011026738863235674, |
| "loss": 1.9592491149902345, |
| "mean_token_accuracy": 0.49931023020028253, |
| "num_tokens": 280651605.0, |
| "step": 1800 |
| }, |
| { |
| "entropy": 2.743908617496538, |
| "epoch": 1.4291643824888403, |
| "grad_norm": 0.51171875, |
| "learning_rate": 0.00010363049065257416, |
| "loss": 1.9920062255859374, |
| "mean_token_accuracy": 0.49101824791568294, |
| "num_tokens": 284514323.0, |
| "step": 1825 |
| }, |
| { |
| "entropy": 2.7106320855852593, |
| "epoch": 1.448743049573185, |
| "grad_norm": 0.54296875, |
| "learning_rate": 9.71469875495186e-05, |
| "loss": 1.970909881591797, |
| "mean_token_accuracy": 0.49648378670825294, |
| "num_tokens": 288427479.0, |
| "step": 1850 |
| }, |
| { |
| "entropy": 2.6800678695504154, |
| "epoch": 1.46832171665753, |
| "grad_norm": 0.54296875, |
| "learning_rate": 9.082367401809396e-05, |
| "loss": 1.948299102783203, |
| "mean_token_accuracy": 0.4985318521843758, |
| "num_tokens": 292338476.0, |
| "step": 1875 |
| }, |
| { |
| "entropy": 2.7168465775671655, |
| "epoch": 1.487900383741875, |
| "grad_norm": 0.6015625, |
| "learning_rate": 8.466717687493703e-05, |
| "loss": 1.9598713684082032, |
| "mean_token_accuracy": 0.4938074109471161, |
| "num_tokens": 296239840.0, |
| "step": 1900 |
| }, |
| { |
| "entropy": 2.7053632534812735, |
| "epoch": 1.5074790508262197, |
| "grad_norm": 0.54296875, |
| "learning_rate": 7.868394811353039e-05, |
| "loss": 1.9856521606445312, |
| "mean_token_accuracy": 0.4938124810525953, |
| "num_tokens": 300130010.0, |
| "step": 1925 |
| }, |
| { |
| "entropy": 2.6828013073823707, |
| "epoch": 1.5270577179105647, |
| "grad_norm": 0.5390625, |
| "learning_rate": 7.288025814252918e-05, |
| "loss": 1.9554457092285156, |
| "mean_token_accuracy": 0.4990507491186839, |
| "num_tokens": 304047616.0, |
| "step": 1950 |
| }, |
| { |
| "entropy": 2.6844564534367192, |
| "epoch": 1.5466363849949096, |
| "grad_norm": 0.515625, |
| "learning_rate": 6.726218921438915e-05, |
| "loss": 1.946611328125, |
| "mean_token_accuracy": 0.4998197396157143, |
| "num_tokens": 307919609.0, |
| "step": 1975 |
| }, |
| { |
| "entropy": 2.677877172118015, |
| "epoch": 1.5662150520792544, |
| "grad_norm": 0.53125, |
| "learning_rate": 6.183562905118107e-05, |
| "loss": 1.9413558959960937, |
| "mean_token_accuracy": 0.4990722147850662, |
| "num_tokens": 311806214.0, |
| "step": 2000 |
| }, |
| { |
| "epoch": 1.5662150520792544, |
| "eval_bpd": 2.741979334114817, |
| "eval_entropy": 2.526833154664517, |
| "eval_loss": 1.8973326683044434, |
| "eval_mean_token_accuracy": 0.5078092915779853, |
| "eval_nll": 1.9005952445953216, |
| "eval_num_tokens": 311806214.0, |
| "eval_ppl": 6.689875369506092, |
| "eval_runtime": 317.8347, |
| "eval_samples_per_second": 571.395, |
| "eval_steps_per_second": 8.929, |
| "step": 2000 |
| }, |
| { |
| "entropy": 2.7164615675383526, |
| "epoch": 1.5857937191635993, |
| "grad_norm": 0.5234375, |
| "learning_rate": 5.660626467427307e-05, |
| "loss": 1.9794667053222657, |
| "mean_token_accuracy": 0.4929969155160395, |
| "num_tokens": 315733264.0, |
| "step": 2025 |
| }, |
| { |
| "entropy": 2.7066155855772465, |
| "epoch": 1.6053723862479443, |
| "grad_norm": 0.48046875, |
| "learning_rate": 5.157957644434627e-05, |
| "loss": 1.9774436950683594, |
| "mean_token_accuracy": 0.4956127546833086, |
| "num_tokens": 319661012.0, |
| "step": 2050 |
| }, |
| { |
| "entropy": 2.670972178570003, |
| "epoch": 1.624951053332289, |
| "grad_norm": 0.54296875, |
| "learning_rate": 4.676083231799086e-05, |
| "loss": 1.9275872802734375, |
| "mean_token_accuracy": 0.5002606406547293, |
| "num_tokens": 323574025.0, |
| "step": 2075 |
| }, |
| { |
| "entropy": 2.711799482902222, |
| "epoch": 1.644529720416634, |
| "grad_norm": 0.5625, |
| "learning_rate": 4.2155082326900877e-05, |
| "loss": 1.9813533020019531, |
| "mean_token_accuracy": 0.49327668520870743, |
| "num_tokens": 327471092.0, |
| "step": 2100 |
| }, |
| { |
| "entropy": 2.6970261707972836, |
| "epoch": 1.664108387500979, |
| "grad_norm": 0.57421875, |
| "learning_rate": 3.7767153285453686e-05, |
| "loss": 1.9576905822753907, |
| "mean_token_accuracy": 0.49707455761602976, |
| "num_tokens": 331389183.0, |
| "step": 2125 |
| }, |
| { |
| "entropy": 2.677772254526351, |
| "epoch": 1.6836870545853238, |
| "grad_norm": 0.59375, |
| "learning_rate": 3.360164373222063e-05, |
| "loss": 1.957755126953125, |
| "mean_token_accuracy": 0.49902625771959186, |
| "num_tokens": 335283715.0, |
| "step": 2150 |
| }, |
| { |
| "entropy": 2.6987997353257436, |
| "epoch": 1.7032657216696687, |
| "grad_norm": 0.7421875, |
| "learning_rate": 2.966291911071012e-05, |
| "loss": 1.9490525817871094, |
| "mean_token_accuracy": 0.4974400475960837, |
| "num_tokens": 339192022.0, |
| "step": 2175 |
| }, |
| { |
| "entropy": 2.6854547652224467, |
| "epoch": 1.7228443887540137, |
| "grad_norm": 0.59765625, |
| "learning_rate": 2.5955107194394062e-05, |
| "loss": 1.9636650085449219, |
| "mean_token_accuracy": 0.49844507096807833, |
| "num_tokens": 343093506.0, |
| "step": 2200 |
| }, |
| { |
| "entropy": 2.745404202931747, |
| "epoch": 1.7424230558383584, |
| "grad_norm": 0.5625, |
| "learning_rate": 2.2482093760811617e-05, |
| "loss": 1.998858642578125, |
| "mean_token_accuracy": 0.49048741045360417, |
| "num_tokens": 346973141.0, |
| "step": 2225 |
| }, |
| { |
| "entropy": 2.688500183765981, |
| "epoch": 1.7620017229227034, |
| "grad_norm": 0.53125, |
| "learning_rate": 1.9247518519283968e-05, |
| "loss": 1.945882568359375, |
| "mean_token_accuracy": 0.49931712291043334, |
| "num_tokens": 350842781.0, |
| "step": 2250 |
| }, |
| { |
| "epoch": 1.7620017229227034, |
| "eval_bpd": 2.740859968520357, |
| "eval_entropy": 2.529529855314099, |
| "eval_loss": 1.896554946899414, |
| "eval_mean_token_accuracy": 0.5079694414016338, |
| "eval_nll": 1.8998193594895059, |
| "eval_num_tokens": 350842781.0, |
| "eval_ppl": 6.684686807971229, |
| "eval_runtime": 317.8554, |
| "eval_samples_per_second": 571.357, |
| "eval_steps_per_second": 8.929, |
| "step": 2250 |
| }, |
| { |
| "entropy": 2.705022756460653, |
| "epoch": 1.7815803900070484, |
| "grad_norm": 0.546875, |
| "learning_rate": 1.625477129650835e-05, |
| "loss": 1.9555226135253907, |
| "mean_token_accuracy": 0.4978065436114544, |
| "num_tokens": 354757724.0, |
| "step": 2275 |
| }, |
| { |
| "entropy": 2.7167964107785805, |
| "epoch": 1.8011590570913931, |
| "grad_norm": 0.51953125, |
| "learning_rate": 1.3506988484027926e-05, |
| "loss": 1.9781422424316406, |
| "mean_token_accuracy": 0.4929085231575454, |
| "num_tokens": 358654333.0, |
| "step": 2300 |
| }, |
| { |
| "entropy": 2.685449053375581, |
| "epoch": 1.820737724175738, |
| "grad_norm": 0.5625, |
| "learning_rate": 1.1007049751301789e-05, |
| "loss": 1.9553031921386719, |
| "mean_token_accuracy": 0.498819113219479, |
| "num_tokens": 362545095.0, |
| "step": 2325 |
| }, |
| { |
| "entropy": 2.67209738728332, |
| "epoch": 1.840316391260083, |
| "grad_norm": 0.5234375, |
| "learning_rate": 8.757575027818238e-06, |
| "loss": 1.9598594665527345, |
| "mean_token_accuracy": 0.5002877914117176, |
| "num_tokens": 366446317.0, |
| "step": 2350 |
| }, |
| { |
| "entropy": 2.70602050797425, |
| "epoch": 1.8598950583444278, |
| "grad_norm": 0.62890625, |
| "learning_rate": 6.760921757415439e-06, |
| "loss": 1.963476104736328, |
| "mean_token_accuracy": 0.49664319603072654, |
| "num_tokens": 370335504.0, |
| "step": 2375 |
| }, |
| { |
| "entropy": 2.717956616262883, |
| "epoch": 1.8794737254287728, |
| "grad_norm": 0.55859375, |
| "learning_rate": 5.0191824276862585e-06, |
| "loss": 1.9688818359375, |
| "mean_token_accuracy": 0.49326475180265583, |
| "num_tokens": 374199425.0, |
| "step": 2400 |
| }, |
| { |
| "entropy": 2.715174971645218, |
| "epoch": 1.8990523925131177, |
| "grad_norm": 0.56640625, |
| "learning_rate": 3.5341823770564385e-06, |
| "loss": 1.9765179443359375, |
| "mean_token_accuracy": 0.49364000601175223, |
| "num_tokens": 378109465.0, |
| "step": 2425 |
| }, |
| { |
| "entropy": 2.671021122239945, |
| "epoch": 1.9186310595974625, |
| "grad_norm": 0.5, |
| "learning_rate": 2.307477881834663e-06, |
| "loss": 1.9456196594238282, |
| "mean_token_accuracy": 0.4995694641029147, |
| "num_tokens": 382015089.0, |
| "step": 2450 |
| }, |
| { |
| "entropy": 2.6898520242951838, |
| "epoch": 1.9382097266818075, |
| "grad_norm": 0.64453125, |
| "learning_rate": 1.340354525238785e-06, |
| "loss": 1.966468048095703, |
| "mean_token_accuracy": 0.49751518802294914, |
| "num_tokens": 385896919.0, |
| "step": 2475 |
| }, |
| { |
| "entropy": 2.6790956659680316, |
| "epoch": 1.9577883937661524, |
| "grad_norm": 0.53125, |
| "learning_rate": 6.33825850107933e-07, |
| "loss": 1.9549522399902344, |
| "mean_token_accuracy": 0.500419180984845, |
| "num_tokens": 389799725.0, |
| "step": 2500 |
| }, |
| { |
| "epoch": 1.9577883937661524, |
| "eval_bpd": 2.7410050480160044, |
| "eval_entropy": 2.5233276360670986, |
| "eval_loss": 1.8966543674468994, |
| "eval_mean_token_accuracy": 0.5080129268654007, |
| "eval_nll": 1.8999199209328708, |
| "eval_num_tokens": 389799725.0, |
| "eval_ppl": 6.6853590635260085, |
| "eval_runtime": 317.8734, |
| "eval_samples_per_second": 571.325, |
| "eval_steps_per_second": 8.928, |
| "step": 2500 |
| }, |
| { |
| "entropy": 2.700464563382669, |
| "epoch": 1.9773670608504972, |
| "grad_norm": 0.5, |
| "learning_rate": 1.88632296712149e-07, |
| "loss": 1.967222900390625, |
| "mean_token_accuracy": 0.49522956458534473, |
| "num_tokens": 393678888.0, |
| "step": 2525 |
| }, |
| { |
| "entropy": 2.6681114449302297, |
| "epoch": 1.9969457279348422, |
| "grad_norm": 0.77734375, |
| "learning_rate": 5.240426772828988e-09, |
| "loss": 1.9312892150878906, |
| "mean_token_accuracy": 0.5011825757761065, |
| "num_tokens": 397533693.0, |
| "step": 2550 |
| }, |
| { |
| "epoch": 2.0, |
| "eval_bpd": 2.7410092079469774, |
| "eval_entropy": 2.5233090361563453, |
| "eval_loss": 1.8966583013534546, |
| "eval_mean_token_accuracy": 0.5080199040107065, |
| "eval_nll": 1.8999228043772962, |
| "eval_num_tokens": 398148860.0, |
| "eval_ppl": 6.685378340415125, |
| "eval_runtime": 318.7016, |
| "eval_samples_per_second": 569.84, |
| "eval_steps_per_second": 8.905, |
| "step": 2554 |
| } |
| ], |
| "logging_steps": 25, |
| "max_steps": 2554, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 1.2938339313930778e+18, |
| "train_batch_size": 64, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
