{ "best_global_step": 1500, "best_metric": 1.908670071419093, "best_model_checkpoint": "/content/cot-split1/checkpoint-1500", "epoch": 2.3477443609022557, "eval_steps": 250, "global_step": 1500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0, "eval_bpd": 5.449506205976702, "eval_entropy": 3.8259646681880883, "eval_loss": 3.7819511890411377, "eval_mean_token_accuracy": 0.3298281561563004, "eval_nll": 3.777309862116675, "eval_num_tokens": 0.0, "eval_ppl": 43.69832894461198, "eval_runtime": 318.4397, "eval_samples_per_second": 570.309, "eval_steps_per_second": 8.912, "step": 0 }, { "entropy": 3.936198543874438, "epoch": 0.0015664160401002505, "grad_norm": 3.5, "learning_rate": 0.0, "loss": 3.687645435333252, "mean_token_accuracy": 0.34163331851193124, "num_tokens": 154043.0, "step": 1 }, { "entropy": 4.1582471953791025, "epoch": 0.039160401002506263, "grad_norm": 1.8515625, "learning_rate": 9.375e-05, "loss": 3.716327985127767, "mean_token_accuracy": 0.32336124196593746, "num_tokens": 3937675.0, "step": 25 }, { "entropy": 4.004832741749853, "epoch": 0.07832080200501253, "grad_norm": 1.578125, "learning_rate": 0.00019140625, "loss": 3.245400085449219, "mean_token_accuracy": 0.3525532396162817, "num_tokens": 7842040.0, "step": 50 }, { "entropy": 3.7060696659038226, "epoch": 0.1174812030075188, "grad_norm": 1.1328125, "learning_rate": 0.0002890625, "loss": 2.899266357421875, "mean_token_accuracy": 0.3803508952459354, "num_tokens": 11723384.0, "step": 75 }, { "entropy": 3.483661951565247, "epoch": 0.15664160401002505, "grad_norm": 0.796875, "learning_rate": 0.00038671875, "loss": 2.5580235290527344, "mean_token_accuracy": 0.4106306236210145, "num_tokens": 15613476.0, "step": 100 }, { "entropy": 3.279625251379403, "epoch": 0.19580200501253134, "grad_norm": 0.71484375, "learning_rate": 0.000484375, "loss": 2.3971221923828123, "mean_token_accuracy": 0.43163137955735414, "num_tokens": 19537696.0, "step": 125 }, { "entropy": 3.1596573478657435, "epoch": 0.2349624060150376, "grad_norm": 0.640625, "learning_rate": 0.0004999075642455791, "loss": 2.332003479003906, "mean_token_accuracy": 0.44275144542816425, "num_tokens": 23462566.0, "step": 150 }, { "entropy": 3.104312514398783, "epoch": 0.2741228070175439, "grad_norm": 0.609375, "learning_rate": 0.0004995565798713207, "loss": 2.2806515502929687, "mean_token_accuracy": 0.44779701792125276, "num_tokens": 27367329.0, "step": 175 }, { "entropy": 3.062113077731651, "epoch": 0.3132832080200501, "grad_norm": 0.796875, "learning_rate": 0.0004989440608306114, "loss": 2.2435511779785156, "mean_token_accuracy": 0.45369474441778246, "num_tokens": 31290903.0, "step": 200 }, { "entropy": 3.007945648412728, "epoch": 0.3524436090225564, "grad_norm": 0.71875, "learning_rate": 0.0004980706490418613, "loss": 2.214092559814453, "mean_token_accuracy": 0.46001344086021506, "num_tokens": 35158603.0, "step": 225 }, { "entropy": 2.9779412397442524, "epoch": 0.3916040100250627, "grad_norm": 0.65625, "learning_rate": 0.0004969372598384225, "loss": 2.191622314453125, "mean_token_accuracy": 0.46235443734230136, "num_tokens": 39050018.0, "step": 250 }, { "epoch": 0.3916040100250627, "eval_bpd": 3.058715802573539, "eval_entropy": 2.8622500597497296, "eval_loss": 2.117133855819702, "eval_mean_token_accuracy": 0.4719805146180118, "eval_nll": 2.120140234687999, "eval_num_tokens": 39050018.0, "eval_ppl": 8.332305884076892, "eval_runtime": 317.9266, "eval_samples_per_second": 571.229, "eval_steps_per_second": 8.927, "step": 250 }, { "entropy": 2.91319560400022, "epoch": 0.4307644110275689, "grad_norm": 0.59375, "learning_rate": 0.0004955450810093227, "loss": 2.1354435729980468, "mean_token_accuracy": 0.46928864274436577, "num_tokens": 42938915.0, "step": 275 }, { "entropy": 2.974404757708524, "epoch": 0.4699248120300752, "grad_norm": 0.63671875, "learning_rate": 0.0004938955715544644, "loss": 2.180239562988281, "mean_token_accuracy": 0.4623402148856342, "num_tokens": 46830642.0, "step": 300 }, { "entropy": 2.947778592589299, "epoch": 0.5090852130325815, "grad_norm": 0.52734375, "learning_rate": 0.0004919904601555969, "loss": 2.137745666503906, "mean_token_accuracy": 0.4655975033913256, "num_tokens": 50738889.0, "step": 325 }, { "entropy": 2.912182858652311, "epoch": 0.5482456140350878, "grad_norm": 0.73046875, "learning_rate": 0.0004898317433646626, "loss": 2.134133148193359, "mean_token_accuracy": 0.47067078409410695, "num_tokens": 54590447.0, "step": 350 }, { "entropy": 2.8808906902937665, "epoch": 0.5874060150375939, "grad_norm": 0.71484375, "learning_rate": 0.00048742168351141525, "loss": 2.111539001464844, "mean_token_accuracy": 0.47415726155363475, "num_tokens": 58477393.0, "step": 375 }, { "entropy": 2.8902592806431113, "epoch": 0.6265664160401002, "grad_norm": 0.77734375, "learning_rate": 0.00048476280633250404, "loss": 2.1172853088378907, "mean_token_accuracy": 0.47171053962916015, "num_tokens": 62364053.0, "step": 400 }, { "entropy": 2.8472467625827407, "epoch": 0.6657268170426065, "grad_norm": 0.5546875, "learning_rate": 0.00048185789832450813, "loss": 2.079963836669922, "mean_token_accuracy": 0.47701584614870657, "num_tokens": 66254150.0, "step": 425 }, { "entropy": 2.849270481153217, "epoch": 0.7048872180451128, "grad_norm": 0.76171875, "learning_rate": 0.00047871000382369524, "loss": 2.076465911865234, "mean_token_accuracy": 0.477678835790845, "num_tokens": 70155017.0, "step": 450 }, { "entropy": 2.8533846163704104, "epoch": 0.7440476190476191, "grad_norm": 0.75, "learning_rate": 0.0004753224218155648, "loss": 2.076942901611328, "mean_token_accuracy": 0.4763377187304664, "num_tokens": 74017889.0, "step": 475 }, { "entropy": 2.833587274583335, "epoch": 0.7832080200501254, "grad_norm": 0.61328125, "learning_rate": 0.0004716987024775197, "loss": 2.0738652038574217, "mean_token_accuracy": 0.48083406578747806, "num_tokens": 77928779.0, "step": 500 }, { "epoch": 0.7832080200501254, "eval_bpd": 2.898750179783218, "eval_entropy": 2.6689496127647594, "eval_loss": 2.006093978881836, "eval_mean_token_accuracy": 0.4891084327887536, "eval_nll": 2.009260514264372, "eval_num_tokens": 77928779.0, "eval_ppl": 7.457800370714545, "eval_runtime": 317.8942, "eval_samples_per_second": 571.288, "eval_steps_per_second": 8.927, "step": 500 }, { "entropy": 2.8345490746659476, "epoch": 0.8223684210526315, "grad_norm": 0.58984375, "learning_rate": 0.000467842643458289, "loss": 2.084154052734375, "mean_token_accuracy": 0.47867488342803477, "num_tokens": 81819393.0, "step": 525 }, { "entropy": 2.786899686674756, "epoch": 0.8615288220551378, "grad_norm": 0.98046875, "learning_rate": 0.0004637582858980017, "loss": 2.0310736083984375, "mean_token_accuracy": 0.4840942726061862, "num_tokens": 85731124.0, "step": 550 }, { "entropy": 2.8589994783572577, "epoch": 0.9006892230576441, "grad_norm": 0.67578125, "learning_rate": 0.0004594499101930825, "loss": 2.094312896728516, "mean_token_accuracy": 0.47510495282804466, "num_tokens": 89624369.0, "step": 575 }, { "entropy": 2.7673380789023, "epoch": 0.9398496240601504, "grad_norm": 0.5390625, "learning_rate": 0.00045492203151040655, "loss": 2.016443634033203, "mean_token_accuracy": 0.4872064297611196, "num_tokens": 93495507.0, "step": 600 }, { "entropy": 2.821264199014746, "epoch": 0.9790100250626567, "grad_norm": 0.69140625, "learning_rate": 0.00045017939505541626, "loss": 2.049879455566406, "mean_token_accuracy": 0.4826080450793048, "num_tokens": 97422212.0, "step": 625 }, { "entropy": 2.799159714687336, "epoch": 1.0172305764411027, "grad_norm": 0.5859375, "learning_rate": 0.00044522697109915813, "loss": 2.0454527282714845, "mean_token_accuracy": 0.48334792056497855, "num_tokens": 101238525.0, "step": 650 }, { "entropy": 2.751235980421951, "epoch": 1.056390977443609, "grad_norm": 0.58984375, "learning_rate": 0.0004400699497694506, "loss": 2.012087707519531, "mean_token_accuracy": 0.4896175168359564, "num_tokens": 105137633.0, "step": 675 }, { "entropy": 2.7633214804817277, "epoch": 1.0955513784461153, "grad_norm": 0.60546875, "learning_rate": 0.00043471373561164396, "loss": 2.0151519775390625, "mean_token_accuracy": 0.48745193922878793, "num_tokens": 109027156.0, "step": 700 }, { "entropy": 2.764837322151706, "epoch": 1.1347117794486214, "grad_norm": 0.6796875, "learning_rate": 0.00042916394192466997, "loss": 2.011883544921875, "mean_token_accuracy": 0.48801230267546136, "num_tokens": 112918175.0, "step": 725 }, { "entropy": 2.7570182676631614, "epoch": 1.1738721804511278, "grad_norm": 0.609375, "learning_rate": 0.00042342638487831874, "loss": 2.0271852111816404, "mean_token_accuracy": 0.4883824761848996, "num_tokens": 116839662.0, "step": 750 }, { "epoch": 1.1738721804511278, "eval_bpd": 2.8315747644426814, "eval_entropy": 2.6417940543917178, "eval_loss": 1.9594571590423584, "eval_mean_token_accuracy": 0.497060593586316, "eval_nll": 1.9626980645181358, "eval_num_tokens": 116839662.0, "eval_ppl": 7.118507370829529, "eval_runtime": 317.8489, "eval_samples_per_second": 571.369, "eval_steps_per_second": 8.929, "step": 750 }, { "entropy": 2.7844207042311715, "epoch": 1.213032581453634, "grad_norm": 0.68359375, "learning_rate": 0.0004175070774179078, "loss": 2.0220986938476564, "mean_token_accuracy": 0.4860440628692503, "num_tokens": 120758777.0, "step": 775 }, { "entropy": 2.7899788574494933, "epoch": 1.2521929824561404, "grad_norm": 0.55859375, "learning_rate": 0.0004114122229627302, "loss": 2.039529113769531, "mean_token_accuracy": 0.4842958016311785, "num_tokens": 124638155.0, "step": 800 }, { "entropy": 2.7349560071334476, "epoch": 1.2913533834586466, "grad_norm": 0.55859375, "learning_rate": 0.00040514820890488664, "loss": 2.0020352172851563, "mean_token_accuracy": 0.4927521478887577, "num_tokens": 128509722.0, "step": 825 }, { "entropy": 2.745659946454679, "epoch": 1.3305137844611528, "grad_norm": 0.74609375, "learning_rate": 0.00039872159991531533, "loss": 2.0152940368652343, "mean_token_accuracy": 0.48988635025320926, "num_tokens": 132428586.0, "step": 850 }, { "entropy": 2.7871976479888647, "epoch": 1.3696741854636592, "grad_norm": 0.84375, "learning_rate": 0.0003921391310640325, "loss": 2.039204559326172, "mean_token_accuracy": 0.4852042920106659, "num_tokens": 136329200.0, "step": 875 }, { "entropy": 2.7404741305566205, "epoch": 1.4088345864661656, "grad_norm": 0.51953125, "learning_rate": 0.0003854077007617969, "loss": 1.9955612182617188, "mean_token_accuracy": 0.4914711251122463, "num_tokens": 140228250.0, "step": 900 }, { "entropy": 2.731783048426117, "epoch": 1.4479949874686717, "grad_norm": 0.56640625, "learning_rate": 0.0003785343635305919, "loss": 1.9946755981445312, "mean_token_accuracy": 0.492961033900984, "num_tokens": 144119470.0, "step": 925 }, { "entropy": 2.734028948778845, "epoch": 1.487155388471178, "grad_norm": 0.6640625, "learning_rate": 0.00037152632261050483, "loss": 1.9961911010742188, "mean_token_accuracy": 0.4915906915552139, "num_tokens": 148006939.0, "step": 950 }, { "entropy": 2.7352198930393166, "epoch": 1.526315789473684, "grad_norm": 0.57421875, "learning_rate": 0.0003643909224107492, "loss": 1.9914035034179687, "mean_token_accuracy": 0.49272672598435946, "num_tokens": 151885356.0, "step": 975 }, { "entropy": 2.744184872854561, "epoch": 1.5654761904761905, "grad_norm": 0.5703125, "learning_rate": 0.00035713564081274257, "loss": 2.0107650756835938, "mean_token_accuracy": 0.4889519028709596, "num_tokens": 155788149.0, "step": 1000 }, { "epoch": 1.5654761904761905, "eval_bpd": 2.7929365814183416, "eval_entropy": 2.561582521237903, "eval_loss": 1.9326468706130981, "eval_mean_token_accuracy": 0.5020671009912576, "eval_nll": 1.9359161168928556, "eval_num_tokens": 155788149.0, "eval_ppl": 6.930390195563555, "eval_runtime": 318.0478, "eval_samples_per_second": 571.012, "eval_steps_per_second": 8.923, "step": 1000 }, { "entropy": 2.707526650239954, "epoch": 1.6046365914786969, "grad_norm": 0.5546875, "learning_rate": 0.0003497680813333055, "loss": 1.9714834594726562, "mean_token_accuracy": 0.49562785553365774, "num_tokens": 159699795.0, "step": 1025 }, { "entropy": 2.72805733193072, "epoch": 1.643796992481203, "grad_norm": 0.765625, "learning_rate": 0.0003422959651561952, "loss": 1.971968536376953, "mean_token_accuracy": 0.49344011441359326, "num_tokens": 163574307.0, "step": 1050 }, { "entropy": 2.6893244178308024, "epoch": 1.6829573934837092, "grad_norm": 0.52734375, "learning_rate": 0.0003347271230403245, "loss": 1.9534384155273437, "mean_token_accuracy": 0.497666397797124, "num_tokens": 167453694.0, "step": 1075 }, { "entropy": 2.7430753006255277, "epoch": 1.7221177944862154, "grad_norm": 0.5390625, "learning_rate": 0.00032706948711314597, "loss": 1.9894099426269531, "mean_token_accuracy": 0.4906959803986491, "num_tokens": 171375635.0, "step": 1100 }, { "entropy": 2.737667996911359, "epoch": 1.7612781954887218, "grad_norm": 0.63671875, "learning_rate": 0.0003193310825578028, "loss": 1.995572052001953, "mean_token_accuracy": 0.4914335705931985, "num_tokens": 175271550.0, "step": 1125 }, { "entropy": 2.736452384012791, "epoch": 1.8004385964912282, "grad_norm": 0.58984375, "learning_rate": 0.0003115200192027566, "loss": 1.9998680114746095, "mean_token_accuracy": 0.49136637186282833, "num_tokens": 179170243.0, "step": 1150 }, { "entropy": 2.6840253382543815, "epoch": 1.8395989974937343, "grad_norm": 0.5625, "learning_rate": 0.00030364448302270787, "loss": 1.949139404296875, "mean_token_accuracy": 0.49796950154057085, "num_tokens": 183048860.0, "step": 1175 }, { "entropy": 2.7671729360693793, "epoch": 1.8787593984962405, "grad_norm": 0.6796875, "learning_rate": 0.0002957127275597154, "loss": 2.0151930236816407, "mean_token_accuracy": 0.4883661289672581, "num_tokens": 186947863.0, "step": 1200 }, { "entropy": 2.7276026045552215, "epoch": 1.917919799498747, "grad_norm": 0.51953125, "learning_rate": 0.0002877330652735044, "loss": 1.9936309814453126, "mean_token_accuracy": 0.49266964650003753, "num_tokens": 190837792.0, "step": 1225 }, { "entropy": 2.7106085190098876, "epoch": 1.957080200501253, "grad_norm": 0.57421875, "learning_rate": 0.0002797138588300303, "loss": 1.9655186462402343, "mean_token_accuracy": 0.4963224160769589, "num_tokens": 194765738.0, "step": 1250 }, { "epoch": 1.957080200501253, "eval_bpd": 2.7684189685627008, "eval_entropy": 2.5841895770631353, "eval_loss": 1.9156484603881836, "eval_mean_token_accuracy": 0.5042775579796719, "eval_nll": 1.9189218026679078, "eval_num_tokens": 194765738.0, "eval_ppl": 6.813608093356783, "eval_runtime": 317.8737, "eval_samples_per_second": 571.324, "eval_steps_per_second": 8.928, "step": 1250 }, { "entropy": 2.7457793346083226, "epoch": 1.9962406015037595, "grad_norm": 0.9921875, "learning_rate": 0.0002716635123374255, "loss": 2.0136396789550783, "mean_token_accuracy": 0.4905763155027781, "num_tokens": 198681309.0, "step": 1275 }, { "entropy": 2.7296520815516434, "epoch": 2.0344611528822054, "grad_norm": 0.609375, "learning_rate": 0.00026359046253851537, "loss": 1.9885548400878905, "mean_token_accuracy": 0.49141303876894793, "num_tokens": 202488293.0, "step": 1300 }, { "entropy": 2.7237404436481647, "epoch": 2.0736215538847116, "grad_norm": 0.5859375, "learning_rate": 0.00025550316996913314, "loss": 1.9903170776367187, "mean_token_accuracy": 0.4935096359392426, "num_tokens": 206409968.0, "step": 1325 }, { "entropy": 2.7291376300032053, "epoch": 2.112781954887218, "grad_norm": 0.61328125, "learning_rate": 0.00024741011009149944, "loss": 1.9878811645507812, "mean_token_accuracy": 0.49241777430140515, "num_tokens": 210314638.0, "step": 1350 }, { "entropy": 2.727339653278635, "epoch": 2.1519423558897244, "grad_norm": 0.59765625, "learning_rate": 0.0002393197644119594, "loss": 1.9830924987792968, "mean_token_accuracy": 0.49156372149799366, "num_tokens": 214195432.0, "step": 1375 }, { "entropy": 2.6867289156814746, "epoch": 2.1911027568922306, "grad_norm": 0.578125, "learning_rate": 0.00023124061159238553, "loss": 1.9581849670410156, "mean_token_accuracy": 0.4971003970211406, "num_tokens": 218094898.0, "step": 1400 }, { "entropy": 2.7372838236997215, "epoch": 2.2302631578947367, "grad_norm": 0.5703125, "learning_rate": 0.0002231811185645613, "loss": 1.9794134521484374, "mean_token_accuracy": 0.4906712610116262, "num_tokens": 221998326.0, "step": 1425 }, { "entropy": 2.714885350188938, "epoch": 2.269423558897243, "grad_norm": 0.5625, "learning_rate": 0.00021514973165685783, "loss": 1.9843505859375, "mean_token_accuracy": 0.49365376402815986, "num_tokens": 225920259.0, "step": 1450 }, { "entropy": 2.696512448708, "epoch": 2.3085839598997495, "grad_norm": 0.5, "learning_rate": 0.00020715486774250342, "loss": 1.9640107727050782, "mean_token_accuracy": 0.4955928374349573, "num_tokens": 229793611.0, "step": 1475 }, { "entropy": 2.7095071681692473, "epoch": 2.3477443609022557, "grad_norm": 0.56640625, "learning_rate": 0.0001992049054187214, "loss": 1.9568646240234375, "mean_token_accuracy": 0.4964784048245484, "num_tokens": 233686342.0, "step": 1500 }, { "epoch": 2.3477443609022557, "eval_bpd": 2.753628846729509, "eval_entropy": 2.5548404387573136, "eval_loss": 1.9053977727890015, "eval_mean_token_accuracy": 0.5062358641819159, "eval_nll": 1.908670071419093, "eval_num_tokens": 233686342.0, "eval_ppl": 6.744113642210042, "eval_runtime": 317.8614, "eval_samples_per_second": 571.346, "eval_steps_per_second": 8.928, "step": 1500 } ], "logging_steps": 25, "max_steps": 2554, "num_input_tokens_seen": 0, "num_train_epochs": 4, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 7.602066804062976e+17, "train_batch_size": 64, "trial_name": null, "trial_params": null }