diff --git "a/checkpoint-22815/trainer_state.json" "b/checkpoint-22815/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-22815/trainer_state.json" @@ -0,0 +1,169816 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9126, + "global_step": 22815, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 2e-06, + "loss": 2.6877, + "theoretical_loss": 3.484799908140819, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.0, + "learning_rate": 4e-06, + "loss": 2.5424, + "theoretical_loss": 3.4847880270596066, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.0, + "learning_rate": 6e-06, + "loss": 2.755, + "theoretical_loss": 3.4847761465781497, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.0, + "learning_rate": 8e-06, + "loss": 2.591, + "theoretical_loss": 3.484764266696394, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.0, + "learning_rate": 1e-05, + "loss": 2.7837, + "theoretical_loss": 3.4847523874142867, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.0, + "learning_rate": 1.2e-05, + "loss": 2.7135, + "theoretical_loss": 3.4847405087317727, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.0, + "learning_rate": 1.4e-05, + "loss": 2.4644, + "theoretical_loss": 3.4847286306487986, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.0, + "learning_rate": 1.6e-05, + "loss": 2.6662, + "theoretical_loss": 3.48471675316531, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.0, + "learning_rate": 1.8e-05, + "loss": 2.5318, + "theoretical_loss": 3.4847048762812536, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.0, + "learning_rate": 2e-05, + "loss": 2.5842, + "theoretical_loss": 3.4846929999965757, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.0, + "learning_rate": 2.2e-05, + "loss": 2.7033, + "theoretical_loss": 3.484681124311222, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.0, + "learning_rate": 2.4e-05, + "loss": 2.6288, + "theoretical_loss": 3.4846692492251385, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.0, + "learning_rate": 2.6e-05, + "loss": 2.5491, + "theoretical_loss": 3.484657374738272, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.0, + "learning_rate": 2.8e-05, + "loss": 2.7427, + "theoretical_loss": 3.4846455008505677, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.0, + "learning_rate": 3e-05, + "loss": 2.801, + "theoretical_loss": 3.484633627561972, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.0, + "learning_rate": 3.2e-05, + "loss": 2.9133, + "theoretical_loss": 3.484621754872432, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.0, + "learning_rate": 3.4000000000000007e-05, + "loss": 2.5832, + "theoretical_loss": 3.4846098827818928, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.0, + "learning_rate": 3.6e-05, + "loss": 2.6186, + "theoretical_loss": 3.4845980112903003, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.0, + "learning_rate": 3.8e-05, + "loss": 2.451, + "theoretical_loss": 3.484586140397602, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.0, + "learning_rate": 4e-05, + "loss": 2.4464, + "theoretical_loss": 3.484574270103743, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.0, + "learning_rate": 4.2000000000000004e-05, + "loss": 2.6498, + "theoretical_loss": 3.484562400408671, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.0, + "learning_rate": 4.4e-05, + "loss": 2.7089, + "theoretical_loss": 3.4845505313123297, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.0, + "learning_rate": 4.6e-05, + "loss": 2.6416, + "theoretical_loss": 3.4845386628146677, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.0, + "learning_rate": 4.8e-05, + "loss": 2.6498, + "theoretical_loss": 3.4845267949156296, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 941634, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.817304849624634, + "objective/train/theoretical_loss": 3.4845149276151623, + "objective/train/tokens_used": 22097376, + "theoretical_loss": 3.4845149276151623, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5e-05, + "loss": 2.7623, + "theoretical_loss": 3.4845149276151623, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.0, + "learning_rate": 5.2e-05, + "loss": 2.449, + "theoretical_loss": 3.484503060913212, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.0, + "learning_rate": 5.4e-05, + "loss": 2.592, + "theoretical_loss": 3.484491194809725, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.0, + "learning_rate": 5.6e-05, + "loss": 2.332, + "theoretical_loss": 3.484479329304647, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.0, + "learning_rate": 5.800000000000001e-05, + "loss": 2.5444, + "theoretical_loss": 3.484467464397925, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.0, + "learning_rate": 6e-05, + "loss": 2.6551, + "theoretical_loss": 3.484455600089505, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.0, + "learning_rate": 6.2e-05, + "loss": 2.6407, + "theoretical_loss": 3.4844437363793332, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.0, + "learning_rate": 6.4e-05, + "loss": 2.6282, + "theoretical_loss": 3.4844318732673556, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.0, + "learning_rate": 6.6e-05, + "loss": 2.6431, + "theoretical_loss": 3.484420010753519, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.0, + "learning_rate": 6.800000000000001e-05, + "loss": 2.7098, + "theoretical_loss": 3.484408148837769, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.0, + "learning_rate": 7.000000000000001e-05, + "loss": 2.6963, + "theoretical_loss": 3.4843962875200525, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.0, + "learning_rate": 7.2e-05, + "loss": 2.3914, + "theoretical_loss": 3.484384426800316, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.0, + "learning_rate": 7.4e-05, + "loss": 2.5538, + "theoretical_loss": 3.4843725666785046, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.0, + "learning_rate": 7.6e-05, + "loss": 2.5626, + "theoretical_loss": 3.484360707154566, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.0, + "learning_rate": 7.8e-05, + "loss": 2.6302, + "theoretical_loss": 3.484348848228446, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.0, + "learning_rate": 8e-05, + "loss": 2.7498, + "theoretical_loss": 3.4843369899000907, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.0, + "learning_rate": 8.2e-05, + "loss": 2.5581, + "theoretical_loss": 3.484325132169446, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.0, + "learning_rate": 8.400000000000001e-05, + "loss": 2.5001, + "theoretical_loss": 3.4843132750364596, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.0, + "learning_rate": 8.599999999999999e-05, + "loss": 2.7069, + "theoretical_loss": 3.4843014185010768, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.0, + "learning_rate": 8.8e-05, + "loss": 2.5802, + "theoretical_loss": 3.4842895625632444, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.0, + "learning_rate": 8.999999999999999e-05, + "loss": 2.7858, + "theoretical_loss": 3.4842777072229083, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.0, + "learning_rate": 9.2e-05, + "loss": 2.5412, + "theoretical_loss": 3.484265852480015, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.0, + "learning_rate": 9.400000000000001e-05, + "loss": 2.5786, + "theoretical_loss": 3.4842539983345118, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.0, + "learning_rate": 9.6e-05, + "loss": 2.7589, + "theoretical_loss": 3.484242144786344, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.0, + "learning_rate": 9.800000000000001e-05, + "loss": 2.5893, + "theoretical_loss": 3.4842302918354577, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 943199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6825146675109863, + "objective/train/theoretical_loss": 3.4842184394818, + "objective/train/tokens_used": 23735776, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 2.5327, + "theoretical_loss": 3.4842184394818, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.0, + "learning_rate": 0.000102, + "loss": 2.3894, + "theoretical_loss": 3.4842065877253177, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.0, + "learning_rate": 0.000104, + "loss": 2.332, + "theoretical_loss": 3.4841947365659562, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.0, + "learning_rate": 0.000106, + "loss": 2.5794, + "theoretical_loss": 3.4841828860036625, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.0, + "learning_rate": 0.000108, + "loss": 2.7348, + "theoretical_loss": 3.484171036038383, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011, + "loss": 2.7432, + "theoretical_loss": 3.4841591866700643, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.0, + "learning_rate": 0.000112, + "loss": 2.6142, + "theoretical_loss": 3.4841473378986523, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.0, + "learning_rate": 0.000114, + "loss": 2.5629, + "theoretical_loss": 3.4841354897240935, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011600000000000001, + "loss": 2.7109, + "theoretical_loss": 3.4841236421463346, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.0, + "learning_rate": 0.000118, + "loss": 2.4379, + "theoretical_loss": 3.4841117951653224, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012, + "loss": 2.4586, + "theoretical_loss": 3.4840999487810027, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.0, + "learning_rate": 0.000122, + "loss": 2.4983, + "theoretical_loss": 3.4840881029933226, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.0, + "learning_rate": 0.000124, + "loss": 2.4585, + "theoretical_loss": 3.484076257802228, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.0, + "learning_rate": 0.000126, + "loss": 2.635, + "theoretical_loss": 3.4840644132076655, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.0, + "learning_rate": 0.000128, + "loss": 2.7968, + "theoretical_loss": 3.484052569209582, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013000000000000002, + "loss": 2.5231, + "theoretical_loss": 3.4840407258079233, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.0, + "learning_rate": 0.000132, + "loss": 2.5757, + "theoretical_loss": 3.4840288830026367, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.0, + "learning_rate": 0.000134, + "loss": 2.65, + "theoretical_loss": 3.484017040793668, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013600000000000003, + "loss": 2.4863, + "theoretical_loss": 3.4840051991809644, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013800000000000002, + "loss": 2.4278, + "theoretical_loss": 3.4839933581644718, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014000000000000001, + "loss": 2.7386, + "theoretical_loss": 3.4839815177441373, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014199999999999998, + "loss": 2.5047, + "theoretical_loss": 3.4839696779199065, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.0, + "learning_rate": 0.000144, + "loss": 2.5392, + "theoretical_loss": 3.483957838691727, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.0, + "learning_rate": 0.000146, + "loss": 2.2598, + "theoretical_loss": 3.483946000059545, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.0, + "learning_rate": 0.000148, + "loss": 2.5401, + "theoretical_loss": 3.483934162023307, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 943856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.97538423538208, + "objective/train/theoretical_loss": 3.4839223245829594, + "objective/train/tokens_used": 25374176, + "theoretical_loss": 3.4839223245829594, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015, + "loss": 2.5632, + "theoretical_loss": 3.4839223245829594, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.0, + "learning_rate": 0.000152, + "loss": 2.7443, + "theoretical_loss": 3.4839104877384486, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.0, + "learning_rate": 0.000154, + "loss": 2.7109, + "theoretical_loss": 3.483898651489722, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.0, + "learning_rate": 0.000156, + "loss": 2.3997, + "theoretical_loss": 3.4838868158367253, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.0, + "learning_rate": 0.000158, + "loss": 2.7272, + "theoretical_loss": 3.4838749807794063, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016, + "loss": 2.7979, + "theoretical_loss": 3.4838631463177103, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.0, + "learning_rate": 0.000162, + "loss": 2.6363, + "theoretical_loss": 3.483851312451584, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.0, + "learning_rate": 0.000164, + "loss": 2.49, + "theoretical_loss": 3.4838394791809746, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016600000000000002, + "loss": 2.8356, + "theoretical_loss": 3.4838276465058287, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016800000000000002, + "loss": 2.6892, + "theoretical_loss": 3.4838158144260927, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017, + "loss": 2.5959, + "theoretical_loss": 3.483803982941714, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017199999999999998, + "loss": 2.3662, + "theoretical_loss": 3.4837921520526374, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.0, + "learning_rate": 0.000174, + "loss": 2.6618, + "theoretical_loss": 3.483780321758811, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.0, + "learning_rate": 0.000176, + "loss": 2.8043, + "theoretical_loss": 3.4837684920601815, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.0, + "learning_rate": 0.000178, + "loss": 2.5749, + "theoretical_loss": 3.483756662956695, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017999999999999998, + "loss": 2.531, + "theoretical_loss": 3.483744834448298, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.0, + "learning_rate": 0.000182, + "loss": 2.6165, + "theoretical_loss": 3.483733006534938, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.0, + "learning_rate": 0.000184, + "loss": 2.5413, + "theoretical_loss": 3.483721179216561, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.0, + "learning_rate": 0.000186, + "loss": 2.6614, + "theoretical_loss": 3.4837093524931144, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018800000000000002, + "loss": 2.8339, + "theoretical_loss": 3.4836975263645438, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019, + "loss": 2.3986, + "theoretical_loss": 3.483685700830797, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.0, + "learning_rate": 0.000192, + "loss": 2.5187, + "theoretical_loss": 3.48367387589182, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.0, + "learning_rate": 0.000194, + "loss": 2.6812, + "theoretical_loss": 3.48366205154756, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019600000000000002, + "loss": 2.749, + "theoretical_loss": 3.4836502277979626, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019800000000000002, + "loss": 2.6367, + "theoretical_loss": 3.4836384046429765, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 945126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.340965509414673, + "objective/train/theoretical_loss": 3.4836265820825467, + "objective/train/tokens_used": 27012576, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 2.2674, + "theoretical_loss": 3.4836265820825467, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.0, + "learning_rate": 0.000202, + "loss": 2.6021, + "theoretical_loss": 3.483614760116621, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000204, + "loss": 2.7658, + "theoretical_loss": 3.4836029387451455, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.0, + "learning_rate": 0.000206, + "loss": 2.6777, + "theoretical_loss": 3.483591117968067, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.0, + "learning_rate": 0.000208, + "loss": 2.4825, + "theoretical_loss": 3.483579297785333, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021, + "loss": 2.6108, + "theoretical_loss": 3.483567478196889, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.0, + "learning_rate": 0.000212, + "loss": 2.5857, + "theoretical_loss": 3.483555659202683, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.0, + "learning_rate": 0.000214, + "loss": 2.6165, + "theoretical_loss": 3.4835438408026613, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.0, + "learning_rate": 0.000216, + "loss": 2.3427, + "theoretical_loss": 3.483532022996771, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.0, + "learning_rate": 0.000218, + "loss": 2.5738, + "theoretical_loss": 3.483520205784958, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022, + "loss": 2.6296, + "theoretical_loss": 3.48350838916717, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.0, + "learning_rate": 0.000222, + "loss": 2.5203, + "theoretical_loss": 3.4834965731433534, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.0, + "learning_rate": 0.000224, + "loss": 2.7331, + "theoretical_loss": 3.4834847577134553, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022600000000000002, + "loss": 2.5389, + "theoretical_loss": 3.4834729428774223, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.0, + "learning_rate": 0.000228, + "loss": 2.4499, + "theoretical_loss": 3.4834611286352013, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023, + "loss": 2.7387, + "theoretical_loss": 3.4834493149867387, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023200000000000003, + "loss": 2.3361, + "theoretical_loss": 3.483437501931982, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023400000000000002, + "loss": 2.6547, + "theoretical_loss": 3.483425689470878, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000236, + "loss": 2.6733, + "theoretical_loss": 3.4834138776033736, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023799999999999998, + "loss": 2.4096, + "theoretical_loss": 3.483402066329415, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024, + "loss": 2.5645, + "theoretical_loss": 3.48339025564895, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.0, + "learning_rate": 0.000242, + "loss": 2.5705, + "theoretical_loss": 3.4833784455619248, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.0, + "learning_rate": 0.000244, + "loss": 2.673, + "theoretical_loss": 3.4833666360682867, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246, + "loss": 2.4731, + "theoretical_loss": 3.4833548271679824, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248, + "loss": 2.4912, + "theoretical_loss": 3.4833430188609587, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 945699, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.995269536972046, + "objective/train/theoretical_loss": 3.4833312111471626, + "objective/train/tokens_used": 28650976, + "theoretical_loss": 3.4833312111471626, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 2.8199, + "theoretical_loss": 3.4833312111471626, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000252, + "loss": 2.7714, + "theoretical_loss": 3.483319404026541, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.01, + "learning_rate": 0.000254, + "loss": 2.3447, + "theoretical_loss": 3.4833075974990404, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.01, + "learning_rate": 0.000256, + "loss": 2.6911, + "theoretical_loss": 3.4832957915646086, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025800000000000004, + "loss": 2.5768, + "theoretical_loss": 3.4832839862231926, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026000000000000003, + "loss": 2.7109, + "theoretical_loss": 3.4832721814747383, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.01, + "learning_rate": 0.000262, + "loss": 2.7003, + "theoretical_loss": 3.4832603773191932, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000264, + "loss": 2.5429, + "theoretical_loss": 3.4832485737565047, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.01, + "learning_rate": 0.000266, + "loss": 2.8269, + "theoretical_loss": 3.483236770786619, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.01, + "learning_rate": 0.000268, + "loss": 2.361, + "theoretical_loss": 3.4832249684094836, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027, + "loss": 2.6062, + "theoretical_loss": 3.4832131666250454, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027200000000000005, + "loss": 2.6648, + "theoretical_loss": 3.483201365433251, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027400000000000005, + "loss": 2.7098, + "theoretical_loss": 3.4831895648340474, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027600000000000004, + "loss": 2.478, + "theoretical_loss": 3.4831777648273823, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027800000000000004, + "loss": 2.4818, + "theoretical_loss": 3.4831659654132023, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028000000000000003, + "loss": 2.6658, + "theoretical_loss": 3.4831541665914543, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028199999999999997, + "loss": 2.579, + "theoretical_loss": 3.4831423683620852, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028399999999999996, + "loss": 2.2501, + "theoretical_loss": 3.4831305707250424, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028599999999999996, + "loss": 2.9361, + "theoretical_loss": 3.483118773680273, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.01, + "learning_rate": 0.000288, + "loss": 2.7043, + "theoretical_loss": 3.4831069772277234, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029, + "loss": 2.7107, + "theoretical_loss": 3.483095181367341, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.01, + "learning_rate": 0.000292, + "loss": 2.8449, + "theoretical_loss": 3.4830833860990733, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000294, + "loss": 2.5954, + "theoretical_loss": 3.4830715914228665, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000296, + "loss": 2.5906, + "theoretical_loss": 3.4830597973386683, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000298, + "loss": 2.7968, + "theoretical_loss": 3.4830480038464255, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 946982, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6374800205230713, + "objective/train/theoretical_loss": 3.4830362109460857, + "objective/train/tokens_used": 30289376, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003, + "loss": 2.5244, + "theoretical_loss": 3.4830362109460857, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000302, + "loss": 2.792, + "theoretical_loss": 3.483024418637595, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000304, + "loss": 2.6392, + "theoretical_loss": 3.4830126269209014, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.01, + "learning_rate": 0.000306, + "loss": 2.6077, + "theoretical_loss": 3.4830008357959517, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000308, + "loss": 2.7558, + "theoretical_loss": 3.4829890452626926, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031, + "loss": 2.4685, + "theoretical_loss": 3.482977255321072, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.01, + "learning_rate": 0.000312, + "loss": 2.6854, + "theoretical_loss": 3.482965465971036, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314, + "loss": 2.4591, + "theoretical_loss": 3.4829536772125325, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000316, + "loss": 2.6594, + "theoretical_loss": 3.482941889045509, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031800000000000003, + "loss": 2.4577, + "theoretical_loss": 3.4829301014699117, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032, + "loss": 2.7174, + "theoretical_loss": 3.482918314485688, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.01, + "learning_rate": 0.000322, + "loss": 2.8312, + "theoretical_loss": 3.4829065280927853, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.01, + "learning_rate": 0.000324, + "loss": 2.4986, + "theoretical_loss": 3.482894742291151, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.01, + "learning_rate": 0.000326, + "loss": 2.6921, + "theoretical_loss": 3.4828829570807316, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000328, + "loss": 2.5031, + "theoretical_loss": 3.4828711724614747, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033, + "loss": 2.8116, + "theoretical_loss": 3.4828593884333277, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033200000000000005, + "loss": 2.8915, + "theoretical_loss": 3.4828476049962367, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033400000000000004, + "loss": 2.7403, + "theoretical_loss": 3.4828358221501503, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033600000000000004, + "loss": 2.656, + "theoretical_loss": 3.4828240398950143, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033800000000000003, + "loss": 2.4222, + "theoretical_loss": 3.4828122582307777, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034, + "loss": 2.6088, + "theoretical_loss": 3.482800477157386, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.01, + "learning_rate": 0.000342, + "loss": 2.4636, + "theoretical_loss": 3.4827886966747874, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034399999999999996, + "loss": 2.9066, + "theoretical_loss": 3.482776916782929, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000346, + "loss": 2.5033, + "theoretical_loss": 3.482765137481757, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000348, + "loss": 2.6, + "theoretical_loss": 3.4827533587712205, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 947668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.733534812927246, + "objective/train/theoretical_loss": 3.4827415806512656, + "objective/train/tokens_used": 31927776, + "theoretical_loss": 3.4827415806512656, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035, + "loss": 2.8327, + "theoretical_loss": 3.4827415806512656, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000352, + "loss": 2.6611, + "theoretical_loss": 3.4827298031218397, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000354, + "loss": 2.5624, + "theoretical_loss": 3.48271802618289, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000356, + "loss": 2.696, + "theoretical_loss": 3.4827062498343637, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.01, + "learning_rate": 0.000358, + "loss": 2.576, + "theoretical_loss": 3.4826944740762085, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035999999999999997, + "loss": 2.8337, + "theoretical_loss": 3.4826826989083712, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.01, + "learning_rate": 0.000362, + "loss": 2.8242, + "theoretical_loss": 3.4826709243307996, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000364, + "loss": 2.623, + "theoretical_loss": 3.4826591503434408, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000366, + "loss": 2.6458, + "theoretical_loss": 3.4826473769462414, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.01, + "learning_rate": 0.000368, + "loss": 2.6256, + "theoretical_loss": 3.4826356041391495, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037, + "loss": 2.613, + "theoretical_loss": 3.482623831922113, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.01, + "learning_rate": 0.000372, + "loss": 2.6292, + "theoretical_loss": 3.4826120602950774, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000374, + "loss": 2.6374, + "theoretical_loss": 3.4826002892579915, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037600000000000003, + "loss": 2.4571, + "theoretical_loss": 3.482588518810802, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.01, + "learning_rate": 0.000378, + "loss": 2.5032, + "theoretical_loss": 3.4825767489534574, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038, + "loss": 2.6598, + "theoretical_loss": 3.482564979685903, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.01, + "learning_rate": 0.000382, + "loss": 2.6801, + "theoretical_loss": 3.482553211008088, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000384, + "loss": 2.7927, + "theoretical_loss": 3.482541442919959, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.01, + "learning_rate": 0.000386, + "loss": 2.8021, + "theoretical_loss": 3.482529675421463, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.01, + "learning_rate": 0.000388, + "loss": 2.3866, + "theoretical_loss": 3.4825179085125484, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039000000000000005, + "loss": 2.5351, + "theoretical_loss": 3.4825061421931616, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039200000000000004, + "loss": 2.5761, + "theoretical_loss": 3.4824943764632503, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039400000000000004, + "loss": 2.5201, + "theoretical_loss": 3.4824826113227623, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039600000000000003, + "loss": 2.603, + "theoretical_loss": 3.4824708467716445, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.01, + "learning_rate": 0.000398, + "loss": 2.428, + "theoretical_loss": 3.4824590828098447, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 948408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7703728675842285, + "objective/train/theoretical_loss": 3.48244731943731, + "objective/train/tokens_used": 33566176, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 2.8112, + "theoretical_loss": 3.48244731943731, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000402, + "loss": 2.536, + "theoretical_loss": 3.4824355566539875, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000404, + "loss": 2.3951, + "theoretical_loss": 3.4824237944598257, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040600000000000006, + "loss": 2.5714, + "theoretical_loss": 3.482412032854771, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000408, + "loss": 2.7056, + "theoretical_loss": 3.482400271838771, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041, + "loss": 2.6448, + "theoretical_loss": 3.482388511411774, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000412, + "loss": 2.4682, + "theoretical_loss": 3.4823767515737267, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.01, + "learning_rate": 0.000414, + "loss": 2.6172, + "theoretical_loss": 3.4823649923245767, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.01, + "learning_rate": 0.000416, + "loss": 2.711, + "theoretical_loss": 3.4823532336642717, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041799999999999997, + "loss": 2.7001, + "theoretical_loss": 3.482341475592759, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042, + "loss": 2.7769, + "theoretical_loss": 3.4823297181099857, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.01, + "learning_rate": 0.000422, + "loss": 2.482, + "theoretical_loss": 3.4823179612159, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000424, + "loss": 2.5695, + "theoretical_loss": 3.4823062049104485, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426, + "loss": 2.634, + "theoretical_loss": 3.4822944491935797, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000428, + "loss": 2.6891, + "theoretical_loss": 3.4822826940652405, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043, + "loss": 2.7086, + "theoretical_loss": 3.482270939525379, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.01, + "learning_rate": 0.000432, + "loss": 2.5677, + "theoretical_loss": 3.482259185573942, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043400000000000003, + "loss": 2.7734, + "theoretical_loss": 3.4822474322108774, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.01, + "learning_rate": 0.000436, + "loss": 2.6515, + "theoretical_loss": 3.4822356794361324, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.01, + "learning_rate": 0.000438, + "loss": 2.715, + "theoretical_loss": 3.482223927249655, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044, + "loss": 2.4302, + "theoretical_loss": 3.482212175651393, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000442, + "loss": 2.5429, + "theoretical_loss": 3.4822004246412934, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000444, + "loss": 2.6395, + "theoretical_loss": 3.4821886742193042, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.01, + "learning_rate": 0.000446, + "loss": 2.6917, + "theoretical_loss": 3.4821769243853717, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000448, + "loss": 2.4181, + "theoretical_loss": 3.4821651751394453, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 949481, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6700518131256104, + "objective/train/theoretical_loss": 3.4821534264814717, + "objective/train/tokens_used": 35204576, + "theoretical_loss": 3.4821534264814717, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045000000000000004, + "loss": 2.6615, + "theoretical_loss": 3.4821534264814717, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045200000000000004, + "loss": 2.6455, + "theoretical_loss": 3.4821416784113985, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045400000000000003, + "loss": 2.5993, + "theoretical_loss": 3.4821299309291733, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000456, + "loss": 2.6412, + "theoretical_loss": 3.4821181840347437, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.01, + "learning_rate": 0.000458, + "loss": 2.6212, + "theoretical_loss": 3.4821064377280577, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046, + "loss": 2.5679, + "theoretical_loss": 3.4820946920090625, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.01, + "learning_rate": 0.000462, + "loss": 2.6689, + "theoretical_loss": 3.4820829468777057, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046400000000000006, + "loss": 2.5793, + "theoretical_loss": 3.482071202333935, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046600000000000005, + "loss": 2.4957, + "theoretical_loss": 3.482059458377698, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046800000000000005, + "loss": 2.5759, + "theoretical_loss": 3.4820477150089433, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047, + "loss": 2.5035, + "theoretical_loss": 3.4820359722276164, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.01, + "learning_rate": 0.000472, + "loss": 2.7539, + "theoretical_loss": 3.4820242300336677, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.01, + "learning_rate": 0.000474, + "loss": 2.6753, + "theoretical_loss": 3.4820124884270425, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047599999999999997, + "loss": 2.7376, + "theoretical_loss": 3.4820007474076893, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047799999999999996, + "loss": 2.6736, + "theoretical_loss": 3.481989006975556, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048, + "loss": 2.8743, + "theoretical_loss": 3.4819772671305906, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000482, + "loss": 2.2899, + "theoretical_loss": 3.48196552787274, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.01, + "learning_rate": 0.000484, + "loss": 2.6535, + "theoretical_loss": 3.4819537892019525, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.01, + "learning_rate": 0.000486, + "loss": 2.5608, + "theoretical_loss": 3.4819420511181756, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488, + "loss": 2.6851, + "theoretical_loss": 3.481930313621357, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049, + "loss": 2.7921, + "theoretical_loss": 3.4819185767114442, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492, + "loss": 2.749, + "theoretical_loss": 3.4819068403883855, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000494, + "loss": 2.5647, + "theoretical_loss": 3.4818951046521276, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496, + "loss": 2.5925, + "theoretical_loss": 3.4818833695026195, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 2.6431, + "theoretical_loss": 3.4818716349398082, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 950694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.532388925552368, + "objective/train/theoretical_loss": 3.481859900963642, + "objective/train/tokens_used": 36842976, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 2.61, + "theoretical_loss": 3.481859900963642, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499979797979798, + "loss": 2.9134, + "theoretical_loss": 3.481848167574068, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999595959595959, + "loss": 2.7905, + "theoretical_loss": 3.481836434771034, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999393939393939, + "loss": 2.519, + "theoretical_loss": 3.4818247025544884, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499919191919192, + "loss": 2.7406, + "theoretical_loss": 3.481812970924379, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049989898989899, + "loss": 2.6038, + "theoretical_loss": 3.481801239880652, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998787878787879, + "loss": 2.569, + "theoretical_loss": 3.4817895094232574, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998585858585859, + "loss": 2.9226, + "theoretical_loss": 3.481777779552141, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998383838383839, + "loss": 2.4896, + "theoretical_loss": 3.481766050267253, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998181818181818, + "loss": 2.646, + "theoretical_loss": 3.481754321568539, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997979797979798, + "loss": 2.6977, + "theoretical_loss": 3.4817425934559476, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997777777777778, + "loss": 2.7864, + "theoretical_loss": 3.4817308659294266, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997575757575757, + "loss": 2.6019, + "theoretical_loss": 3.4817191389889244, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997373737373737, + "loss": 2.6333, + "theoretical_loss": 3.481707412634388, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997171717171717, + "loss": 2.5995, + "theoretical_loss": 3.481695686865766, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996969696969697, + "loss": 2.7652, + "theoretical_loss": 3.481683961683005, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996767676767676, + "loss": 2.4889, + "theoretical_loss": 3.4816722370860544, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996565656565657, + "loss": 2.7457, + "theoretical_loss": 3.4816605130748615, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996363636363637, + "loss": 2.5843, + "theoretical_loss": 3.4816487896493733, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996161616161617, + "loss": 2.814, + "theoretical_loss": 3.4816370668095393, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995959595959596, + "loss": 2.8138, + "theoretical_loss": 3.4816253445553063, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995757575757576, + "loss": 2.7419, + "theoretical_loss": 3.4816136228866217, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995555555555555, + "loss": 2.8775, + "theoretical_loss": 3.481601901803435, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995353535353536, + "loss": 2.7713, + "theoretical_loss": 3.481590181305693, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995151515151515, + "loss": 2.6531, + "theoretical_loss": 3.4815784613933434, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 951339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5657424926757812, + "objective/train/theoretical_loss": 3.481566742066335, + "objective/train/tokens_used": 38481376, + "theoretical_loss": 3.481566742066335, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994949494949495, + "loss": 2.4996, + "theoretical_loss": 3.481566742066335, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994747474747474, + "loss": 2.6613, + "theoretical_loss": 3.4815550233246153, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994545454545454, + "loss": 2.6106, + "theoretical_loss": 3.481543305168132, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994343434343435, + "loss": 2.5593, + "theoretical_loss": 3.4815315875968333, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994141414141415, + "loss": 2.9138, + "theoretical_loss": 3.4815198706106676, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993939393939394, + "loss": 2.5428, + "theoretical_loss": 3.481508154209582, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993737373737374, + "loss": 2.886, + "theoretical_loss": 3.4814964383935245, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993535353535354, + "loss": 2.7435, + "theoretical_loss": 3.4814847231624437, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993333333333334, + "loss": 2.7701, + "theoretical_loss": 3.4814730085162875, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993131313131313, + "loss": 2.5468, + "theoretical_loss": 3.4814612944550034, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992929292929293, + "loss": 2.5983, + "theoretical_loss": 3.4814495809785395, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992727272727272, + "loss": 2.4265, + "theoretical_loss": 3.4814378680868447, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992525252525252, + "loss": 2.9062, + "theoretical_loss": 3.4814261557798654, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992323232323232, + "loss": 2.6454, + "theoretical_loss": 3.4814144440575507, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992121212121212, + "loss": 2.8316, + "theoretical_loss": 3.4814027329198485, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991919191919192, + "loss": 2.5862, + "theoretical_loss": 3.481391022366707, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991717171717172, + "loss": 2.7143, + "theoretical_loss": 3.481379312398073, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991515151515152, + "loss": 2.5823, + "theoretical_loss": 3.4813676030138963, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991313131313132, + "loss": 2.2637, + "theoretical_loss": 3.481355894214124, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991111111111111, + "loss": 2.7928, + "theoretical_loss": 3.481344185998704, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990909090909091, + "loss": 2.6647, + "theoretical_loss": 3.4813324783675847, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990707070707071, + "loss": 2.6527, + "theoretical_loss": 3.481320771320714, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990505050505051, + "loss": 2.6264, + "theoretical_loss": 3.4813090648580403, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499030303030303, + "loss": 2.5514, + "theoretical_loss": 3.4812973589795106, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499010101010101, + "loss": 2.9298, + "theoretical_loss": 3.4812856536850747, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 952727, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6764137744903564, + "objective/train/theoretical_loss": 3.4812739489746796, + "objective/train/tokens_used": 40119776, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498989898989899, + "loss": 2.7139, + "theoretical_loss": 3.4812739489746796, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498969696969697, + "loss": 2.6319, + "theoretical_loss": 3.4812622448482733, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498949494949495, + "loss": 2.8492, + "theoretical_loss": 3.481250541305804, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498929292929293, + "loss": 2.7347, + "theoretical_loss": 3.4812388383472204, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989090909090909, + "loss": 2.5917, + "theoretical_loss": 3.4812271359724702, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988888888888889, + "loss": 2.7193, + "theoretical_loss": 3.481215434181501, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988686868686869, + "loss": 2.5198, + "theoretical_loss": 3.481203732974262, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988484848484849, + "loss": 2.8625, + "theoretical_loss": 3.481192032350701, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988282828282828, + "loss": 2.7043, + "theoretical_loss": 3.481180332310765, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988080808080808, + "loss": 2.685, + "theoretical_loss": 3.481168632854404, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987878787878788, + "loss": 2.7342, + "theoretical_loss": 3.4811569339815644, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987676767676767, + "loss": 2.7771, + "theoretical_loss": 3.4811452356921953, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987474747474747, + "loss": 2.875, + "theoretical_loss": 3.4811335379862456, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987272727272728, + "loss": 2.6466, + "theoretical_loss": 3.4811218408636617, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987070707070708, + "loss": 2.6764, + "theoretical_loss": 3.481110144324393, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986868686868687, + "loss": 2.6408, + "theoretical_loss": 3.481098448368387, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986666666666667, + "loss": 2.7324, + "theoretical_loss": 3.4810867529955933, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986464646464647, + "loss": 2.5388, + "theoretical_loss": 3.481075058205958, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986262626262627, + "loss": 2.763, + "theoretical_loss": 3.4810633639994313, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986060606060606, + "loss": 2.6408, + "theoretical_loss": 3.48105167037596, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985858585858586, + "loss": 2.7675, + "theoretical_loss": 3.4810399773354925, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656565656565, + "loss": 2.6876, + "theoretical_loss": 3.4810282848779783, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985454545454545, + "loss": 2.6173, + "theoretical_loss": 3.481016593003364, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985252525252525, + "loss": 2.5695, + "theoretical_loss": 3.4810049017115983, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985050505050506, + "loss": 2.8194, + "theoretical_loss": 3.4809932110026303, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 953335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.877380132675171, + "objective/train/theoretical_loss": 3.4809815208764077, + "objective/train/tokens_used": 41758176, + "theoretical_loss": 3.4809815208764077, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984848484848485, + "loss": 2.6744, + "theoretical_loss": 3.4809815208764077, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984646464646465, + "loss": 2.7991, + "theoretical_loss": 3.480969831332878, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984444444444445, + "loss": 2.8559, + "theoretical_loss": 3.4809581423719904, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984242424242425, + "loss": 2.6315, + "theoretical_loss": 3.480946453993693, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984040404040404, + "loss": 2.7081, + "theoretical_loss": 3.480934766197934, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983838383838384, + "loss": 2.7087, + "theoretical_loss": 3.480923078984662, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983636363636363, + "loss": 2.795, + "theoretical_loss": 3.480911392353825, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983434343434344, + "loss": 2.7585, + "theoretical_loss": 3.4808997063053706, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983232323232323, + "loss": 2.5014, + "theoretical_loss": 3.480888020839248, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983030303030303, + "loss": 2.6384, + "theoretical_loss": 3.480876335955406, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982828282828282, + "loss": 2.8254, + "theoretical_loss": 3.4808646516537918, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982626262626263, + "loss": 2.7822, + "theoretical_loss": 3.4808529679343536, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982424242424243, + "loss": 2.8832, + "theoretical_loss": 3.480841284797041, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982222222222223, + "loss": 2.5462, + "theoretical_loss": 3.4808296022418013, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982020202020202, + "loss": 2.4467, + "theoretical_loss": 3.480817920268583, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981818181818182, + "loss": 2.7244, + "theoretical_loss": 3.480806238877335, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981616161616162, + "loss": 2.5184, + "theoretical_loss": 3.4807945580680055, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981414141414142, + "loss": 2.5498, + "theoretical_loss": 3.480782877840542, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981212121212121, + "loss": 2.7517, + "theoretical_loss": 3.480771198194894, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981010101010101, + "loss": 2.604, + "theoretical_loss": 3.4807595191310092, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498080808080808, + "loss": 2.7308, + "theoretical_loss": 3.4807478406488364, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498060606060606, + "loss": 2.8728, + "theoretical_loss": 3.4807361627483235, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498040404040404, + "loss": 2.7389, + "theoretical_loss": 3.4807244854294197, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980202020202021, + "loss": 2.716, + "theoretical_loss": 3.480712808692072, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498, + "loss": 2.7822, + "theoretical_loss": 3.4807011325362307, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 954676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.085847854614258, + "objective/train/theoretical_loss": 3.4806894569618425, + "objective/train/tokens_used": 43396576, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497979797979798, + "loss": 2.4359, + "theoretical_loss": 3.4806894569618425, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497959595959596, + "loss": 2.637, + "theoretical_loss": 3.480677781968857, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497939393939394, + "loss": 2.7507, + "theoretical_loss": 3.480666107557221, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979191919191919, + "loss": 2.7732, + "theoretical_loss": 3.4806544337268854, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978989898989899, + "loss": 2.6741, + "theoretical_loss": 3.4806427604777967, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978787878787879, + "loss": 2.6305, + "theoretical_loss": 3.4806310878099045, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978585858585858, + "loss": 2.7524, + "theoretical_loss": 3.4806194157231567, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978383838383838, + "loss": 2.6401, + "theoretical_loss": 3.480607744217502, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978181818181818, + "loss": 2.8563, + "theoretical_loss": 3.4805960732928876, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977979797979799, + "loss": 2.8258, + "theoretical_loss": 3.480584402949264, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977777777777778, + "loss": 2.8557, + "theoretical_loss": 3.4805727331865786, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977575757575758, + "loss": 2.8474, + "theoretical_loss": 3.48056106400478, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977373737373738, + "loss": 2.9585, + "theoretical_loss": 3.480549395403817, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977171717171717, + "loss": 2.6838, + "theoretical_loss": 3.4805377273836378, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976969696969697, + "loss": 2.611, + "theoretical_loss": 3.4805260599441903, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976767676767677, + "loss": 2.5526, + "theoretical_loss": 3.480514393085424, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976565656565656, + "loss": 2.6395, + "theoretical_loss": 3.4805027268072877, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976363636363636, + "loss": 2.6906, + "theoretical_loss": 3.4804910611097286, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976161616161616, + "loss": 2.6872, + "theoretical_loss": 3.4804793959926963, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975959595959596, + "loss": 2.5617, + "theoretical_loss": 3.480467731456139, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975757575757575, + "loss": 2.5815, + "theoretical_loss": 3.4804560675000054, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975555555555556, + "loss": 2.5488, + "theoretical_loss": 3.4804444041242437, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975353535353536, + "loss": 2.7545, + "theoretical_loss": 3.480432741328803, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975151515151516, + "loss": 2.6994, + "theoretical_loss": 3.4804210791136314, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974949494949495, + "loss": 2.8704, + "theoretical_loss": 3.4804094174786773, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 955358, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8682773113250732, + "objective/train/theoretical_loss": 3.4803977564238897, + "objective/train/tokens_used": 45034976, + "theoretical_loss": 3.4803977564238897, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974747474747475, + "loss": 2.6428, + "theoretical_loss": 3.4803977564238897, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974545454545454, + "loss": 2.5452, + "theoretical_loss": 3.4803860959492177, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974343434343435, + "loss": 2.7287, + "theoretical_loss": 3.4803744360546087, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974141414141414, + "loss": 2.8716, + "theoretical_loss": 3.4803627767400123, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973939393939394, + "loss": 2.8354, + "theoretical_loss": 3.480351118005376, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973737373737373, + "loss": 2.8261, + "theoretical_loss": 3.48033945985065, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973535353535353, + "loss": 2.6104, + "theoretical_loss": 3.4803278022757818, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973333333333334, + "loss": 2.7865, + "theoretical_loss": 3.4803161452807196, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973131313131314, + "loss": 2.5254, + "theoretical_loss": 3.4803044888654133, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972929292929293, + "loss": 2.6092, + "theoretical_loss": 3.480292833029811, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972727272727273, + "loss": 2.89, + "theoretical_loss": 3.480281177773861, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972525252525253, + "loss": 2.6683, + "theoretical_loss": 3.4802695230975127, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972323232323233, + "loss": 2.73, + "theoretical_loss": 3.480257869000714, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972121212121212, + "loss": 2.6901, + "theoretical_loss": 3.4802462154834135, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971919191919192, + "loss": 2.5487, + "theoretical_loss": 3.4802345625455606, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971717171717171, + "loss": 2.7714, + "theoretical_loss": 3.480222910187104, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971515151515152, + "loss": 2.7269, + "theoretical_loss": 3.4802112584079916, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313131313131, + "loss": 2.5853, + "theoretical_loss": 3.4801996072081725, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971111111111111, + "loss": 2.8781, + "theoretical_loss": 3.4801879565875957, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497090909090909, + "loss": 2.5268, + "theoretical_loss": 3.4801763065462095, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970707070707071, + "loss": 3.0031, + "theoretical_loss": 3.4801646570839626, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970505050505051, + "loss": 2.7106, + "theoretical_loss": 3.480153008200804, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970303030303031, + "loss": 2.6904, + "theoretical_loss": 3.480141359896683, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497010101010101, + "loss": 2.6185, + "theoretical_loss": 3.4801297121715464, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496989898989899, + "loss": 2.5314, + "theoretical_loss": 3.4801180650253447, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 956671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5481631755828857, + "objective/train/theoretical_loss": 3.480106418458026, + "objective/train/tokens_used": 46673376, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496969696969697, + "loss": 2.8021, + "theoretical_loss": 3.480106418458026, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496949494949495, + "loss": 2.7096, + "theoretical_loss": 3.4800947724695392, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969292929292929, + "loss": 2.5675, + "theoretical_loss": 3.4800831270598334, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969090909090909, + "loss": 2.9144, + "theoretical_loss": 3.4800714822288565, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968888888888889, + "loss": 2.6917, + "theoretical_loss": 3.4800598379765577, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968686868686868, + "loss": 2.5936, + "theoretical_loss": 3.4800481943028863, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968484848484849, + "loss": 2.7713, + "theoretical_loss": 3.48003655120779, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968282828282829, + "loss": 2.8709, + "theoretical_loss": 3.4800249086912185, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968080808080808, + "loss": 2.6705, + "theoretical_loss": 3.4800132667531205, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967878787878788, + "loss": 2.8826, + "theoretical_loss": 3.4800016253934447, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967676767676768, + "loss": 2.5403, + "theoretical_loss": 3.4799899846121396, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967474747474748, + "loss": 2.8197, + "theoretical_loss": 3.4799783444091545, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967272727272727, + "loss": 2.8001, + "theoretical_loss": 3.4799667047844376, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967070707070707, + "loss": 2.7365, + "theoretical_loss": 3.479955065737938, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966868686868687, + "loss": 2.8378, + "theoretical_loss": 3.479943427269605, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966666666666666, + "loss": 2.9886, + "theoretical_loss": 3.4799317893793873, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966464646464646, + "loss": 2.7944, + "theoretical_loss": 3.479920152067233, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966262626262627, + "loss": 2.5664, + "theoretical_loss": 3.4799085153330918, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966060606060607, + "loss": 2.8499, + "theoretical_loss": 3.479896879176912, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965858585858586, + "loss": 2.6628, + "theoretical_loss": 3.479885243598643, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965656565656566, + "loss": 2.5959, + "theoretical_loss": 3.4798736085982336, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965454545454546, + "loss": 2.8731, + "theoretical_loss": 3.479861974175632, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965252525252525, + "loss": 2.8918, + "theoretical_loss": 3.4798503403307874, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965050505050505, + "loss": 2.5307, + "theoretical_loss": 3.4798387070636494, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964848484848485, + "loss": 2.9082, + "theoretical_loss": 3.479827074374166, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 957370, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.806737184524536, + "objective/train/theoretical_loss": 3.4798154422622867, + "objective/train/tokens_used": 48311776, + "theoretical_loss": 3.4798154422622867, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964646464646464, + "loss": 2.7645, + "theoretical_loss": 3.4798154422622867, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964444444444444, + "loss": 2.7847, + "theoretical_loss": 3.4798038107279603, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964242424242424, + "loss": 2.8354, + "theoretical_loss": 3.4797921797711355, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964040404040405, + "loss": 2.7748, + "theoretical_loss": 3.4797805493917613, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963838383838384, + "loss": 2.8072, + "theoretical_loss": 3.4797689195897865, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963636363636364, + "loss": 2.9036, + "theoretical_loss": 3.4797572903651606, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963434343434344, + "loss": 2.7364, + "theoretical_loss": 3.479745661717832, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963232323232324, + "loss": 2.6744, + "theoretical_loss": 3.47973403364775, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963030303030303, + "loss": 2.5627, + "theoretical_loss": 3.4797224061548633, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962828282828283, + "loss": 2.7274, + "theoretical_loss": 3.4797107792391206, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962626262626262, + "loss": 2.5577, + "theoretical_loss": 3.4796991529004715, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962424242424243, + "loss": 2.7214, + "theoretical_loss": 3.4796875271388648, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962222222222222, + "loss": 2.628, + "theoretical_loss": 3.4796759019542494, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962020202020202, + "loss": 2.6974, + "theoretical_loss": 3.479664277346574, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961818181818181, + "loss": 2.7414, + "theoretical_loss": 3.479652653315788, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961616161616161, + "loss": 2.6069, + "theoretical_loss": 3.4796410298618405, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961414141414142, + "loss": 2.5844, + "theoretical_loss": 3.4796294069846803, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961212121212122, + "loss": 2.5572, + "theoretical_loss": 3.4796177846842564, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961010101010101, + "loss": 2.6523, + "theoretical_loss": 3.479606162960518, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960808080808081, + "loss": 2.9691, + "theoretical_loss": 3.4795945418134133, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960606060606061, + "loss": 2.8851, + "theoretical_loss": 3.479582921242893, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960404040404041, + "loss": 2.5608, + "theoretical_loss": 3.4795713012489045, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496020202020202, + "loss": 2.7754, + "theoretical_loss": 3.4795596818313976, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496, + "loss": 2.7357, + "theoretical_loss": 3.4795480629903217, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959797979797979, + "loss": 2.8105, + "theoretical_loss": 3.479536444725625, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 958661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6299262046813965, + "objective/train/theoretical_loss": 3.4795248270372574, + "objective/train/tokens_used": 49950176, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959595959595959, + "loss": 2.7366, + "theoretical_loss": 3.4795248270372574, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495939393939394, + "loss": 2.8606, + "theoretical_loss": 3.479513209925167, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495919191919192, + "loss": 2.4786, + "theoretical_loss": 3.479501593389304, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958989898989899, + "loss": 2.9367, + "theoretical_loss": 3.479489977429617, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958787878787879, + "loss": 2.8186, + "theoretical_loss": 3.4794783620460548, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958585858585859, + "loss": 2.896, + "theoretical_loss": 3.479466747238567, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958383838383839, + "loss": 2.6276, + "theoretical_loss": 3.4794551330071024, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958181818181818, + "loss": 2.7327, + "theoretical_loss": 3.47944351935161, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957979797979798, + "loss": 2.7203, + "theoretical_loss": 3.4794319062720396, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957777777777778, + "loss": 2.8663, + "theoretical_loss": 3.479420293768339, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957575757575757, + "loss": 2.7838, + "theoretical_loss": 3.479408681840459, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957373737373737, + "loss": 2.644, + "theoretical_loss": 3.4793970704883477, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171717171717, + "loss": 2.6502, + "theoretical_loss": 3.4793854597119545, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956969696969698, + "loss": 2.8993, + "theoretical_loss": 3.4793738495112283, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956767676767677, + "loss": 2.3379, + "theoretical_loss": 3.4793622398861186, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956565656565657, + "loss": 2.656, + "theoretical_loss": 3.4793506308365747, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956363636363637, + "loss": 2.6013, + "theoretical_loss": 3.479339022362545, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956161616161616, + "loss": 2.7012, + "theoretical_loss": 3.47932741446398, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955959595959596, + "loss": 2.6748, + "theoretical_loss": 3.4793158071408277, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955757575757576, + "loss": 2.8417, + "theoretical_loss": 3.4793042003930377, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955555555555556, + "loss": 2.5976, + "theoretical_loss": 3.479292594220559, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955353535353535, + "loss": 2.7204, + "theoretical_loss": 3.4792809886233416, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955151515151515, + "loss": 2.7222, + "theoretical_loss": 3.4792693836013333, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954949494949495, + "loss": 2.799, + "theoretical_loss": 3.4792577791544845, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954747474747474, + "loss": 2.8752, + "theoretical_loss": 3.4792461752827446, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 959159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6778666973114014, + "objective/train/theoretical_loss": 3.479234571986062, + "objective/train/tokens_used": 51588576, + "theoretical_loss": 3.479234571986062, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954545454545455, + "loss": 2.6791, + "theoretical_loss": 3.479234571986062, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954343434343435, + "loss": 2.7594, + "theoretical_loss": 3.479222969264386, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954141414141415, + "loss": 2.6786, + "theoretical_loss": 3.4792113671176663, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953939393939394, + "loss": 2.7265, + "theoretical_loss": 3.479199765545852, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953737373737374, + "loss": 2.6815, + "theoretical_loss": 3.4791881645488916, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953535353535354, + "loss": 2.8073, + "theoretical_loss": 3.479176564126736, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953333333333334, + "loss": 2.5998, + "theoretical_loss": 3.4791649642793328, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953131313131313, + "loss": 2.7401, + "theoretical_loss": 3.479153365006632, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952929292929293, + "loss": 2.6499, + "theoretical_loss": 3.479141766308584, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952727272727272, + "loss": 2.4676, + "theoretical_loss": 3.479130168185136, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952525252525252, + "loss": 2.7173, + "theoretical_loss": 3.4791185706362384, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952323232323232, + "loss": 2.6965, + "theoretical_loss": 3.4791069736618407, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952121212121213, + "loss": 2.7766, + "theoretical_loss": 3.479095377261892, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951919191919192, + "loss": 2.7344, + "theoretical_loss": 3.4790837814363407, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951717171717172, + "loss": 2.6926, + "theoretical_loss": 3.479072186185138, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951515151515152, + "loss": 2.6573, + "theoretical_loss": 3.4790605915082313, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951313131313132, + "loss": 2.5615, + "theoretical_loss": 3.479048997405571, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951111111111111, + "loss": 2.7683, + "theoretical_loss": 3.479037403877107, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950909090909091, + "loss": 2.6245, + "theoretical_loss": 3.4790258109227867, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495070707070707, + "loss": 2.6609, + "theoretical_loss": 3.479014218542561, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950505050505051, + "loss": 2.5353, + "theoretical_loss": 3.4790026267363796, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495030303030303, + "loss": 2.7477, + "theoretical_loss": 3.478991035504191, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495010101010101, + "loss": 2.6838, + "theoretical_loss": 3.4789794448459443, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494989898989899, + "loss": 2.7132, + "theoretical_loss": 3.4789678547615894, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494969696969697, + "loss": 2.7488, + "theoretical_loss": 3.478956265251076, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 960536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3512511253356934, + "objective/train/theoretical_loss": 3.478944676314353, + "objective/train/tokens_used": 53226976, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494949494949495, + "loss": 2.6821, + "theoretical_loss": 3.478944676314353, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494929292929293, + "loss": 2.9596, + "theoretical_loss": 3.4789330879513694, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949090909090909, + "loss": 2.7531, + "theoretical_loss": 3.478921500162076, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948888888888889, + "loss": 2.6802, + "theoretical_loss": 3.4789099129464205, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948686868686869, + "loss": 2.667, + "theoretical_loss": 3.4788983263043534, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948484848484849, + "loss": 2.8355, + "theoretical_loss": 3.478886740235824, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948282828282828, + "loss": 2.8359, + "theoretical_loss": 3.4788751547407815, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948080808080808, + "loss": 2.8404, + "theoretical_loss": 3.478863569819176, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947878787878787, + "loss": 2.6452, + "theoretical_loss": 3.478851985470956, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947676767676767, + "loss": 2.8162, + "theoretical_loss": 3.478840401696071, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947474747474748, + "loss": 2.6697, + "theoretical_loss": 3.4788288184944713, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947272727272728, + "loss": 2.7656, + "theoretical_loss": 3.4788172358661056, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947070707070707, + "loss": 2.6518, + "theoretical_loss": 3.478805653810924, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946868686868687, + "loss": 2.5567, + "theoretical_loss": 3.478794072328875, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946666666666667, + "loss": 2.6502, + "theoretical_loss": 3.478782491419909, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946464646464647, + "loss": 2.5949, + "theoretical_loss": 3.4787709110839753, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946262626262626, + "loss": 2.7938, + "theoretical_loss": 3.478759331321023, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946060606060606, + "loss": 2.6666, + "theoretical_loss": 3.478747752131002, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945858585858586, + "loss": 2.6853, + "theoretical_loss": 3.4787361735138624, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945656565656565, + "loss": 2.6651, + "theoretical_loss": 3.478724595469552, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945454545454545, + "loss": 2.6555, + "theoretical_loss": 3.4787130179980217, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945252525252526, + "loss": 2.7459, + "theoretical_loss": 3.478701441099221, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945050505050506, + "loss": 2.8495, + "theoretical_loss": 3.4786898647730986, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944848484848485, + "loss": 2.7032, + "theoretical_loss": 3.478678289019605, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944646464646465, + "loss": 2.8087, + "theoretical_loss": 3.478666713838689, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 961177, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.580803632736206, + "objective/train/theoretical_loss": 3.478655139230301, + "objective/train/tokens_used": 54865376, + "theoretical_loss": 3.478655139230301, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944444444444445, + "loss": 2.6687, + "theoretical_loss": 3.478655139230301, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944242424242424, + "loss": 2.658, + "theoretical_loss": 3.478643565194389, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944040404040404, + "loss": 2.6445, + "theoretical_loss": 3.478631991730904, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943838383838384, + "loss": 2.7122, + "theoretical_loss": 3.478620418839795, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943636363636363, + "loss": 2.8942, + "theoretical_loss": 3.4786088465210123, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943434343434343, + "loss": 2.7271, + "theoretical_loss": 3.478597274774504, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943232323232323, + "loss": 2.6342, + "theoretical_loss": 3.4785857036002215, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943030303030303, + "loss": 2.7143, + "theoretical_loss": 3.478574132998113, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828282828283, + "loss": 2.685, + "theoretical_loss": 3.478562562968129, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942626262626263, + "loss": 2.907, + "theoretical_loss": 3.4785509935102183, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942424242424243, + "loss": 2.6646, + "theoretical_loss": 3.4785394246243317, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942222222222223, + "loss": 2.9157, + "theoretical_loss": 3.478527856310417, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942020202020202, + "loss": 2.7783, + "theoretical_loss": 3.4785162885684255, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941818181818182, + "loss": 2.7898, + "theoretical_loss": 3.478504721398306, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941616161616161, + "loss": 2.7686, + "theoretical_loss": 3.4784931548000086, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941414141414142, + "loss": 2.759, + "theoretical_loss": 3.4784815887734823, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941212121212121, + "loss": 2.8057, + "theoretical_loss": 3.4784700233186774, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941010101010101, + "loss": 2.6089, + "theoretical_loss": 3.4784584584355436, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494080808080808, + "loss": 2.8811, + "theoretical_loss": 3.47844689412403, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494060606060606, + "loss": 2.8443, + "theoretical_loss": 3.4784353303840865, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940404040404041, + "loss": 2.7193, + "theoretical_loss": 3.478423767215663, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940202020202021, + "loss": 2.7096, + "theoretical_loss": 3.478412204618709, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494, + "loss": 2.6727, + "theoretical_loss": 3.4784006425931744, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493979797979798, + "loss": 2.5442, + "theoretical_loss": 3.478389081139009, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493959595959596, + "loss": 2.7096, + "theoretical_loss": 3.4783775202561618, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 961828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0153701305389404, + "objective/train/theoretical_loss": 3.4783659599445826, + "objective/train/tokens_used": 56503776, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493939393939394, + "loss": 2.7048, + "theoretical_loss": 3.4783659599445826, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939191919191919, + "loss": 2.5246, + "theoretical_loss": 3.478354400204222, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938989898989899, + "loss": 2.741, + "theoretical_loss": 3.4783428410350297, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938787878787878, + "loss": 2.6484, + "theoretical_loss": 3.4783312824369537, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938585858585859, + "loss": 2.8418, + "theoretical_loss": 3.478319724409946, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938383838383838, + "loss": 2.5867, + "theoretical_loss": 3.478308166953955, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938181818181819, + "loss": 2.7361, + "theoretical_loss": 3.4782966100689308, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937979797979798, + "loss": 2.7319, + "theoretical_loss": 3.4782850537548233, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937777777777778, + "loss": 2.6264, + "theoretical_loss": 3.478273498011582, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937575757575758, + "loss": 2.6073, + "theoretical_loss": 3.4782619428391564, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937373737373738, + "loss": 2.7967, + "theoretical_loss": 3.478250388237497, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937171717171717, + "loss": 2.6165, + "theoretical_loss": 3.478238834206553, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936969696969697, + "loss": 2.7607, + "theoretical_loss": 3.4782272807462746, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936767676767677, + "loss": 2.5925, + "theoretical_loss": 3.4782157278566115, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936565656565657, + "loss": 2.7032, + "theoretical_loss": 3.4782041755375133, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936363636363636, + "loss": 2.8028, + "theoretical_loss": 3.47819262378893, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936161616161616, + "loss": 2.8439, + "theoretical_loss": 3.478181072610811, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935959595959597, + "loss": 2.3605, + "theoretical_loss": 3.478169522003107, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935757575757576, + "loss": 2.5842, + "theoretical_loss": 3.478157971965767, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935555555555556, + "loss": 2.6084, + "theoretical_loss": 3.4781464224987415, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935353535353536, + "loss": 2.3504, + "theoretical_loss": 3.478134873601979, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935151515151515, + "loss": 2.7918, + "theoretical_loss": 3.4781233252754316, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934949494949495, + "loss": 2.707, + "theoretical_loss": 3.4781117775190467, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934747474747475, + "loss": 2.5683, + "theoretical_loss": 3.4781002303327764, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934545454545455, + "loss": 2.5424, + "theoretical_loss": 3.4780886837165688, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 963095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.469271183013916, + "objective/train/theoretical_loss": 3.4780771376703745, + "objective/train/tokens_used": 58142176, + "theoretical_loss": 3.4780771376703745, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934343434343434, + "loss": 2.6869, + "theoretical_loss": 3.4780771376703745, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934141414141414, + "loss": 2.5349, + "theoretical_loss": 3.4780655921941435, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933939393939394, + "loss": 2.6974, + "theoretical_loss": 3.4780540472878254, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933737373737373, + "loss": 2.7998, + "theoretical_loss": 3.47804250295137, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933535353535354, + "loss": 2.7086, + "theoretical_loss": 3.478030959184728, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933333333333334, + "loss": 2.7899, + "theoretical_loss": 3.4780194159878484, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933131313131314, + "loss": 2.8103, + "theoretical_loss": 3.4780078733606814, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932929292929293, + "loss": 2.6782, + "theoretical_loss": 3.477996331303177, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932727272727273, + "loss": 2.8208, + "theoretical_loss": 3.4779847898152854, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932525252525253, + "loss": 2.9004, + "theoretical_loss": 3.477973248896956, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932323232323232, + "loss": 2.9608, + "theoretical_loss": 3.4779617085481385, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932121212121212, + "loss": 2.6838, + "theoretical_loss": 3.4779501687687837, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931919191919192, + "loss": 2.831, + "theoretical_loss": 3.477938629558841, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931717171717171, + "loss": 2.8781, + "theoretical_loss": 3.4779270909182607, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931515151515151, + "loss": 2.7625, + "theoretical_loss": 3.4779155528469925, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931313131313131, + "loss": 2.6384, + "theoretical_loss": 3.477904015344986, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931111111111112, + "loss": 2.9184, + "theoretical_loss": 3.4778924784121914, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930909090909091, + "loss": 2.6385, + "theoretical_loss": 3.47788094204856, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930707070707071, + "loss": 2.772, + "theoretical_loss": 3.4778694062540394, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930505050505051, + "loss": 2.5837, + "theoretical_loss": 3.4778578710285815, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930303030303031, + "loss": 2.6547, + "theoretical_loss": 3.477846336372136, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493010101010101, + "loss": 3.115, + "theoretical_loss": 3.4778348022846517, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492989898989899, + "loss": 2.8655, + "theoretical_loss": 3.47782326876608, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929696969696969, + "loss": 2.6883, + "theoretical_loss": 3.4778117358163705, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492949494949495, + "loss": 2.7285, + "theoretical_loss": 3.4778002034354722, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 963811, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5752992630004883, + "objective/train/theoretical_loss": 3.477788671623337, + "objective/train/tokens_used": 59780576, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929292929292929, + "loss": 2.7345, + "theoretical_loss": 3.477788671623337, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929090909090909, + "loss": 2.8721, + "theoretical_loss": 3.4777771403799136, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928888888888888, + "loss": 2.8679, + "theoretical_loss": 3.4777656097051524, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686868686869, + "loss": 2.8323, + "theoretical_loss": 3.4777540795990034, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928484848484849, + "loss": 2.5528, + "theoretical_loss": 3.477742550061417, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928282828282829, + "loss": 2.599, + "theoretical_loss": 3.4777310210923424, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928080808080808, + "loss": 2.752, + "theoretical_loss": 3.4777194926917305, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927878787878788, + "loss": 2.714, + "theoretical_loss": 3.4777079648595315, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927676767676768, + "loss": 2.8395, + "theoretical_loss": 3.4776964375956947, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927474747474748, + "loss": 2.6313, + "theoretical_loss": 3.477684910900171, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927272727272727, + "loss": 2.6812, + "theoretical_loss": 3.4776733847729098, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927070707070707, + "loss": 2.6424, + "theoretical_loss": 3.4776618592138613, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926868686868686, + "loss": 2.8452, + "theoretical_loss": 3.477650334222976, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926666666666666, + "loss": 2.6647, + "theoretical_loss": 3.477638809800204, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926464646464647, + "loss": 2.5417, + "theoretical_loss": 3.477627285945495, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926262626262627, + "loss": 2.5457, + "theoretical_loss": 3.4776157626587993, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926060606060606, + "loss": 2.7281, + "theoretical_loss": 3.4776042399400673, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925858585858586, + "loss": 2.5789, + "theoretical_loss": 3.477592717789248, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925656565656566, + "loss": 2.5252, + "theoretical_loss": 3.4775811962062937, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925454545454546, + "loss": 2.7796, + "theoretical_loss": 3.4775696751911527, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925252525252525, + "loss": 2.8003, + "theoretical_loss": 3.4775581547437757, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925050505050505, + "loss": 2.7379, + "theoretical_loss": 3.477546634864113, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924848484848485, + "loss": 2.8065, + "theoretical_loss": 3.477535115552115, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924646464646464, + "loss": 2.8194, + "theoretical_loss": 3.4775235968077314, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924444444444444, + "loss": 2.8824, + "theoretical_loss": 3.4775120786309124, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 965136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6549181938171387, + "objective/train/theoretical_loss": 3.4775005610216088, + "objective/train/tokens_used": 61418976, + "theoretical_loss": 3.4775005610216088, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924242424242425, + "loss": 2.6931, + "theoretical_loss": 3.4775005610216088, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924040404040405, + "loss": 2.6565, + "theoretical_loss": 3.4774890439797694, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923838383838384, + "loss": 2.8956, + "theoretical_loss": 3.4774775275053464, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923636363636364, + "loss": 2.7234, + "theoretical_loss": 3.4774660115982883, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923434343434344, + "loss": 2.7057, + "theoretical_loss": 3.477454496258546, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923232323232323, + "loss": 2.5114, + "theoretical_loss": 3.47744298148607, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923030303030303, + "loss": 2.5906, + "theoretical_loss": 3.4774314672808098, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922828282828283, + "loss": 2.6151, + "theoretical_loss": 3.477419953642716, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922626262626262, + "loss": 2.6406, + "theoretical_loss": 3.477408440571739, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922424242424242, + "loss": 2.5679, + "theoretical_loss": 3.4773969280678294, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922222222222222, + "loss": 2.3941, + "theoretical_loss": 3.4773854161309363, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922020202020202, + "loss": 2.6135, + "theoretical_loss": 3.477373904761011, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921818181818182, + "loss": 2.6847, + "theoretical_loss": 3.4773623939580034, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921616161616162, + "loss": 2.8308, + "theoretical_loss": 3.477350883721863, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921414141414142, + "loss": 2.6937, + "theoretical_loss": 3.477339374052542, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921212121212122, + "loss": 2.5952, + "theoretical_loss": 3.477327864949989, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921010101010101, + "loss": 2.7978, + "theoretical_loss": 3.4773163564141547, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920808080808081, + "loss": 2.7641, + "theoretical_loss": 3.4773048484449895, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920606060606061, + "loss": 2.7152, + "theoretical_loss": 3.4772933410424436, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920404040404041, + "loss": 2.9613, + "theoretical_loss": 3.477281834206468, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492020202020202, + "loss": 2.7052, + "theoretical_loss": 3.4772703279370116, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492, + "loss": 2.6769, + "theoretical_loss": 3.477258822234026, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919797979797979, + "loss": 2.6896, + "theoretical_loss": 3.4772473170974614, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491959595959596, + "loss": 2.5231, + "theoretical_loss": 3.477235812527267, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491939393939394, + "loss": 2.9854, + "theoretical_loss": 3.4772243085233945, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 965825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.397887706756592, + "objective/train/theoretical_loss": 3.4772128050857933, + "objective/train/tokens_used": 63057376, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491919191919192, + "loss": 2.5887, + "theoretical_loss": 3.4772128050857933, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918989898989899, + "loss": 2.7116, + "theoretical_loss": 3.4772013022144144, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918787878787879, + "loss": 2.7875, + "theoretical_loss": 3.4771897999092083, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918585858585859, + "loss": 3.0771, + "theoretical_loss": 3.4771782981701245, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918383838383839, + "loss": 2.7627, + "theoretical_loss": 3.4771667969971136, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918181818181818, + "loss": 2.6938, + "theoretical_loss": 3.4771552963901264, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917979797979798, + "loss": 2.7743, + "theoretical_loss": 3.477143796349113, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917777777777777, + "loss": 2.7977, + "theoretical_loss": 3.477132296874024, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917575757575758, + "loss": 2.7312, + "theoretical_loss": 3.47712079796481, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917373737373737, + "loss": 2.7644, + "theoretical_loss": 3.4771092996214206, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917171717171718, + "loss": 2.661, + "theoretical_loss": 3.4770978018438066, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916969696969697, + "loss": 2.5738, + "theoretical_loss": 3.4770863046319187, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916767676767677, + "loss": 2.8679, + "theoretical_loss": 3.4770748079857072, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916565656565657, + "loss": 2.9251, + "theoretical_loss": 3.4770633119051224, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916363636363637, + "loss": 2.6935, + "theoretical_loss": 3.4770518163901145, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916161616161616, + "loss": 2.6515, + "theoretical_loss": 3.4770403214406347, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915959595959596, + "loss": 2.5411, + "theoretical_loss": 3.4770288270566327, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915757575757576, + "loss": 2.7723, + "theoretical_loss": 3.477017333238059, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915555555555556, + "loss": 2.5689, + "theoretical_loss": 3.4770058399848645, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915353535353535, + "loss": 2.7889, + "theoretical_loss": 3.476994347296999, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915151515151515, + "loss": 2.644, + "theoretical_loss": 3.476982855174414, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914949494949494, + "loss": 2.7991, + "theoretical_loss": 3.476971363617059, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914747474747475, + "loss": 2.6404, + "theoretical_loss": 3.476959872624885, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914545454545455, + "loss": 2.7666, + "theoretical_loss": 3.4769483821978424, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343434343435, + "loss": 2.585, + "theoretical_loss": 3.4769368923358814, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 966739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.854780912399292, + "objective/train/theoretical_loss": 3.476925403038953, + "objective/train/tokens_used": 64695776, + "theoretical_loss": 3.476925403038953, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914141414141414, + "loss": 3.0976, + "theoretical_loss": 3.476925403038953, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913939393939394, + "loss": 2.8485, + "theoretical_loss": 3.4769139143070076, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913737373737374, + "loss": 2.7823, + "theoretical_loss": 3.476902426139995, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913535353535354, + "loss": 2.8487, + "theoretical_loss": 3.4768909385378666, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913333333333333, + "loss": 2.6317, + "theoretical_loss": 3.4768794515005723, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913131313131313, + "loss": 2.6958, + "theoretical_loss": 3.476867965028063, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912929292929293, + "loss": 2.5511, + "theoretical_loss": 3.47685647912029, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912727272727272, + "loss": 2.9583, + "theoretical_loss": 3.476844993777202, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912525252525252, + "loss": 2.4678, + "theoretical_loss": 3.476833508998751, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912323232323233, + "loss": 2.6116, + "theoretical_loss": 3.4768220247848873, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912121212121213, + "loss": 2.7879, + "theoretical_loss": 3.4768105411355608, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911919191919192, + "loss": 2.8152, + "theoretical_loss": 3.476799058050723, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911717171717172, + "loss": 2.7598, + "theoretical_loss": 3.4767875755303237, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911515151515152, + "loss": 2.4356, + "theoretical_loss": 3.476776093574314, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911313131313131, + "loss": 2.555, + "theoretical_loss": 3.4767646121826443, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911111111111111, + "loss": 2.6728, + "theoretical_loss": 3.4767531313552658, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910909090909091, + "loss": 2.699, + "theoretical_loss": 3.4767416510921274, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491070707070707, + "loss": 2.5046, + "theoretical_loss": 3.4767301713931813, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491050505050505, + "loss": 2.7166, + "theoretical_loss": 3.4767186922583777, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491030303030303, + "loss": 2.7154, + "theoretical_loss": 3.4767072136876673, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910101010101011, + "loss": 2.6896, + "theoretical_loss": 3.4766957356810004, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490989898989899, + "loss": 2.5532, + "theoretical_loss": 3.4766842582383277, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490969696969697, + "loss": 2.8812, + "theoretical_loss": 3.4766727813596, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490949494949495, + "loss": 2.7658, + "theoretical_loss": 3.476661305044768, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490929292929293, + "loss": 2.6469, + "theoretical_loss": 3.476649829293782, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 967356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6082024574279785, + "objective/train/theoretical_loss": 3.4766383541065933, + "objective/train/tokens_used": 66334176, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909090909090909, + "loss": 2.6874, + "theoretical_loss": 3.4766383541065933, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908888888888889, + "loss": 2.5705, + "theoretical_loss": 3.4766268794831516, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908686868686868, + "loss": 2.433, + "theoretical_loss": 3.4766154054234084, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908484848484849, + "loss": 2.6565, + "theoretical_loss": 3.4766039319273143, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908282828282828, + "loss": 2.5097, + "theoretical_loss": 3.4765924589948196, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908080808080808, + "loss": 2.4507, + "theoretical_loss": 3.4765809866258754, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907878787878787, + "loss": 2.6336, + "theoretical_loss": 3.476569514820432, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907676767676768, + "loss": 2.8124, + "theoretical_loss": 3.4765580435784402, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907474747474748, + "loss": 2.9799, + "theoretical_loss": 3.4765465728998506, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907272727272728, + "loss": 2.9473, + "theoretical_loss": 3.476535102784614, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907070707070707, + "loss": 2.5432, + "theoretical_loss": 3.4765236332326817, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906868686868687, + "loss": 2.512, + "theoretical_loss": 3.4765121642440038, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906666666666667, + "loss": 2.8169, + "theoretical_loss": 3.476500695818531, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906464646464647, + "loss": 2.8062, + "theoretical_loss": 3.476489227956214, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906262626262626, + "loss": 2.8448, + "theoretical_loss": 3.476477760657004, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906060606060606, + "loss": 2.7593, + "theoretical_loss": 3.4764662939208515, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905858585858585, + "loss": 2.5364, + "theoretical_loss": 3.476454827747707, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905656565656565, + "loss": 2.8466, + "theoretical_loss": 3.476443362137522, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905454545454546, + "loss": 2.6994, + "theoretical_loss": 3.4764318970902464, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905252525252526, + "loss": 2.9362, + "theoretical_loss": 3.4764204326058317, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905050505050505, + "loss": 2.626, + "theoretical_loss": 3.476408968684228, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904848484848485, + "loss": 2.719, + "theoretical_loss": 3.4763975053253864, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904646464646465, + "loss": 2.7069, + "theoretical_loss": 3.4763860425292576, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904444444444445, + "loss": 2.5189, + "theoretical_loss": 3.4763745802957926, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904242424242424, + "loss": 2.771, + "theoretical_loss": 3.476363118624943, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 968332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.779672622680664, + "objective/train/theoretical_loss": 3.4763516575166573, + "objective/train/tokens_used": 67972576, + "theoretical_loss": 3.4763516575166573, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904040404040404, + "loss": 2.6798, + "theoretical_loss": 3.4763516575166573, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903838383838384, + "loss": 2.6028, + "theoretical_loss": 3.476340196970889, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903636363636364, + "loss": 2.8995, + "theoretical_loss": 3.476328736987587, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903434343434343, + "loss": 2.7354, + "theoretical_loss": 3.476317277566703, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903232323232323, + "loss": 2.4798, + "theoretical_loss": 3.4763058187081874, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903030303030304, + "loss": 2.7076, + "theoretical_loss": 3.476294360411991, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902828282828283, + "loss": 2.7732, + "theoretical_loss": 3.4762829026780655, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902626262626263, + "loss": 2.8485, + "theoretical_loss": 3.4762714455063612, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902424242424243, + "loss": 2.6412, + "theoretical_loss": 3.4762599888968286, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902222222222222, + "loss": 2.6458, + "theoretical_loss": 3.476248532849419, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902020202020202, + "loss": 2.5095, + "theoretical_loss": 3.4762370773640834, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901818181818182, + "loss": 2.7252, + "theoretical_loss": 3.476225622440772, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901616161616162, + "loss": 2.676, + "theoretical_loss": 3.4762141680794367, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901414141414141, + "loss": 2.7279, + "theoretical_loss": 3.476202714280028, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901212121212121, + "loss": 2.7917, + "theoretical_loss": 3.476191261042496, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901010101010101, + "loss": 2.599, + "theoretical_loss": 3.4761798083667923, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490080808080808, + "loss": 2.9732, + "theoretical_loss": 3.476168356252868, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900606060606061, + "loss": 2.6708, + "theoretical_loss": 3.476156904700674, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900404040404041, + "loss": 2.9314, + "theoretical_loss": 3.476145453710161, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900202020202021, + "loss": 2.7338, + "theoretical_loss": 3.4761340032812793, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 2.6087, + "theoretical_loss": 3.4761225534139806, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489979797979798, + "loss": 2.7554, + "theoretical_loss": 3.476111104108216, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489959595959596, + "loss": 2.748, + "theoretical_loss": 3.476099655363936, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899393939393939, + "loss": 2.7153, + "theoretical_loss": 3.476088207181092, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899191919191919, + "loss": 2.6549, + "theoretical_loss": 3.4760767595596347, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 968886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7255613803863525, + "objective/train/theoretical_loss": 3.4760653124995144, + "objective/train/tokens_used": 69610976, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898989898989899, + "loss": 3.0056, + "theoretical_loss": 3.4760653124995144, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898787878787878, + "loss": 2.5703, + "theoretical_loss": 3.4760538660006834, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898585858585858, + "loss": 2.6349, + "theoretical_loss": 3.4760424200630915, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898383838383839, + "loss": 2.7111, + "theoretical_loss": 3.4760309746866906, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898181818181819, + "loss": 2.7104, + "theoretical_loss": 3.476019529871431, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897979797979798, + "loss": 2.7612, + "theoretical_loss": 3.4760080856172637, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897777777777778, + "loss": 2.7436, + "theoretical_loss": 3.475996641924141, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897575757575758, + "loss": 2.6489, + "theoretical_loss": 3.475985198792012, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897373737373738, + "loss": 2.6012, + "theoretical_loss": 3.4759737562208284, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897171717171717, + "loss": 2.7872, + "theoretical_loss": 3.4759623142105416, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896969696969697, + "loss": 2.6431, + "theoretical_loss": 3.4759508727611026, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896767676767676, + "loss": 2.537, + "theoretical_loss": 3.475939431872462, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896565656565657, + "loss": 2.6232, + "theoretical_loss": 3.4759279915445713, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896363636363636, + "loss": 2.6851, + "theoretical_loss": 3.4759165517773813, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896161616161617, + "loss": 2.5994, + "theoretical_loss": 3.4759051125708433, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895959595959596, + "loss": 2.5025, + "theoretical_loss": 3.475893673924908, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895757575757576, + "loss": 2.8659, + "theoretical_loss": 3.475882235839527, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895555555555556, + "loss": 2.6973, + "theoretical_loss": 3.4758707983146504, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895353535353536, + "loss": 2.6454, + "theoretical_loss": 3.47585936135023, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895151515151515, + "loss": 2.9223, + "theoretical_loss": 3.475847924946217, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894949494949495, + "loss": 2.6701, + "theoretical_loss": 3.475836489102562, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894747474747475, + "loss": 2.74, + "theoretical_loss": 3.4758250538192166, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894545454545455, + "loss": 2.8278, + "theoretical_loss": 3.4758136190961313, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894343434343434, + "loss": 2.8369, + "theoretical_loss": 3.475802184933258, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894141414141414, + "loss": 2.4811, + "theoretical_loss": 3.475790751330547, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 970327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5952835083007812, + "objective/train/theoretical_loss": 3.47577931828795, + "objective/train/tokens_used": 71249376, + "theoretical_loss": 3.47577931828795, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893939393939393, + "loss": 2.7346, + "theoretical_loss": 3.47577931828795, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893737373737374, + "loss": 2.259, + "theoretical_loss": 3.4757678858054177, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893535353535354, + "loss": 2.8657, + "theoretical_loss": 3.4757564538829016, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893333333333334, + "loss": 2.7831, + "theoretical_loss": 3.4757450225203526, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893131313131313, + "loss": 2.8222, + "theoretical_loss": 3.475733591717722, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892929292929293, + "loss": 2.6357, + "theoretical_loss": 3.4757221614749607, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892727272727273, + "loss": 2.7023, + "theoretical_loss": 3.47571073179202, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892525252525253, + "loss": 2.702, + "theoretical_loss": 3.4756993026688514, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892323232323232, + "loss": 2.8007, + "theoretical_loss": 3.475687874105406, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892121212121212, + "loss": 2.7147, + "theoretical_loss": 3.4756764461016347, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891919191919192, + "loss": 2.7137, + "theoretical_loss": 3.4756650186574882, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891717171717171, + "loss": 2.7671, + "theoretical_loss": 3.475653591772918, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891515151515151, + "loss": 2.9201, + "theoretical_loss": 3.4756421654478764, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891313131313132, + "loss": 2.7879, + "theoretical_loss": 3.475630739682313, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891111111111112, + "loss": 2.7977, + "theoretical_loss": 3.47561931447618, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890909090909091, + "loss": 2.5631, + "theoretical_loss": 3.475607889829428, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890707070707071, + "loss": 2.7458, + "theoretical_loss": 3.475596465742009, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890505050505051, + "loss": 2.7407, + "theoretical_loss": 3.475585042213874, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489030303030303, + "loss": 2.7923, + "theoretical_loss": 3.4755736192449733, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489010101010101, + "loss": 2.6676, + "theoretical_loss": 3.4755621968352592, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488989898989899, + "loss": 2.8569, + "theoretical_loss": 3.475550774984682, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889696969696969, + "loss": 2.6467, + "theoretical_loss": 3.4755393536931942, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889494949494949, + "loss": 2.4896, + "theoretical_loss": 3.4755279329607465, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889292929292929, + "loss": 2.8168, + "theoretical_loss": 3.4755165127872893, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488909090909091, + "loss": 2.7639, + "theoretical_loss": 3.4755050931727753, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 971016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9580161571502686, + "objective/train/theoretical_loss": 3.4754936741171543, + "objective/train/tokens_used": 72887776, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888888888888889, + "loss": 2.6953, + "theoretical_loss": 3.4754936741171543, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888686868686869, + "loss": 2.7064, + "theoretical_loss": 3.475482255620379, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888484848484849, + "loss": 2.758, + "theoretical_loss": 3.4754708376823995, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888282828282829, + "loss": 2.7395, + "theoretical_loss": 3.475459420303168, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888080808080808, + "loss": 2.7501, + "theoretical_loss": 3.4754480034826356, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887878787878788, + "loss": 2.5826, + "theoretical_loss": 3.475436587220753, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887676767676768, + "loss": 2.6967, + "theoretical_loss": 3.4754251715174727, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887474747474748, + "loss": 2.887, + "theoretical_loss": 3.4754137563727445, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887272727272727, + "loss": 2.8722, + "theoretical_loss": 3.4754023417865207, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887070707070707, + "loss": 2.746, + "theoretical_loss": 3.4753909277587525, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886868686868686, + "loss": 2.7844, + "theoretical_loss": 3.475379514289391, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886666666666667, + "loss": 2.7742, + "theoretical_loss": 3.475368101378388, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886464646464647, + "loss": 2.5727, + "theoretical_loss": 3.475356689025694, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886262626262627, + "loss": 2.7937, + "theoretical_loss": 3.4753452772312614, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886060606060606, + "loss": 2.5695, + "theoretical_loss": 3.475333865995041, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885858585858586, + "loss": 2.6901, + "theoretical_loss": 3.4753224553169835, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656565656566, + "loss": 2.5409, + "theoretical_loss": 3.4753110451970417, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885454545454546, + "loss": 2.6908, + "theoretical_loss": 3.475299635635166, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885252525252525, + "loss": 2.6651, + "theoretical_loss": 3.475288226631308, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885050505050505, + "loss": 2.5693, + "theoretical_loss": 3.475276818185419, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884848484848484, + "loss": 2.522, + "theoretical_loss": 3.475265410297451, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884646464646465, + "loss": 2.5581, + "theoretical_loss": 3.4752540029673544, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884444444444445, + "loss": 2.5222, + "theoretical_loss": 3.4752425961950815, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884242424242425, + "loss": 2.7674, + "theoretical_loss": 3.4752311899805832, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884040404040404, + "loss": 2.7891, + "theoretical_loss": 3.475219784323811, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 972281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5824227333068848, + "objective/train/theoretical_loss": 3.475208379224716, + "objective/train/tokens_used": 74526176, + "theoretical_loss": 3.475208379224716, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883838383838384, + "loss": 2.7513, + "theoretical_loss": 3.475208379224716, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883636363636364, + "loss": 2.7579, + "theoretical_loss": 3.475196974683251, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883434343434344, + "loss": 2.7134, + "theoretical_loss": 3.4751855706993657, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883232323232323, + "loss": 2.6353, + "theoretical_loss": 3.4751741672730123, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883030303030303, + "loss": 2.4706, + "theoretical_loss": 3.4751627644041427, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882828282828283, + "loss": 2.8204, + "theoretical_loss": 3.4751513620927073, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882626262626263, + "loss": 2.8287, + "theoretical_loss": 3.475139960338659, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882424242424243, + "loss": 2.6758, + "theoretical_loss": 3.4751285591419476, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822222222222225, + "loss": 2.734, + "theoretical_loss": 3.475117158502526, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882020202020202, + "loss": 2.7341, + "theoretical_loss": 3.4751057584203444, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818181818181816, + "loss": 2.6042, + "theoretical_loss": 3.4750943588953556, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048816161616161617, + "loss": 2.6838, + "theoretical_loss": 3.47508295992751, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881414141414142, + "loss": 2.6906, + "theoretical_loss": 3.47507156151676, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048812121212121214, + "loss": 2.8033, + "theoretical_loss": 3.4750601636630565, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881010101010101, + "loss": 2.5701, + "theoretical_loss": 3.475048766366351, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048808080808080805, + "loss": 2.5439, + "theoretical_loss": 3.4750373696265955, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880606060606061, + "loss": 2.5974, + "theoretical_loss": 3.4750259734437408, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880404040404041, + "loss": 2.6446, + "theoretical_loss": 3.4750145778177393, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802020202020203, + "loss": 2.8791, + "theoretical_loss": 3.4750031827485417, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488, + "loss": 2.7261, + "theoretical_loss": 3.4749917882360997, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048797979797979795, + "loss": 2.5626, + "theoretical_loss": 3.4749803942803656, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487959595959596, + "loss": 3.0468, + "theoretical_loss": 3.4749690008812903, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793939393939397, + "loss": 2.7726, + "theoretical_loss": 3.4749576080388254, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879191919191919, + "loss": 2.9184, + "theoretical_loss": 3.4749462157529227, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878989898989899, + "loss": 2.6087, + "theoretical_loss": 3.4749348240235336, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 972817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.845362901687622, + "objective/train/theoretical_loss": 3.47492343285061, + "objective/train/tokens_used": 76164576, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048787878787878784, + "loss": 2.944, + "theoretical_loss": 3.47492343285061, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878585858585859, + "loss": 2.826, + "theoretical_loss": 3.474912042234102, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783838383838386, + "loss": 2.5829, + "theoretical_loss": 3.4749006521739636, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878181818181818, + "loss": 2.5812, + "theoretical_loss": 3.4748892626701444, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877979797979798, + "loss": 2.6497, + "theoretical_loss": 3.4748778737225976, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877777777777778, + "loss": 2.7823, + "theoretical_loss": 3.4748664853312734, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877575757575758, + "loss": 2.5843, + "theoretical_loss": 3.474855097496124, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773737373737375, + "loss": 2.8728, + "theoretical_loss": 3.474843710217101, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877171717171717, + "loss": 2.7087, + "theoretical_loss": 3.474832323494156, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769696969696967, + "loss": 2.7008, + "theoretical_loss": 3.474820937327241, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876767676767677, + "loss": 2.5779, + "theoretical_loss": 3.474809551716307, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876565656565657, + "loss": 2.4991, + "theoretical_loss": 3.474798166661306, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048763636363636365, + "loss": 2.6801, + "theoretical_loss": 3.4747867821621896, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876161616161616, + "loss": 2.6759, + "theoretical_loss": 3.4747753982189096, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875959595959596, + "loss": 2.6234, + "theoretical_loss": 3.474764014831417, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048757575757575757, + "loss": 2.6152, + "theoretical_loss": 3.4747526319996647, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875555555555556, + "loss": 2.6105, + "theoretical_loss": 3.474741249723603, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753535353535354, + "loss": 2.5571, + "theoretical_loss": 3.4747298680031844, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875151515151515, + "loss": 2.8331, + "theoretical_loss": 3.4747184868383605, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874949494949495, + "loss": 2.7477, + "theoretical_loss": 3.474707106229083, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048747474747474746, + "loss": 2.632, + "theoretical_loss": 3.4746957261753035, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874545454545455, + "loss": 2.6545, + "theoretical_loss": 3.4746843466769737, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743434343434343, + "loss": 2.7711, + "theoretical_loss": 3.4746729677340453, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048741414141414144, + "loss": 2.6124, + "theoretical_loss": 3.47466158934647, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004873939393939394, + "loss": 2.4784, + "theoretical_loss": 3.4746502115141995, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 973424, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.251652240753174, + "objective/train/theoretical_loss": 3.4746388342371857, + "objective/train/tokens_used": 77802976, + "theoretical_loss": 3.4746388342371857, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737373737373736, + "loss": 2.6246, + "theoretical_loss": 3.4746388342371857, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735353535353537, + "loss": 2.8193, + "theoretical_loss": 3.47462745751538, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873333333333333, + "loss": 2.595, + "theoretical_loss": 3.474616081348735, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731313131313134, + "loss": 2.5577, + "theoretical_loss": 3.4746047057372014, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872929292929293, + "loss": 2.7767, + "theoretical_loss": 3.4745933306807313, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727272727272725, + "loss": 2.6097, + "theoretical_loss": 3.474581956179277, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048725252525252526, + "loss": 2.8316, + "theoretical_loss": 3.474570582232789, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723232323232327, + "loss": 2.6922, + "theoretical_loss": 3.4745592088412205, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048721212121212123, + "loss": 2.6333, + "theoretical_loss": 3.4745478360045228, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871919191919192, + "loss": 2.7247, + "theoretical_loss": 3.4745364637226466, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048717171717171714, + "loss": 2.7753, + "theoretical_loss": 3.4745250919955453, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715151515151515, + "loss": 2.4626, + "theoretical_loss": 3.47451372082317, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713131313131316, + "loss": 2.7074, + "theoretical_loss": 3.474502350205472, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871111111111111, + "loss": 2.6095, + "theoretical_loss": 3.474490980142404, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870909090909091, + "loss": 2.6282, + "theoretical_loss": 3.4744796106339173, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707070707070704, + "loss": 2.7157, + "theoretical_loss": 3.474468241679964, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870505050505051, + "loss": 2.6526, + "theoretical_loss": 3.474456873280496, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703030303030306, + "loss": 2.7501, + "theoretical_loss": 3.4744455054354644, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487010101010101, + "loss": 2.645, + "theoretical_loss": 3.474434138144822, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048698989898989897, + "loss": 2.5336, + "theoretical_loss": 3.47442277140852, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486969696969697, + "loss": 2.9669, + "theoretical_loss": 3.47441140522651, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486949494949495, + "loss": 3.1097, + "theoretical_loss": 3.474400039598745, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048692929292929295, + "loss": 2.6356, + "theoretical_loss": 3.4743886745251755, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869090909090909, + "loss": 2.7856, + "theoretical_loss": 3.4743773100057544, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688888888888886, + "loss": 2.656, + "theoretical_loss": 3.4743659460404333, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 974650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9588029384613037, + "objective/train/theoretical_loss": 3.4743545826291635, + "objective/train/tokens_used": 79441376, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686868686868693, + "loss": 2.7832, + "theoretical_loss": 3.4743545826291635, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868484848484849, + "loss": 2.5648, + "theoretical_loss": 3.4743432197718978, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682828282828284, + "loss": 2.5712, + "theoretical_loss": 3.4743318574685875, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868080808080808, + "loss": 2.4929, + "theoretical_loss": 3.4743204957191844, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678787878787876, + "loss": 2.5813, + "theoretical_loss": 3.474309134523641, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867676767676768, + "loss": 2.6235, + "theoretical_loss": 3.4742977738819087, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867474747474748, + "loss": 2.8261, + "theoretical_loss": 3.4742864137939398, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048672727272727274, + "loss": 2.582, + "theoretical_loss": 3.474275054259685, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867070707070707, + "loss": 2.7205, + "theoretical_loss": 3.4742636952790984, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866868686868687, + "loss": 2.7145, + "theoretical_loss": 3.47425233685213, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866666666666667, + "loss": 2.6738, + "theoretical_loss": 3.474240978978733, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664646464646467, + "loss": 2.5304, + "theoretical_loss": 3.474229621658859, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048662626262626263, + "loss": 2.8267, + "theoretical_loss": 3.474218264892459, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866060606060606, + "loss": 2.7056, + "theoretical_loss": 3.4742069086794864, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865858585858586, + "loss": 2.6254, + "theoretical_loss": 3.474195553019892, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865656565656566, + "loss": 2.6474, + "theoretical_loss": 3.4741841979136288, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048654545454545456, + "loss": 2.7789, + "theoretical_loss": 3.4741728433606482, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865252525252525, + "loss": 2.7534, + "theoretical_loss": 3.4741614893609016, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650505050505053, + "loss": 2.7393, + "theoretical_loss": 3.4741501359143423, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864848484848485, + "loss": 2.4025, + "theoretical_loss": 3.4741387830209214, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864646464646465, + "loss": 2.4853, + "theoretical_loss": 3.474127430680591, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644444444444446, + "loss": 2.525, + "theoretical_loss": 3.474116078893303, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864242424242424, + "loss": 2.6792, + "theoretical_loss": 3.47410472765901, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864040404040404, + "loss": 2.4953, + "theoretical_loss": 3.474093376977664, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863838383838384, + "loss": 2.6069, + "theoretical_loss": 3.474082026849216, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 975130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6752278804779053, + "objective/train/theoretical_loss": 3.4740706772736187, + "objective/train/tokens_used": 81079776, + "theoretical_loss": 3.4740706772736187, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863636363636364, + "loss": 2.823, + "theoretical_loss": 3.4740706772736187, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048634343434343435, + "loss": 2.5366, + "theoretical_loss": 3.4740593282508243, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632323232323236, + "loss": 2.7492, + "theoretical_loss": 3.474047979780785, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863030303030303, + "loss": 2.6195, + "theoretical_loss": 3.474036631863452, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862828282828283, + "loss": 2.6902, + "theoretical_loss": 3.4740252844987785, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862626262626263, + "loss": 2.5544, + "theoretical_loss": 3.474013937686715, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624242424242424, + "loss": 2.6836, + "theoretical_loss": 3.4740025914272152, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048622222222222225, + "loss": 2.5702, + "theoretical_loss": 3.4739912457202307, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862020202020202, + "loss": 2.7872, + "theoretical_loss": 3.4739799005657126, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048618181818181817, + "loss": 2.7781, + "theoretical_loss": 3.473968555963614, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861616161616162, + "loss": 2.874, + "theoretical_loss": 3.473957211913887, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861414141414142, + "loss": 2.4919, + "theoretical_loss": 3.4739458684164832, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612121212121215, + "loss": 2.6111, + "theoretical_loss": 3.473934525471355, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861010101010101, + "loss": 2.6207, + "theoretical_loss": 3.4739231830784543, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608080808080806, + "loss": 2.7133, + "theoretical_loss": 3.473911841237733, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606060606060607, + "loss": 2.6698, + "theoretical_loss": 3.473900499949144, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860404040404041, + "loss": 2.7573, + "theoretical_loss": 3.4738891592126393, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048602020202020204, + "loss": 2.5727, + "theoretical_loss": 3.4738778190281705, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486, + "loss": 2.6351, + "theoretical_loss": 3.4738664793956895, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048597979797979795, + "loss": 2.8587, + "theoretical_loss": 3.473855140315149, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048595959595959596, + "loss": 2.6785, + "theoretical_loss": 3.473843801786501, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485939393939394, + "loss": 2.8857, + "theoretical_loss": 3.473832463809698, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048591919191919193, + "loss": 2.6995, + "theoretical_loss": 3.4738211263846916, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858989898989899, + "loss": 2.6175, + "theoretical_loss": 3.473809789511434, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048587878787878784, + "loss": 2.5915, + "theoretical_loss": 3.473798453189878, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 976352, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.807490348815918, + "objective/train/theoretical_loss": 3.4737871174199757, + "objective/train/tokens_used": 82718176, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858585858585859, + "loss": 2.7339, + "theoretical_loss": 3.4737871174199757, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048583838383838387, + "loss": 2.5858, + "theoretical_loss": 3.4737757822016784, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858181818181818, + "loss": 2.6262, + "theoretical_loss": 3.473764447534939, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857979797979798, + "loss": 2.6693, + "theoretical_loss": 3.4737531134197095, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577777777777774, + "loss": 2.6928, + "theoretical_loss": 3.473741779855942, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857575757575758, + "loss": 2.8387, + "theoretical_loss": 3.4737304468435894, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573737373737376, + "loss": 2.6573, + "theoretical_loss": 3.473719114382603, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857171717171717, + "loss": 2.8101, + "theoretical_loss": 3.4737077824729354, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856969696969697, + "loss": 2.6919, + "theoretical_loss": 3.4736964511145385, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856767676767677, + "loss": 2.3924, + "theoretical_loss": 3.473685120307365, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856565656565657, + "loss": 2.7154, + "theoretical_loss": 3.4736737900513672, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563636363636365, + "loss": 2.7527, + "theoretical_loss": 3.473662460346497, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856161616161616, + "loss": 2.7677, + "theoretical_loss": 3.4736511311927067, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559595959595957, + "loss": 2.7468, + "theoretical_loss": 3.4736398025899486, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855757575757576, + "loss": 2.8561, + "theoretical_loss": 3.4736284745381756, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855555555555556, + "loss": 2.6859, + "theoretical_loss": 3.4736171470373387, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048553535353535355, + "loss": 2.7905, + "theoretical_loss": 3.473605820087391, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855151515151515, + "loss": 2.5174, + "theoretical_loss": 3.4735944936882848, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854949494949495, + "loss": 2.9664, + "theoretical_loss": 3.473583167839972, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048547474747474747, + "loss": 2.6424, + "theoretical_loss": 3.473571842542405, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854545454545455, + "loss": 2.7058, + "theoretical_loss": 3.473560517795536, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048543434343434344, + "loss": 2.8821, + "theoretical_loss": 3.473549193599318, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854141414141414, + "loss": 2.7736, + "theoretical_loss": 3.4735378699537027, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853939393939394, + "loss": 2.4991, + "theoretical_loss": 3.4735265468586425, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048537373737373736, + "loss": 2.7198, + "theoretical_loss": 3.4735152243140894, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 977030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.249246120452881, + "objective/train/theoretical_loss": 3.4735039023199965, + "objective/train/tokens_used": 84356576, + "theoretical_loss": 3.4735039023199965, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853535353535354, + "loss": 2.6702, + "theoretical_loss": 3.4735039023199965, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533333333333333, + "loss": 2.5773, + "theoretical_loss": 3.4734925808763153, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048531313131313134, + "loss": 2.7432, + "theoretical_loss": 3.473481259982999, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852929292929293, + "loss": 2.5795, + "theoretical_loss": 3.4734699396399993, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852727272727273, + "loss": 2.6497, + "theoretical_loss": 3.4734586198472686, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525252525252527, + "loss": 2.5608, + "theoretical_loss": 3.4734473006047595, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852323232323232, + "loss": 2.5798, + "theoretical_loss": 3.4734359819124245, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521212121212123, + "loss": 2.7358, + "theoretical_loss": 3.4734246637702153, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851919191919192, + "loss": 2.6388, + "theoretical_loss": 3.4734133461780847, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851717171717172, + "loss": 2.7245, + "theoretical_loss": 3.4734020291359853, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048515151515151516, + "loss": 2.7105, + "theoretical_loss": 3.473390712643869, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513131313131317, + "loss": 2.6479, + "theoretical_loss": 3.473379396701689, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851111111111111, + "loss": 2.6154, + "theoretical_loss": 3.473368081309397, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850909090909091, + "loss": 2.8949, + "theoretical_loss": 3.473356766466945, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850707070707071, + "loss": 2.6675, + "theoretical_loss": 3.4733454521742866, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048505050505050505, + "loss": 2.4718, + "theoretical_loss": 3.4733341384313734, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503030303030306, + "loss": 2.5198, + "theoretical_loss": 3.4733228252381574, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485010101010101, + "loss": 2.4897, + "theoretical_loss": 3.4733115125945924, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484989898989899, + "loss": 2.9024, + "theoretical_loss": 3.47330020050063, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484969696969697, + "loss": 2.7003, + "theoretical_loss": 3.4732888889562226, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484949494949495, + "loss": 2.5823, + "theoretical_loss": 3.4732775779613227, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492929292929296, + "loss": 2.636, + "theoretical_loss": 3.473266267515883, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849090909090909, + "loss": 2.7526, + "theoretical_loss": 3.4732549576198553, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048488888888888887, + "loss": 2.9381, + "theoretical_loss": 3.4732436482731925, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848686868686869, + "loss": 2.7722, + "theoretical_loss": 3.4732323394758478, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 978106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.543290376663208, + "objective/train/theoretical_loss": 3.473221031227773, + "objective/train/tokens_used": 85994976, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848484848484849, + "loss": 2.7245, + "theoretical_loss": 3.473221031227773, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048482828282828285, + "loss": 2.5946, + "theoretical_loss": 3.47320972352892, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848080808080808, + "loss": 2.6746, + "theoretical_loss": 3.473198416379242, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048478787878787876, + "loss": 2.8105, + "theoretical_loss": 3.4731871097786913, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476767676767683, + "loss": 2.5783, + "theoretical_loss": 3.4731758037272202, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847474747474748, + "loss": 2.7382, + "theoretical_loss": 3.473164498224782, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472727272727274, + "loss": 2.7656, + "theoretical_loss": 3.4731531932713287, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847070707070707, + "loss": 2.6224, + "theoretical_loss": 3.473141888866812, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468686868686865, + "loss": 2.8365, + "theoretical_loss": 3.4731305850111855, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846666666666667, + "loss": 2.6977, + "theoretical_loss": 3.473119281704402, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846464646464647, + "loss": 2.6204, + "theoretical_loss": 3.473107978946413, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462626262626263, + "loss": 2.6088, + "theoretical_loss": 3.4730966767371716, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846060606060606, + "loss": 2.6701, + "theoretical_loss": 3.4730853750766304, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048458585858585855, + "loss": 2.7009, + "theoretical_loss": 3.4730740739647414, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845656565656566, + "loss": 2.6164, + "theoretical_loss": 3.4730627734014576, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454545454545457, + "loss": 2.3539, + "theoretical_loss": 3.4730514733867315, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845252525252525, + "loss": 2.6729, + "theoretical_loss": 3.4730401739205163, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845050505050505, + "loss": 2.7273, + "theoretical_loss": 3.4730288750027634, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844848484848485, + "loss": 2.5672, + "theoretical_loss": 3.473017576633426, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844646464646465, + "loss": 2.4749, + "theoretical_loss": 3.4730062788124565, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048444444444444446, + "loss": 2.6967, + "theoretical_loss": 3.472994981539808, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844242424242424, + "loss": 2.7834, + "theoretical_loss": 3.4729836848154325, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844040404040404, + "loss": 2.6153, + "theoretical_loss": 3.472972388639283, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843838383838384, + "loss": 2.7989, + "theoretical_loss": 3.4729610930113117, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843636363636364, + "loss": 2.5639, + "theoretical_loss": 3.472949797931472, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 978835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.777024745941162, + "objective/train/theoretical_loss": 3.472938503399715, + "objective/train/tokens_used": 87633376, + "theoretical_loss": 3.472938503399715, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048434343434343435, + "loss": 2.8183, + "theoretical_loss": 3.472938503399715, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843232323232323, + "loss": 2.8991, + "theoretical_loss": 3.472927209415995, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843030303030303, + "loss": 2.4973, + "theoretical_loss": 3.4729159159802636, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828282828283, + "loss": 2.7018, + "theoretical_loss": 3.4729046230924743, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842626262626263, + "loss": 2.8986, + "theoretical_loss": 3.472893330752579, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048424242424242425, + "loss": 2.6149, + "theoretical_loss": 3.4728820389605306, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842222222222222, + "loss": 2.8363, + "theoretical_loss": 3.4728707477162817, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842020202020202, + "loss": 2.8066, + "theoretical_loss": 3.472859457019785, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048418181818181817, + "loss": 2.3374, + "theoretical_loss": 3.4728481668709934, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841616161616162, + "loss": 2.6328, + "theoretical_loss": 3.4728368772698586, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414141414141414, + "loss": 2.6452, + "theoretical_loss": 3.4728255882163346, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048412121212121215, + "loss": 2.4226, + "theoretical_loss": 3.4728142997103735, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841010101010101, + "loss": 2.6552, + "theoretical_loss": 3.472803011751928, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408080808080807, + "loss": 2.5425, + "theoretical_loss": 3.472791724340951, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840606060606061, + "loss": 2.8217, + "theoretical_loss": 3.472780437477395, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048404040404040403, + "loss": 2.7488, + "theoretical_loss": 3.472769151161213, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048402020202020204, + "loss": 2.7111, + "theoretical_loss": 3.4727578653923565, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484, + "loss": 2.945, + "theoretical_loss": 3.4727465801707798, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397979797979796, + "loss": 2.6126, + "theoretical_loss": 3.472735295496435, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048395959595959597, + "loss": 2.6608, + "theoretical_loss": 3.4727240113692748, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483939393939394, + "loss": 2.7868, + "theoretical_loss": 3.472712727789252, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048391919191919194, + "loss": 2.538, + "theoretical_loss": 3.4727014447563196, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838989898989899, + "loss": 2.703, + "theoretical_loss": 3.472690162270429, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048387878787878785, + "loss": 2.7455, + "theoretical_loss": 3.472678880331535, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048385858585858586, + "loss": 2.5767, + "theoretical_loss": 3.4726675989395894, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 979527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8225674629211426, + "objective/train/theoretical_loss": 3.472656318094545, + "objective/train/tokens_used": 89271776, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048383838383838387, + "loss": 2.5989, + "theoretical_loss": 3.472656318094545, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381818181818183, + "loss": 2.7386, + "theoretical_loss": 3.4726450377963545, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837979797979798, + "loss": 2.7309, + "theoretical_loss": 3.47263375804497, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377777777777774, + "loss": 2.7987, + "theoretical_loss": 3.472622478840346, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837575757575758, + "loss": 2.5695, + "theoretical_loss": 3.472611200182434, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373737373737377, + "loss": 2.6617, + "theoretical_loss": 3.472599922071187, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837171717171717, + "loss": 2.6842, + "theoretical_loss": 3.472588644506558, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836969696969697, + "loss": 2.812, + "theoretical_loss": 3.4725773674884994, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048367676767676764, + "loss": 2.8218, + "theoretical_loss": 3.4725660910169647, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836565656565657, + "loss": 2.5643, + "theoretical_loss": 3.472554815091906, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048363636363636366, + "loss": 2.7306, + "theoretical_loss": 3.472543539713277, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836161616161616, + "loss": 2.5076, + "theoretical_loss": 3.4725322648810297, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359595959595957, + "loss": 2.771, + "theoretical_loss": 3.4725209905951173, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357575757575764, + "loss": 2.8183, + "theoretical_loss": 3.472509716855493, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835555555555556, + "loss": 2.6278, + "theoretical_loss": 3.4724984436621087, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048353535353535355, + "loss": 2.5696, + "theoretical_loss": 3.4724871710149183, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835151515151515, + "loss": 2.6113, + "theoretical_loss": 3.4724758989138738, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349494949494946, + "loss": 2.6433, + "theoretical_loss": 3.472464627358929, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048347474747474753, + "loss": 2.7699, + "theoretical_loss": 3.4724533563500355, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834545454545455, + "loss": 2.7966, + "theoretical_loss": 3.472442085887147, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343434343434344, + "loss": 2.7717, + "theoretical_loss": 3.472430815970217, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834141414141414, + "loss": 2.5935, + "theoretical_loss": 3.4724195465991974, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833939393939394, + "loss": 2.8673, + "theoretical_loss": 3.4724082777740413, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833737373737374, + "loss": 2.6391, + "theoretical_loss": 3.472397009494702, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833535353535354, + "loss": 2.7217, + "theoretical_loss": 3.4723857417611317, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 980288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.298222064971924, + "objective/train/theoretical_loss": 3.4723744745732836, + "objective/train/tokens_used": 90910176, + "theoretical_loss": 3.4723744745732836, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048333333333333334, + "loss": 2.1876, + "theoretical_loss": 3.4723744745732836, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833131313131313, + "loss": 2.7602, + "theoretical_loss": 3.472363207931111, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832929292929293, + "loss": 2.835, + "theoretical_loss": 3.4723519418345665, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832727272727273, + "loss": 2.8478, + "theoretical_loss": 3.4723406762836033, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048325252525252527, + "loss": 2.5133, + "theoretical_loss": 3.472329411278174, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323232323232323, + "loss": 2.732, + "theoretical_loss": 3.472318146818232, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048321212121212124, + "loss": 2.6205, + "theoretical_loss": 3.4723068829037294, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831919191919192, + "loss": 2.7342, + "theoretical_loss": 3.47229561953462, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831717171717172, + "loss": 2.7776, + "theoretical_loss": 3.4722843567108566, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315151515151516, + "loss": 2.5522, + "theoretical_loss": 3.4722730944323916, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831313131313131, + "loss": 2.784, + "theoretical_loss": 3.4722618326991785, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048311111111111113, + "loss": 2.8411, + "theoretical_loss": 3.4722505715111707, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830909090909091, + "loss": 2.8035, + "theoretical_loss": 3.4722393108683205, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830707070707071, + "loss": 2.5237, + "theoretical_loss": 3.472228050770581, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048305050505050506, + "loss": 2.4995, + "theoretical_loss": 3.472216791217905, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048303030303030307, + "loss": 2.821, + "theoretical_loss": 3.4722055322102463, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483010101010101, + "loss": 2.8766, + "theoretical_loss": 3.472194273747557, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482989898989899, + "loss": 2.65, + "theoretical_loss": 3.47218301582979, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482969696969697, + "loss": 2.6007, + "theoretical_loss": 3.4721717584569003, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294949494949495, + "loss": 2.7881, + "theoretical_loss": 3.472160501628838, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292929292929296, + "loss": 2.802, + "theoretical_loss": 3.4721492453455585, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829090909090909, + "loss": 2.4568, + "theoretical_loss": 3.4721379896070133, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828888888888889, + "loss": 2.7808, + "theoretical_loss": 3.4721267344131563, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828686868686869, + "loss": 2.7256, + "theoretical_loss": 3.4721154797639406, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828484848484849, + "loss": 2.7409, + "theoretical_loss": 3.4721042256593186, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 981887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.213197946548462, + "objective/train/theoretical_loss": 3.472092972099244, + "objective/train/tokens_used": 92548576, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048282828282828285, + "loss": 2.714, + "theoretical_loss": 3.472092972099244, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828080808080808, + "loss": 2.573, + "theoretical_loss": 3.4720817190836692, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048278787878787877, + "loss": 2.5293, + "theoretical_loss": 3.472070466612548, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827676767676768, + "loss": 2.6715, + "theoretical_loss": 3.4720592146858333, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827474747474748, + "loss": 2.8534, + "theoretical_loss": 3.4720479633034778, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048272727272727275, + "loss": 2.6411, + "theoretical_loss": 3.472036712465435, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827070707070707, + "loss": 2.5797, + "theoretical_loss": 3.4720254621716578, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048268686868686866, + "loss": 2.714, + "theoretical_loss": 3.472014212422099, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266666666666667, + "loss": 2.5281, + "theoretical_loss": 3.4720029632167124, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826464646464647, + "loss": 2.499, + "theoretical_loss": 3.471991714555451, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048262626262626264, + "loss": 2.6674, + "theoretical_loss": 3.471980466438267, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826060606060606, + "loss": 2.7975, + "theoretical_loss": 3.471969218865115, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258585858585855, + "loss": 2.7174, + "theoretical_loss": 3.4719579718359466, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825656565656566, + "loss": 2.9137, + "theoretical_loss": 3.4719467253507164, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825454545454546, + "loss": 2.9169, + "theoretical_loss": 3.471935479409376, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252525252525253, + "loss": 2.6869, + "theoretical_loss": 3.4719242340118797, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825050505050505, + "loss": 2.576, + "theoretical_loss": 3.471912989158181, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048248484848484845, + "loss": 2.6921, + "theoretical_loss": 3.471901744848231, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824646464646465, + "loss": 2.7276, + "theoretical_loss": 3.471890501081985, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244444444444447, + "loss": 2.863, + "theoretical_loss": 3.4718792578593956, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824242424242424, + "loss": 2.7152, + "theoretical_loss": 3.4718680151804158, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824040404040404, + "loss": 2.6315, + "theoretical_loss": 3.4718567730449985, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823838383838384, + "loss": 2.5291, + "theoretical_loss": 3.4718455314530976, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823636363636364, + "loss": 2.5517, + "theoretical_loss": 3.471834290404665, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048234343434343436, + "loss": 2.7281, + "theoretical_loss": 3.4718230498996556, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 982466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4402005672454834, + "objective/train/theoretical_loss": 3.4718118099380213, + "objective/train/tokens_used": 94186976, + "theoretical_loss": 3.4718118099380213, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823232323232323, + "loss": 2.5035, + "theoretical_loss": 3.4718118099380213, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823030303030303, + "loss": 2.6198, + "theoretical_loss": 3.4718005705197164, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822828282828283, + "loss": 2.7942, + "theoretical_loss": 3.4717893316446933, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822626262626263, + "loss": 2.6933, + "theoretical_loss": 3.471778093312905, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224242424242425, + "loss": 2.8013, + "theoretical_loss": 3.4717668555243053, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822222222222222, + "loss": 2.7106, + "theoretical_loss": 3.4717556182788476, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822020202020202, + "loss": 2.4461, + "theoretical_loss": 3.471744381576485, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821818181818182, + "loss": 2.7407, + "theoretical_loss": 3.4717331454171703, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821616161616162, + "loss": 2.6126, + "theoretical_loss": 3.471721909800857, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048214141414141415, + "loss": 2.5327, + "theoretical_loss": 3.4717106747274986, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821212121212121, + "loss": 2.9424, + "theoretical_loss": 3.471699440197048, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821010101010101, + "loss": 2.7032, + "theoretical_loss": 3.4716882062094587, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048208080808080807, + "loss": 2.6112, + "theoretical_loss": 3.4716769727646835, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820606060606061, + "loss": 2.8054, + "theoretical_loss": 3.4716657398626767, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204040404040404, + "loss": 2.4609, + "theoretical_loss": 3.471654507503391, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048202020202020205, + "loss": 2.7502, + "theoretical_loss": 3.4716432756867794, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482, + "loss": 2.7791, + "theoretical_loss": 3.4716320444127957, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048197979797979796, + "loss": 2.6131, + "theoretical_loss": 3.471620813681393, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481959595959596, + "loss": 2.6483, + "theoretical_loss": 3.4716095834925245, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193939393939393, + "loss": 2.8038, + "theoretical_loss": 3.4715983538461432, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048191919191919194, + "loss": 2.7972, + "theoretical_loss": 3.4715871247422037, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818989898989899, + "loss": 2.8568, + "theoretical_loss": 3.4715758961806573, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818787878787879, + "loss": 2.7921, + "theoretical_loss": 3.4715646681614594, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048185858585858587, + "loss": 2.6927, + "theoretical_loss": 3.4715534406845627, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818383838383839, + "loss": 2.6901, + "theoretical_loss": 3.4715422137499194, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 983872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6384646892547607, + "objective/train/theoretical_loss": 3.4715309873574847, + "objective/train/tokens_used": 95825376, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048181818181818184, + "loss": 2.5924, + "theoretical_loss": 3.4715309873574847, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817979797979798, + "loss": 2.8885, + "theoretical_loss": 3.4715197615072104, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817777777777778, + "loss": 2.6904, + "theoretical_loss": 3.4715085361990505, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175757575757576, + "loss": 2.5645, + "theoretical_loss": 3.471497311432959, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048173737373737377, + "loss": 2.7105, + "theoretical_loss": 3.4714860872088877, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171717171717173, + "loss": 2.8841, + "theoretical_loss": 3.471474863526791, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816969696969697, + "loss": 2.9052, + "theoretical_loss": 3.471463640386623, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816767676767677, + "loss": 2.7558, + "theoretical_loss": 3.4714524177883352, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816565656565657, + "loss": 2.4644, + "theoretical_loss": 3.4714411957318827, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163636363636366, + "loss": 2.7226, + "theoretical_loss": 3.4714299742172186, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816161616161616, + "loss": 2.5989, + "theoretical_loss": 3.4714187532442953, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815959595959596, + "loss": 2.6259, + "theoretical_loss": 3.471407532813067, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815757575757576, + "loss": 2.5093, + "theoretical_loss": 3.4713963129234875, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815555555555556, + "loss": 2.6729, + "theoretical_loss": 3.4713850935755093, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048153535353535356, + "loss": 2.7453, + "theoretical_loss": 3.4713738747690863, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815151515151515, + "loss": 2.7665, + "theoretical_loss": 3.471362656504172, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149494949494947, + "loss": 2.8182, + "theoretical_loss": 3.4713514387807196, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147474747474754, + "loss": 2.6878, + "theoretical_loss": 3.471340221598683, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814545454545455, + "loss": 2.8049, + "theoretical_loss": 3.4713290049580157, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048143434343434345, + "loss": 2.6932, + "theoretical_loss": 3.4713177888586704, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814141414141414, + "loss": 2.7684, + "theoretical_loss": 3.471306573300601, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139393939393936, + "loss": 2.7707, + "theoretical_loss": 3.4712953582837613, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137373737373743, + "loss": 2.7301, + "theoretical_loss": 3.471284143808104, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813535353535354, + "loss": 2.7747, + "theoretical_loss": 3.4712729298735834, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133333333333334, + "loss": 2.6179, + "theoretical_loss": 3.471261716480152, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 984348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1180520057678223, + "objective/train/theoretical_loss": 3.4712505036277648, + "objective/train/tokens_used": 97463776, + "theoretical_loss": 3.4712505036277648, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813131313131313, + "loss": 2.7178, + "theoretical_loss": 3.4712505036277648, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129292929292926, + "loss": 2.8219, + "theoretical_loss": 3.4712392913163734, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812727272727273, + "loss": 2.7317, + "theoretical_loss": 3.4712280795459334, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812525252525253, + "loss": 2.8462, + "theoretical_loss": 3.4712168683163966, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048123232323232323, + "loss": 2.5972, + "theoretical_loss": 3.4712056576277175, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812121212121212, + "loss": 2.8409, + "theoretical_loss": 3.4711944474798493, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811919191919192, + "loss": 2.5882, + "theoretical_loss": 3.4711832378727454, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811717171717172, + "loss": 2.6525, + "theoretical_loss": 3.4711720288063592, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048115151515151517, + "loss": 2.8493, + "theoretical_loss": 3.4711608202806445, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048113131313131313, + "loss": 2.5986, + "theoretical_loss": 3.471149612295555, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811111111111111, + "loss": 2.6461, + "theoretical_loss": 3.4711384048510445, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810909090909091, + "loss": 2.6746, + "theoretical_loss": 3.471127197947066, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810707070707071, + "loss": 2.8123, + "theoretical_loss": 3.4711159915835728, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048105050505050506, + "loss": 2.5385, + "theoretical_loss": 3.4711047857605193, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481030303030303, + "loss": 2.7396, + "theoretical_loss": 3.4710935804778584, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101010101010103, + "loss": 2.6643, + "theoretical_loss": 3.471082375735544, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480989898989899, + "loss": 2.6482, + "theoretical_loss": 3.47107117153353, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480969696969697, + "loss": 2.8035, + "theoretical_loss": 3.471059967871769, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048094949494949496, + "loss": 2.6043, + "theoretical_loss": 3.4710487647502157, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809292929292929, + "loss": 2.8306, + "theoretical_loss": 3.4710375621688234, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809090909090909, + "loss": 2.6981, + "theoretical_loss": 3.471026360127545, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808888888888889, + "loss": 2.6039, + "theoretical_loss": 3.471015158626335, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808686868686869, + "loss": 2.5663, + "theoretical_loss": 3.471003957665147, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084848484848485, + "loss": 2.5783, + "theoretical_loss": 3.470992757243934, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048082828282828286, + "loss": 2.6783, + "theoretical_loss": 3.4709815573626495, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 985100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.260004758834839, + "objective/train/theoretical_loss": 3.4709703580212485, + "objective/train/tokens_used": 99102176, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808080808080808, + "loss": 2.9963, + "theoretical_loss": 3.4709703580212485, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807878787878788, + "loss": 2.5721, + "theoretical_loss": 3.470959159219683, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807676767676768, + "loss": 2.8963, + "theoretical_loss": 3.4709479609579077, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048074747474747474, + "loss": 2.8664, + "theoretical_loss": 3.470936763235876, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072727272727275, + "loss": 2.6729, + "theoretical_loss": 3.470925566053541, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807070707070707, + "loss": 2.8778, + "theoretical_loss": 3.4709143694108575, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068686868686867, + "loss": 2.8135, + "theoretical_loss": 3.4709031733077786, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806666666666667, + "loss": 3.0498, + "theoretical_loss": 3.4708919777442575, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806464646464647, + "loss": 2.819, + "theoretical_loss": 3.470880782720249, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048062626262626264, + "loss": 2.5524, + "theoretical_loss": 3.470869588235705, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806060606060606, + "loss": 2.7275, + "theoretical_loss": 3.4708583942905813, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058585858585856, + "loss": 2.8643, + "theoretical_loss": 3.4708472008848297, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048056565656565657, + "loss": 2.6051, + "theoretical_loss": 3.4708360080184057, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805454545454546, + "loss": 2.7241, + "theoretical_loss": 3.4708248156912616, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052525252525254, + "loss": 2.6136, + "theoretical_loss": 3.470813623903352, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805050505050505, + "loss": 2.6912, + "theoretical_loss": 3.47080243265463, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048048484848484845, + "loss": 2.5619, + "theoretical_loss": 3.47079124194505, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804646464646465, + "loss": 2.618, + "theoretical_loss": 3.4707800517745646, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804444444444445, + "loss": 2.6961, + "theoretical_loss": 3.470768862143129, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042424242424243, + "loss": 2.5643, + "theoretical_loss": 3.470757673050696, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804040404040404, + "loss": 2.7059, + "theoretical_loss": 3.4707464844972193, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038383838383834, + "loss": 2.6824, + "theoretical_loss": 3.470735296482653, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803636363636364, + "loss": 2.6512, + "theoretical_loss": 3.470724109006951, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034343434343437, + "loss": 2.8463, + "theoretical_loss": 3.470712922070067, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803232323232323, + "loss": 2.7123, + "theoretical_loss": 3.4707017356719545, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 986266, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.291090965270996, + "objective/train/theoretical_loss": 3.4706905498125673, + "objective/train/tokens_used": 100740576, + "theoretical_loss": 3.4706905498125673, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803030303030303, + "loss": 2.7817, + "theoretical_loss": 3.4706905498125673, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802828282828283, + "loss": 2.8213, + "theoretical_loss": 3.47067936449186, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802626262626263, + "loss": 2.7238, + "theoretical_loss": 3.4706681797097847, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048024242424242426, + "loss": 2.8662, + "theoretical_loss": 3.470656995466297, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802222222222222, + "loss": 2.5868, + "theoretical_loss": 3.47064581176135, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020202020202017, + "loss": 2.7397, + "theoretical_loss": 3.470634628594897, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801818181818182, + "loss": 2.5641, + "theoretical_loss": 3.4706234459668925, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801616161616162, + "loss": 2.6906, + "theoretical_loss": 3.47061226387729, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014141414141415, + "loss": 2.5981, + "theoretical_loss": 3.470601082326043, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801212121212121, + "loss": 2.721, + "theoretical_loss": 3.4705899013131063, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801010101010101, + "loss": 2.7245, + "theoretical_loss": 3.470578720838433, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048008080808080813, + "loss": 2.5229, + "theoretical_loss": 3.4705675409019774, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800606060606061, + "loss": 2.7938, + "theoretical_loss": 3.4705563615036925, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004040404040404, + "loss": 2.5731, + "theoretical_loss": 3.4705451826435327, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480020202020202, + "loss": 2.4975, + "theoretical_loss": 3.4705340043214523, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 2.7554, + "theoretical_loss": 3.4705228265374046, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479979797979798, + "loss": 2.9406, + "theoretical_loss": 3.470511649291344, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479959595959596, + "loss": 2.5484, + "theoretical_loss": 3.4705004725832236, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047993939393939394, + "loss": 2.7902, + "theoretical_loss": 3.470489296412998, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047991919191919195, + "loss": 2.8054, + "theoretical_loss": 3.470478120780621, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798989898989899, + "loss": 2.5875, + "theoretical_loss": 3.4704669456860455, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798787878787879, + "loss": 2.4622, + "theoretical_loss": 3.4704557711292265, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047985858585858587, + "loss": 2.6323, + "theoretical_loss": 3.470444597110118, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983838383838383, + "loss": 2.5833, + "theoretical_loss": 3.4704334236286734, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047981818181818184, + "loss": 2.7322, + "theoretical_loss": 3.4704222506848463, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 986906, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.303271770477295, + "objective/train/theoretical_loss": 3.470411078278591, + "objective/train/tokens_used": 102378976, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797979797979798, + "loss": 2.6483, + "theoretical_loss": 3.470411078278591, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797777777777778, + "loss": 2.6322, + "theoretical_loss": 3.4703999064098623, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975757575757577, + "loss": 2.9993, + "theoretical_loss": 3.470388735078613, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797373737373738, + "loss": 2.7296, + "theoretical_loss": 3.4703775642847967, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047971717171717173, + "loss": 2.5097, + "theoretical_loss": 3.4703663940283684, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796969696969697, + "loss": 2.6714, + "theoretical_loss": 3.470355224309282, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796767676767677, + "loss": 2.9121, + "theoretical_loss": 3.470344055127491, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047965656565656566, + "loss": 2.9563, + "theoretical_loss": 3.4703328864829492, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047963636363636367, + "loss": 2.8004, + "theoretical_loss": 3.4703217183756108, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796161616161616, + "loss": 2.5658, + "theoretical_loss": 3.47031055080543, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795959595959596, + "loss": 2.5716, + "theoretical_loss": 3.470299383772361, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795757575757576, + "loss": 2.8171, + "theoretical_loss": 3.470288217276357, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795555555555556, + "loss": 2.6697, + "theoretical_loss": 3.4702770513173724, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953535353535356, + "loss": 2.516, + "theoretical_loss": 3.470265885895361, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795151515151515, + "loss": 2.7924, + "theoretical_loss": 3.4702547210102774, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794949494949495, + "loss": 2.6405, + "theoretical_loss": 3.470243556662075, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794747474747475, + "loss": 2.4795, + "theoretical_loss": 3.470232392850708, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794545454545455, + "loss": 2.767, + "theoretical_loss": 3.470221229576131, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943434343434345, + "loss": 2.5854, + "theoretical_loss": 3.4702100668382965, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794141414141414, + "loss": 2.6928, + "theoretical_loss": 3.47019890463716, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939393939393937, + "loss": 2.5936, + "theoretical_loss": 3.470187742972675, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793737373737374, + "loss": 2.8456, + "theoretical_loss": 3.470176581844796, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793535353535354, + "loss": 2.7196, + "theoretical_loss": 3.4701654212534763, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047933333333333335, + "loss": 2.6887, + "theoretical_loss": 3.4701542611986698, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793131313131313, + "loss": 2.6971, + "theoretical_loss": 3.470143101680332, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 988319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.722970485687256, + "objective/train/theoretical_loss": 3.4701319426984156, + "objective/train/tokens_used": 104017376, + "theoretical_loss": 3.4701319426984156, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047929292929292926, + "loss": 2.6675, + "theoretical_loss": 3.4701319426984156, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792727272727273, + "loss": 2.5615, + "theoretical_loss": 3.4701207842528747, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792525252525253, + "loss": 2.6597, + "theoretical_loss": 3.470109626343664, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923232323232324, + "loss": 2.8405, + "theoretical_loss": 3.470098468970738, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792121212121212, + "loss": 2.455, + "theoretical_loss": 3.470087312134049, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919191919191915, + "loss": 2.5948, + "theoretical_loss": 3.470076155833553, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791717171717172, + "loss": 2.7171, + "theoretical_loss": 3.4700650000692033, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791515151515152, + "loss": 2.8403, + "theoretical_loss": 3.4700538448409537, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913131313131313, + "loss": 2.7391, + "theoretical_loss": 3.470042690148759, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791111111111111, + "loss": 2.7464, + "theoretical_loss": 3.4700315359925726, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790909090909091, + "loss": 2.6238, + "theoretical_loss": 3.470020382372349, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790707070707071, + "loss": 2.5908, + "theoretical_loss": 3.470009229288043, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905050505050507, + "loss": 2.582, + "theoretical_loss": 3.469998076739607, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479030303030303, + "loss": 2.5008, + "theoretical_loss": 3.469986924726997, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479010101010101, + "loss": 2.8017, + "theoretical_loss": 3.469975773250166, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478989898989899, + "loss": 2.5246, + "theoretical_loss": 3.4699646223090688, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478969696969697, + "loss": 2.6751, + "theoretical_loss": 3.469953471903659, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894949494949496, + "loss": 2.6898, + "theoretical_loss": 3.469942322033891, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789292929292929, + "loss": 2.9576, + "theoretical_loss": 3.4699311726997193, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890909090909093, + "loss": 2.7507, + "theoretical_loss": 3.469920023901097, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788888888888889, + "loss": 2.7436, + "theoretical_loss": 3.46990887563798, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788686868686869, + "loss": 2.8719, + "theoretical_loss": 3.469897727910321, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047884848484848485, + "loss": 2.6318, + "theoretical_loss": 3.4698865807180748, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788282828282828, + "loss": 2.9676, + "theoretical_loss": 3.4698754340611955, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788080808080808, + "loss": 2.6234, + "theoretical_loss": 3.4698642879396373, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 989102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.321385622024536, + "objective/train/theoretical_loss": 3.4698531423533545, + "objective/train/tokens_used": 105655776, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787878787878788, + "loss": 2.4364, + "theoretical_loss": 3.4698531423533545, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787676767676768, + "loss": 2.4427, + "theoretical_loss": 3.469841997302301, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874747474747475, + "loss": 2.7983, + "theoretical_loss": 3.4698308527864317, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872727272727276, + "loss": 2.7157, + "theoretical_loss": 3.4698197088056997, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787070707070707, + "loss": 2.6562, + "theoretical_loss": 3.4698085653600605, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047868686868686867, + "loss": 2.5761, + "theoretical_loss": 3.469797422449467, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786666666666667, + "loss": 2.6611, + "theoretical_loss": 3.4697862800738752, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864646464646464, + "loss": 2.978, + "theoretical_loss": 3.4697751382332376, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862626262626265, + "loss": 2.4101, + "theoretical_loss": 3.46976399692751, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786060606060606, + "loss": 2.7225, + "theoretical_loss": 3.469752856156645, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858585858585856, + "loss": 2.7948, + "theoretical_loss": 3.469741715920598, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785656565656566, + "loss": 2.6369, + "theoretical_loss": 3.4697305762193227, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785454545454546, + "loss": 2.6362, + "theoretical_loss": 3.469719437052774, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047852525252525254, + "loss": 2.5745, + "theoretical_loss": 3.4697082984209056, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785050505050505, + "loss": 2.637, + "theoretical_loss": 3.469697160323672, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848484848484846, + "loss": 2.8247, + "theoretical_loss": 3.469686022761028, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047846464646464647, + "loss": 2.7787, + "theoretical_loss": 3.469674885732927, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784444444444445, + "loss": 2.6952, + "theoretical_loss": 3.4696637492393236, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047842424242424244, + "loss": 2.4277, + "theoretical_loss": 3.4696526132801724, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784040404040404, + "loss": 2.4748, + "theoretical_loss": 3.4696414778554274, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783838383838384, + "loss": 2.8423, + "theoretical_loss": 3.4696303429650435, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783636363636364, + "loss": 2.6706, + "theoretical_loss": 3.469619208608974, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047834343434343437, + "loss": 2.7299, + "theoretical_loss": 3.469608074787174, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832323232323233, + "loss": 2.5596, + "theoretical_loss": 3.4695969414995975, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783030303030303, + "loss": 2.6606, + "theoretical_loss": 3.4695858087461993, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 990185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.444274663925171, + "objective/train/theoretical_loss": 3.469574676526933, + "objective/train/tokens_used": 107294176, + "theoretical_loss": 3.469574676526933, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782828282828283, + "loss": 2.4709, + "theoretical_loss": 3.469574676526933, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782626262626263, + "loss": 2.7695, + "theoretical_loss": 3.469563544841754, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824242424242426, + "loss": 2.6848, + "theoretical_loss": 3.4695524136906153, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782222222222222, + "loss": 2.7675, + "theoretical_loss": 3.4695412830734726, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782020202020202, + "loss": 2.8053, + "theoretical_loss": 3.46953015299028, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818181818181824, + "loss": 2.6393, + "theoretical_loss": 3.469519023440991, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781616161616162, + "loss": 2.871, + "theoretical_loss": 3.4695078944255604, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047814141414141416, + "loss": 2.6308, + "theoretical_loss": 3.4694967659439433, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781212121212121, + "loss": 2.7099, + "theoretical_loss": 3.469485637996093, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810101010101007, + "loss": 2.6317, + "theoretical_loss": 3.4694745105819647, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808080808080814, + "loss": 2.4006, + "theoretical_loss": 3.4694633837015125, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780606060606061, + "loss": 2.7594, + "theoretical_loss": 3.469452257354691, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047804040404040405, + "loss": 2.7311, + "theoretical_loss": 3.469441131541454, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478020202020202, + "loss": 2.8405, + "theoretical_loss": 3.4694300062617565, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799999999999996, + "loss": 2.8065, + "theoretical_loss": 3.469418881515553, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047797979797979803, + "loss": 2.3884, + "theoretical_loss": 3.4694077573027977, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477959595959596, + "loss": 2.5529, + "theoretical_loss": 3.469396633623445, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047793939393939394, + "loss": 2.4781, + "theoretical_loss": 3.4693855104774496, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779191919191919, + "loss": 2.7568, + "theoretical_loss": 3.4693743878647654, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778989898989899, + "loss": 2.4734, + "theoretical_loss": 3.469363265785348, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778787878787879, + "loss": 2.8889, + "theoretical_loss": 3.46935214423915, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778585858585859, + "loss": 2.821, + "theoretical_loss": 3.4693410232261277, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783838383838384, + "loss": 2.7586, + "theoretical_loss": 3.4693299027462343, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778181818181818, + "loss": 2.6601, + "theoretical_loss": 3.469318782799425, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777979797979798, + "loss": 2.8503, + "theoretical_loss": 3.4693076633856545, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 990903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9547412395477295, + "objective/train/theoretical_loss": 3.4692965445048767, + "objective/train/tokens_used": 108932576, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777777777777778, + "loss": 2.8144, + "theoretical_loss": 3.4692965445048767, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775757575757577, + "loss": 2.7384, + "theoretical_loss": 3.469285426157046, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047773737373737373, + "loss": 2.8771, + "theoretical_loss": 3.469274308342117, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771717171717174, + "loss": 2.8332, + "theoretical_loss": 3.469263191060045, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776969696969697, + "loss": 2.9329, + "theoretical_loss": 3.4692520743107833, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776767676767677, + "loss": 2.4978, + "theoretical_loss": 3.469240958094287, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047765656565656566, + "loss": 2.4177, + "theoretical_loss": 3.469229842410511, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776363636363636, + "loss": 2.6286, + "theoretical_loss": 3.4692187272594093, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761616161616163, + "loss": 2.7693, + "theoretical_loss": 3.469207612640936, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775959595959596, + "loss": 2.6324, + "theoretical_loss": 3.469196498555047, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775757575757576, + "loss": 2.576, + "theoretical_loss": 3.469185385001696, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047755555555555556, + "loss": 2.6205, + "theoretical_loss": 3.4691742719808376, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753535353535357, + "loss": 2.6338, + "theoretical_loss": 3.469163159492426, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775151515151515, + "loss": 2.6622, + "theoretical_loss": 3.469152047536416, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774949494949495, + "loss": 2.6942, + "theoretical_loss": 3.4691409361127628, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774747474747475, + "loss": 2.5152, + "theoretical_loss": 3.46912982522142, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745454545454545, + "loss": 2.5977, + "theoretical_loss": 3.469118714862343, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047743434343434346, + "loss": 2.432, + "theoretical_loss": 3.469107605035486, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774141414141414, + "loss": 2.6973, + "theoretical_loss": 3.4690964957408035, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773939393939394, + "loss": 2.6671, + "theoretical_loss": 3.46908538697825, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773737373737374, + "loss": 2.6429, + "theoretical_loss": 3.4690742787477804, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773535353535354, + "loss": 2.6554, + "theoretical_loss": 3.4690631710493496, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733333333333335, + "loss": 2.8213, + "theoretical_loss": 3.469052063882912, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773131313131313, + "loss": 2.8597, + "theoretical_loss": 3.4690409572484215, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729292929292927, + "loss": 2.725, + "theoretical_loss": 3.469029851145833, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 991641, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.545839309692383, + "objective/train/theoretical_loss": 3.469018745575102, + "objective/train/tokens_used": 110570976, + "theoretical_loss": 3.469018745575102, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772727272727273, + "loss": 2.564, + "theoretical_loss": 3.469018745575102, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772525252525253, + "loss": 2.5818, + "theoretical_loss": 3.469007640536182, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047723232323232325, + "loss": 2.5062, + "theoretical_loss": 3.4689965360290285, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772121212121212, + "loss": 2.4478, + "theoretical_loss": 3.4689854320535956, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047719191919191916, + "loss": 2.5787, + "theoretical_loss": 3.4689743286098382, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771717171717172, + "loss": 2.7638, + "theoretical_loss": 3.468963225697711, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771515151515152, + "loss": 2.5107, + "theoretical_loss": 3.4689521233171687, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713131313131314, + "loss": 2.7806, + "theoretical_loss": 3.468941021468166, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771111111111111, + "loss": 2.6073, + "theoretical_loss": 3.468929920150657, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047709090909090905, + "loss": 2.5215, + "theoretical_loss": 3.468918819364597, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770707070707071, + "loss": 2.8717, + "theoretical_loss": 3.4689077191099402, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770505050505051, + "loss": 2.5004, + "theoretical_loss": 3.4688966193866415, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047703030303030303, + "loss": 2.593, + "theoretical_loss": 3.468885520194656, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477010101010101, + "loss": 2.7877, + "theoretical_loss": 3.468874421533938, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476989898989899, + "loss": 2.6607, + "theoretical_loss": 3.4688633234044426, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476969696969697, + "loss": 2.7612, + "theoretical_loss": 3.468852225806123, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047694949494949497, + "loss": 2.7201, + "theoretical_loss": 3.4688411287389362, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769292929292929, + "loss": 2.7394, + "theoretical_loss": 3.4688300322028356, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769090909090909, + "loss": 2.4616, + "theoretical_loss": 3.468818936197776, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768888888888889, + "loss": 2.6833, + "theoretical_loss": 3.4688078407237124, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768686868686869, + "loss": 2.672, + "theoretical_loss": 3.4687967457805993, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684848484848486, + "loss": 2.7548, + "theoretical_loss": 3.4687856513683917, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768282828282828, + "loss": 2.6912, + "theoretical_loss": 3.468774557487044, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680808080808083, + "loss": 2.5082, + "theoretical_loss": 3.4687634641365115, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767878787878788, + "loss": 2.9058, + "theoretical_loss": 3.4687523713167483, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 992944, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7462427616119385, + "objective/train/theoretical_loss": 3.468741279027709, + "objective/train/tokens_used": 112209376, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767676767676768, + "loss": 2.7802, + "theoretical_loss": 3.468741279027709, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047674747474747475, + "loss": 2.5903, + "theoretical_loss": 3.4687301872693497, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767272727272727, + "loss": 2.4471, + "theoretical_loss": 3.4687190960416245, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767070707070707, + "loss": 2.5718, + "theoretical_loss": 3.468708005344487, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047668686868686873, + "loss": 2.7637, + "theoretical_loss": 3.468696915177894, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766666666666667, + "loss": 3.0175, + "theoretical_loss": 3.4686858255417987, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047664646464646465, + "loss": 2.8395, + "theoretical_loss": 3.4686747364361565, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662626262626266, + "loss": 2.5404, + "theoretical_loss": 3.4686636478609225, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766060606060606, + "loss": 2.5444, + "theoretical_loss": 3.4686525598160514, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765858585858586, + "loss": 2.4875, + "theoretical_loss": 3.4686414723014973, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765656565656566, + "loss": 2.6743, + "theoretical_loss": 3.4686303853172165, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654545454545454, + "loss": 2.8086, + "theoretical_loss": 3.468619298863162, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047652525252525255, + "loss": 2.7055, + "theoretical_loss": 3.46860821293929, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765050505050505, + "loss": 2.4811, + "theoretical_loss": 3.4685971275455545, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764848484848485, + "loss": 2.698, + "theoretical_loss": 3.4685860426819106, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764646464646465, + "loss": 2.7089, + "theoretical_loss": 3.468574958348314, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764444444444445, + "loss": 2.701, + "theoretical_loss": 3.468563874544718, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047642424242424244, + "loss": 2.735, + "theoretical_loss": 3.4685527912710787, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764040404040404, + "loss": 2.7894, + "theoretical_loss": 3.4685417085273507, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763838383838384, + "loss": 2.7911, + "theoretical_loss": 3.468530626313488, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047636363636363637, + "loss": 2.6642, + "theoretical_loss": 3.468519544629447, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763434343434344, + "loss": 2.5293, + "theoretical_loss": 3.468508463475181, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047632323232323233, + "loss": 2.5761, + "theoretical_loss": 3.4684973828506465, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763030303030303, + "loss": 2.755, + "theoretical_loss": 3.468486302755797, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762828282828283, + "loss": 2.7845, + "theoretical_loss": 3.4684752231905875, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 993452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8829641342163086, + "objective/train/theoretical_loss": 3.468464144154974, + "objective/train/tokens_used": 113847776, + "theoretical_loss": 3.468464144154974, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762626262626263, + "loss": 2.5102, + "theoretical_loss": 3.468464144154974, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047624242424242427, + "loss": 2.7548, + "theoretical_loss": 3.468453065648911, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762222222222222, + "loss": 2.9095, + "theoretical_loss": 3.468441987672353, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762020202020202, + "loss": 2.6763, + "theoretical_loss": 3.468430910225255, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761818181818182, + "loss": 2.7464, + "theoretical_loss": 3.468419833307572, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761616161616162, + "loss": 2.8318, + "theoretical_loss": 3.468408756919259, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614141414141416, + "loss": 2.7317, + "theoretical_loss": 3.468397681060271, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761212121212121, + "loss": 2.8319, + "theoretical_loss": 3.4683866057305623, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761010101010101, + "loss": 2.6388, + "theoretical_loss": 3.468375530930089, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760808080808081, + "loss": 2.5385, + "theoretical_loss": 3.468364456658805, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760606060606061, + "loss": 2.6943, + "theoretical_loss": 3.4683533829166664, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047604040404040406, + "loss": 2.8349, + "theoretical_loss": 3.468342309703627, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476020202020202, + "loss": 2.4958, + "theoretical_loss": 3.4683312370196426, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047599999999999997, + "loss": 2.8273, + "theoretical_loss": 3.4683201648646675, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047597979797979803, + "loss": 2.5125, + "theoretical_loss": 3.4683090932386573, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475959595959596, + "loss": 2.9006, + "theoretical_loss": 3.4682980221415662, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047593939393939395, + "loss": 2.6251, + "theoretical_loss": 3.46828695157335, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759191919191919, + "loss": 2.7671, + "theoretical_loss": 3.4682758815339634, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589898989898986, + "loss": 2.6265, + "theoretical_loss": 3.4682648120233615, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047587878787878793, + "loss": 2.4114, + "theoretical_loss": 3.4682537430414992, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758585858585859, + "loss": 2.4871, + "theoretical_loss": 3.468242674588331, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047583838383838384, + "loss": 2.6516, + "theoretical_loss": 3.468231606663813, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758181818181818, + "loss": 2.8708, + "theoretical_loss": 3.4682205392678993, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757979797979798, + "loss": 2.5949, + "theoretical_loss": 3.4682094724005457, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757777777777778, + "loss": 2.4427, + "theoretical_loss": 3.4681984060617066, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 994709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5573644638061523, + "objective/train/theoretical_loss": 3.4681873402513372, + "objective/train/tokens_used": 115486176, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757575757575758, + "loss": 2.6787, + "theoretical_loss": 3.4681873402513372, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573737373737373, + "loss": 2.8066, + "theoretical_loss": 3.4681762749693927, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171717171717, + "loss": 2.6246, + "theoretical_loss": 3.4681652102158282, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756969696969697, + "loss": 2.6569, + "theoretical_loss": 3.4681541459905985, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756767676767677, + "loss": 2.9824, + "theoretical_loss": 3.468143082293659, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565656565656567, + "loss": 2.5965, + "theoretical_loss": 3.4681320191249645, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756363636363636, + "loss": 2.5562, + "theoretical_loss": 3.46812095648447, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047561616161616164, + "loss": 2.8371, + "theoretical_loss": 3.468109894372131, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755959595959596, + "loss": 2.7046, + "theoretical_loss": 3.468098832787902, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755757575757576, + "loss": 2.6512, + "theoretical_loss": 3.4680877717317387, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047555555555555556, + "loss": 2.512, + "theoretical_loss": 3.4680767112035955, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755353535353535, + "loss": 2.5847, + "theoretical_loss": 3.4680656512034282, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551515151515153, + "loss": 2.6819, + "theoretical_loss": 3.4680545917311916, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754949494949495, + "loss": 2.5282, + "theoretical_loss": 3.468043532786841, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754747474747475, + "loss": 2.6662, + "theoretical_loss": 3.4680324743703315, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047545454545454545, + "loss": 2.7326, + "theoretical_loss": 3.4680214164816174, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543434343434347, + "loss": 2.6597, + "theoretical_loss": 3.468010359120655, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754141414141414, + "loss": 2.6871, + "theoretical_loss": 3.467999302287399, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753939393939394, + "loss": 2.6719, + "theoretical_loss": 3.4679882459818043, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753737373737374, + "loss": 2.8148, + "theoretical_loss": 3.4679771902038263, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535353535353535, + "loss": 2.6092, + "theoretical_loss": 3.4679661349534197, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047533333333333336, + "loss": 2.6743, + "theoretical_loss": 3.4679550802305408, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753131313131313, + "loss": 2.6968, + "theoretical_loss": 3.467944026035144, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047529292929292927, + "loss": 2.5563, + "theoretical_loss": 3.4679329723671835, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752727272727273, + "loss": 2.4083, + "theoretical_loss": 3.467921919226616, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 995465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.467454433441162, + "objective/train/theoretical_loss": 3.4679108666133964, + "objective/train/tokens_used": 117124576, + "theoretical_loss": 3.4679108666133964, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752525252525253, + "loss": 2.473, + "theoretical_loss": 3.4679108666133964, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047523232323232325, + "loss": 2.7342, + "theoretical_loss": 3.4678998145274793, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752121212121212, + "loss": 2.5775, + "theoretical_loss": 3.467888762968821, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519191919191916, + "loss": 2.7686, + "theoretical_loss": 3.467877711937375, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751717171717172, + "loss": 2.858, + "theoretical_loss": 3.4678666614330975, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751515151515152, + "loss": 2.6743, + "theoretical_loss": 3.467855611455944, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047513131313131314, + "loss": 2.751, + "theoretical_loss": 3.467844562005869, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751111111111111, + "loss": 2.5867, + "theoretical_loss": 3.4678335130828284, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047509090909090906, + "loss": 2.3742, + "theoretical_loss": 3.4678224646867766, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750707070707071, + "loss": 2.5802, + "theoretical_loss": 3.4678114168176695, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750505050505051, + "loss": 2.7136, + "theoretical_loss": 3.467800369475462, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503030303030304, + "loss": 2.6935, + "theoretical_loss": 3.46778932266011, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475010101010101, + "loss": 2.9247, + "theoretical_loss": 3.467778276371568, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474989898989899, + "loss": 2.625, + "theoretical_loss": 3.467767230609791, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474969696969697, + "loss": 2.5746, + "theoretical_loss": 3.4677561853747356, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494949494949497, + "loss": 2.9209, + "theoretical_loss": 3.4677451406663558, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047492929292929293, + "loss": 3.0067, + "theoretical_loss": 3.467734096484607, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749090909090909, + "loss": 2.6645, + "theoretical_loss": 3.467723052829445, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047488888888888895, + "loss": 2.71, + "theoretical_loss": 3.4677120097008247, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748686868686869, + "loss": 2.6275, + "theoretical_loss": 3.4677009670987013, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047484848484848487, + "loss": 2.5603, + "theoretical_loss": 3.4676899250230306, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748282828282828, + "loss": 2.6442, + "theoretical_loss": 3.4676788834737677, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748080808080808, + "loss": 2.7677, + "theoretical_loss": 3.467667842450868, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478787878787884, + "loss": 2.7646, + "theoretical_loss": 3.467656801954286, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747676767676768, + "loss": 2.7292, + "theoretical_loss": 3.4676457619839773, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 996601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9863336086273193, + "objective/train/theoretical_loss": 3.4676347225398985, + "objective/train/tokens_used": 118762976, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047474747474747476, + "loss": 2.6641, + "theoretical_loss": 3.4676347225398985, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747272727272727, + "loss": 2.3739, + "theoretical_loss": 3.467623683622003, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470707070707067, + "loss": 2.558, + "theoretical_loss": 3.467612645230248, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047468686868686874, + "loss": 2.7125, + "theoretical_loss": 3.4676016073645872, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746666666666667, + "loss": 2.7329, + "theoretical_loss": 3.467590570024977, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047464646464646465, + "loss": 2.8293, + "theoretical_loss": 3.4675795332113726, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746262626262626, + "loss": 2.7241, + "theoretical_loss": 3.467568496923729, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746060606060606, + "loss": 2.906, + "theoretical_loss": 3.4675574611620013, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047458585858585863, + "loss": 2.7144, + "theoretical_loss": 3.4675464259261455, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745656565656566, + "loss": 2.6056, + "theoretical_loss": 3.467535391216116, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454545454545454, + "loss": 2.7132, + "theoretical_loss": 3.46752435703187, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745252525252525, + "loss": 2.7943, + "theoretical_loss": 3.4675133233733613, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745050505050505, + "loss": 2.7417, + "theoretical_loss": 3.467502290240546, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744848484848485, + "loss": 2.4408, + "theoretical_loss": 3.4674912576333785, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744646464646465, + "loss": 2.8368, + "theoretical_loss": 3.4674802255518156, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444444444444444, + "loss": 2.498, + "theoretical_loss": 3.467469193995812, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047442424242424245, + "loss": 2.7851, + "theoretical_loss": 3.4674581629653227, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744040404040404, + "loss": 2.7972, + "theoretical_loss": 3.467447132460304, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743838383838384, + "loss": 2.5276, + "theoretical_loss": 3.467436102480711, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436363636363637, + "loss": 2.8544, + "theoretical_loss": 3.4674250730264986, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047434343434343433, + "loss": 2.6348, + "theoretical_loss": 3.4674140440976227, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432323232323234, + "loss": 2.8021, + "theoretical_loss": 3.467403015694039, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743030303030303, + "loss": 2.627, + "theoretical_loss": 3.467391987815702, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742828282828283, + "loss": 2.5873, + "theoretical_loss": 3.4673809604625676, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047426262626262626, + "loss": 2.9232, + "theoretical_loss": 3.4673699336345916, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 997243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0204896926879883, + "objective/train/theoretical_loss": 3.4673589073317297, + "objective/train/tokens_used": 120401376, + "theoretical_loss": 3.4673589073317297, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742424242424243, + "loss": 2.9122, + "theoretical_loss": 3.4673589073317297, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047422222222222223, + "loss": 2.74, + "theoretical_loss": 3.4673478815539363, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742020202020202, + "loss": 2.9096, + "theoretical_loss": 3.4673368563011677, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741818181818182, + "loss": 2.7509, + "theoretical_loss": 3.467325831573379, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416161616161616, + "loss": 2.8065, + "theoretical_loss": 3.467314807370526, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047414141414141417, + "loss": 2.837, + "theoretical_loss": 3.4673037836925635, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741212121212121, + "loss": 2.7206, + "theoretical_loss": 3.467292760539448, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741010101010101, + "loss": 2.5856, + "theoretical_loss": 3.467281737911134, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740808080808081, + "loss": 2.6041, + "theoretical_loss": 3.4672707158075777, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740606060606061, + "loss": 2.8116, + "theoretical_loss": 3.467259694228734, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047404040404040406, + "loss": 2.6927, + "theoretical_loss": 3.467248673174559, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474020202020202, + "loss": 2.6445, + "theoretical_loss": 3.467237652645008, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474, + "loss": 2.5709, + "theoretical_loss": 3.4672266326400365, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473979797979798, + "loss": 2.8204, + "theoretical_loss": 3.4672156131595995, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473959595959596, + "loss": 2.6982, + "theoretical_loss": 3.4672045942036536, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047393939393939395, + "loss": 2.502, + "theoretical_loss": 3.4671935757721535, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739191919191919, + "loss": 2.7126, + "theoretical_loss": 3.467182557865055, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047389898989898987, + "loss": 2.5577, + "theoretical_loss": 3.4671715404823136, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387878787878793, + "loss": 2.7442, + "theoretical_loss": 3.4671605236238854, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738585858585859, + "loss": 2.6568, + "theoretical_loss": 3.467149507289725, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383838383838385, + "loss": 2.7499, + "theoretical_loss": 3.4671384914797887, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738181818181818, + "loss": 2.8602, + "theoretical_loss": 3.4671274761940314, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379797979797976, + "loss": 2.6233, + "theoretical_loss": 3.4671164614324095, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737777777777778, + "loss": 2.7861, + "theoretical_loss": 3.4671054471948777, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737575757575758, + "loss": 2.5657, + "theoretical_loss": 3.467094433481392, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 998400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.486372470855713, + "objective/train/theoretical_loss": 3.4670834202919085, + "objective/train/tokens_used": 122039776, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047373737373737374, + "loss": 2.6815, + "theoretical_loss": 3.4670834202919085, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737171717171717, + "loss": 2.6868, + "theoretical_loss": 3.4670724076263815, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736969696969697, + "loss": 2.6047, + "theoretical_loss": 3.4670613954847678, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736767676767677, + "loss": 2.917, + "theoretical_loss": 3.467050383867023, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736565656565657, + "loss": 2.6532, + "theoretical_loss": 3.467039372773102, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047363636363636363, + "loss": 2.7648, + "theoretical_loss": 3.467028362202961, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736161616161616, + "loss": 2.7479, + "theoretical_loss": 3.467017352156555, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735959595959596, + "loss": 2.8932, + "theoretical_loss": 3.46700634263384, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735757575757576, + "loss": 2.7239, + "theoretical_loss": 3.466995333634772, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047355555555555557, + "loss": 2.5488, + "theoretical_loss": 3.4669843251593058, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735353535353535, + "loss": 2.9252, + "theoretical_loss": 3.4669733172073975, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351515151515154, + "loss": 2.9353, + "theoretical_loss": 3.466962309779003, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734949494949495, + "loss": 2.677, + "theoretical_loss": 3.4669513028740777, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734747474747475, + "loss": 2.5051, + "theoretical_loss": 3.466940296492577, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345454545454546, + "loss": 2.7555, + "theoretical_loss": 3.466929290634457, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734343434343434, + "loss": 2.5905, + "theoretical_loss": 3.466918285299673, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341414141414143, + "loss": 2.7041, + "theoretical_loss": 3.466907280488181, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733939393939394, + "loss": 2.8517, + "theoretical_loss": 3.4668962761999365, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733737373737374, + "loss": 2.6018, + "theoretical_loss": 3.4668852724348955, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047335353535353535, + "loss": 2.6687, + "theoretical_loss": 3.4668742691930126, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047333333333333336, + "loss": 2.5944, + "theoretical_loss": 3.4668632664742454, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733131313131313, + "loss": 2.7005, + "theoretical_loss": 3.4668522642785478, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732929292929293, + "loss": 2.5989, + "theoretical_loss": 3.466841262605876, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732727272727273, + "loss": 2.6536, + "theoretical_loss": 3.466830261456187, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325252525252525, + "loss": 2.9096, + "theoretical_loss": 3.4668192608294346, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 998902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.282055139541626, + "objective/train/theoretical_loss": 3.4668082607255757, + "objective/train/tokens_used": 123678176, + "theoretical_loss": 3.4668082607255757, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047323232323232326, + "loss": 2.7772, + "theoretical_loss": 3.4668082607255757, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732121212121212, + "loss": 2.8023, + "theoretical_loss": 3.4667972611445657, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731919191919192, + "loss": 3.0177, + "theoretical_loss": 3.46678626208636, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731717171717172, + "loss": 2.7335, + "theoretical_loss": 3.466775263550915, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731515151515152, + "loss": 3.0462, + "theoretical_loss": 3.466764265538186, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047313131313131315, + "loss": 2.7058, + "theoretical_loss": 3.466753268048129, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731111111111111, + "loss": 2.7053, + "theoretical_loss": 3.4667422710806997, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730909090909091, + "loss": 2.512, + "theoretical_loss": 3.466731274635854, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730707070707071, + "loss": 2.8189, + "theoretical_loss": 3.466720278713547, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730505050505051, + "loss": 2.8209, + "theoretical_loss": 3.466709283313735, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303030303030304, + "loss": 2.6459, + "theoretical_loss": 3.4666982884363735, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473010101010101, + "loss": 2.6401, + "theoretical_loss": 3.466687294081419, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472989898989899, + "loss": 2.6206, + "theoretical_loss": 3.466676300248827, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472969696969697, + "loss": 2.8661, + "theoretical_loss": 3.4666653069385527, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472949494949495, + "loss": 2.6139, + "theoretical_loss": 3.466654314150552, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292929292929293, + "loss": 2.6546, + "theoretical_loss": 3.4666433218847814, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729090909090909, + "loss": 2.599, + "theoretical_loss": 3.466632330141196, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728888888888889, + "loss": 2.3876, + "theoretical_loss": 3.4666213389197518, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686868686869, + "loss": 2.8088, + "theoretical_loss": 3.4666103482204056, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284848484848487, + "loss": 2.6087, + "theoretical_loss": 3.466599358043111, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047282828282828283, + "loss": 2.702, + "theoretical_loss": 3.4665883683878262, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728080808080808, + "loss": 2.7677, + "theoretical_loss": 3.466577379254506, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727878787878788, + "loss": 2.7801, + "theoretical_loss": 3.4665663906431057, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727676767676768, + "loss": 2.6886, + "theoretical_loss": 3.466555402553582, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047274747474747476, + "loss": 2.9187, + "theoretical_loss": 3.466544414985891, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 999561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8934085369110107, + "objective/train/theoretical_loss": 3.4665334279399875, + "objective/train/tokens_used": 125316576, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727272727272727, + "loss": 2.6765, + "theoretical_loss": 3.4665334279399875, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727070707070707, + "loss": 2.661, + "theoretical_loss": 3.4665224414158278, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268686868686874, + "loss": 2.6852, + "theoretical_loss": 3.466511455413368, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726666666666667, + "loss": 2.7155, + "theoretical_loss": 3.466500469932564, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047264646464646466, + "loss": 2.7491, + "theoretical_loss": 3.466489484973371, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726262626262626, + "loss": 2.5644, + "theoretical_loss": 3.4664785005357457, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260606060606057, + "loss": 2.7855, + "theoretical_loss": 3.4664675166196437, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258585858585864, + "loss": 2.8941, + "theoretical_loss": 3.4664565332250215, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725656565656566, + "loss": 2.8533, + "theoretical_loss": 3.4664455503518337, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254545454545455, + "loss": 2.7847, + "theoretical_loss": 3.466434568000037, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725252525252525, + "loss": 2.2318, + "theoretical_loss": 3.4664235861695873, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725050505050505, + "loss": 2.7664, + "theoretical_loss": 3.46641260486044, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047248484848484853, + "loss": 2.7799, + "theoretical_loss": 3.466401624072552, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724646464646465, + "loss": 2.4236, + "theoretical_loss": 3.4663906438058785, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047244444444444444, + "loss": 2.8905, + "theoretical_loss": 3.4663796640603755, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724242424242424, + "loss": 2.7593, + "theoretical_loss": 3.466368684835999, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724040404040404, + "loss": 2.7997, + "theoretical_loss": 3.466357706132705, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723838383838384, + "loss": 2.52, + "theoretical_loss": 3.46634672795045, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723636363636364, + "loss": 2.6473, + "theoretical_loss": 3.466335750289189, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234343434343433, + "loss": 2.5319, + "theoretical_loss": 3.466324773148878, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047232323232323235, + "loss": 2.8377, + "theoretical_loss": 3.466313796529474, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723030303030303, + "loss": 2.6964, + "theoretical_loss": 3.466302820430932, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722828282828283, + "loss": 2.7875, + "theoretical_loss": 3.466291844853208, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226262626262627, + "loss": 2.6159, + "theoretical_loss": 3.4662808697962584, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722424242424242, + "loss": 2.6594, + "theoretical_loss": 3.466269895260039, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 1000589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8786251544952393, + "objective/train/theoretical_loss": 3.466258921244506, + "objective/train/tokens_used": 126954976, + "theoretical_loss": 3.466258921244506, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047222222222222224, + "loss": 2.8656, + "theoretical_loss": 3.466258921244506, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722020202020202, + "loss": 2.4964, + "theoretical_loss": 3.466247947749615, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721818181818182, + "loss": 2.5477, + "theoretical_loss": 3.466236974775322, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047216161616161616, + "loss": 2.5074, + "theoretical_loss": 3.4662260023215836, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721414141414142, + "loss": 2.7663, + "theoretical_loss": 3.466215030388356, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212121212121213, + "loss": 2.7106, + "theoretical_loss": 3.4662040589755936, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721010101010101, + "loss": 2.603, + "theoretical_loss": 3.466193088083254, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720808080808081, + "loss": 2.711, + "theoretical_loss": 3.466182117711293, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047206060606060606, + "loss": 2.5157, + "theoretical_loss": 3.466171147859666, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204040404040407, + "loss": 2.7149, + "theoretical_loss": 3.46616017852833, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472020202020202, + "loss": 2.5875, + "theoretical_loss": 3.4661492097172397, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472, + "loss": 2.7007, + "theoretical_loss": 3.466138241426352, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471979797979798, + "loss": 2.7404, + "theoretical_loss": 3.466127273655623, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471959595959596, + "loss": 2.8451, + "theoretical_loss": 3.466116306405009, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193939393939396, + "loss": 2.5567, + "theoretical_loss": 3.466105339674465, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719191919191919, + "loss": 2.763, + "theoretical_loss": 3.466094373463948, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718989898989899, + "loss": 2.6775, + "theoretical_loss": 3.4660834077734144, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718787878787879, + "loss": 2.7932, + "theoretical_loss": 3.466072442602819, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718585858585859, + "loss": 2.6442, + "theoretical_loss": 3.4660614779521186, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047183838383838385, + "loss": 2.7261, + "theoretical_loss": 3.4660505138212696, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718181818181818, + "loss": 2.6508, + "theoretical_loss": 3.466039550210228, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047179797979797977, + "loss": 2.7376, + "theoretical_loss": 3.4660285871189496, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177777777777783, + "loss": 2.7291, + "theoretical_loss": 3.4660176245473906, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717575757575758, + "loss": 2.7036, + "theoretical_loss": 3.4660066624955066, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173737373737374, + "loss": 2.9131, + "theoretical_loss": 3.465995700963255, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1001288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0922961235046387, + "objective/train/theoretical_loss": 3.4659847399505903, + "objective/train/tokens_used": 128593376, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717171717171717, + "loss": 2.6541, + "theoretical_loss": 3.4659847399505903, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169696969696966, + "loss": 2.6881, + "theoretical_loss": 3.46597377945747, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716767676767677, + "loss": 2.8552, + "theoretical_loss": 3.46596281948385, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716565656565657, + "loss": 2.7542, + "theoretical_loss": 3.465951860029686, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163636363636364, + "loss": 2.8537, + "theoretical_loss": 3.4659409010949345, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716161616161616, + "loss": 2.8639, + "theoretical_loss": 3.4659299426795513, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715959595959596, + "loss": 2.7626, + "theoretical_loss": 3.4659189847834924, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715757575757576, + "loss": 2.9622, + "theoretical_loss": 3.465908027406715, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715555555555556, + "loss": 2.6239, + "theoretical_loss": 3.465897070549174, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047153535353535353, + "loss": 2.698, + "theoretical_loss": 3.4658861142108264, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715151515151515, + "loss": 2.7017, + "theoretical_loss": 3.4658751583916283, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047149494949494955, + "loss": 2.6616, + "theoretical_loss": 3.4658642030915354, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714747474747475, + "loss": 2.3879, + "theoretical_loss": 3.4658532483105042, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047145454545454547, + "loss": 2.7783, + "theoretical_loss": 3.4658422940484908, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714343434343434, + "loss": 2.5489, + "theoretical_loss": 3.465831340305452, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714141414141414, + "loss": 2.4654, + "theoretical_loss": 3.4658203870813433, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139393939393944, + "loss": 2.9555, + "theoretical_loss": 3.465809434376121, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713737373737374, + "loss": 2.5514, + "theoretical_loss": 3.465798482189742, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135353535353536, + "loss": 2.8421, + "theoretical_loss": 3.465787530522161, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713333333333333, + "loss": 2.6979, + "theoretical_loss": 3.4657765793733355, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713131313131313, + "loss": 2.7339, + "theoretical_loss": 3.4657656287432212, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047129292929292934, + "loss": 2.72, + "theoretical_loss": 3.465754678631775, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712727272727273, + "loss": 2.8459, + "theoretical_loss": 3.4657437290389526, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125252525252525, + "loss": 2.4907, + "theoretical_loss": 3.4657327799647106, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712323232323232, + "loss": 2.6931, + "theoretical_loss": 3.4657218314090046, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1002447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.567986488342285, + "objective/train/theoretical_loss": 3.4657108833717913, + "objective/train/tokens_used": 130231776, + "theoretical_loss": 3.4657108833717913, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712121212121212, + "loss": 2.8232, + "theoretical_loss": 3.4657108833717913, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047119191919191923, + "loss": 2.9515, + "theoretical_loss": 3.465699935853027, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711717171717172, + "loss": 2.5623, + "theoretical_loss": 3.465688988852668, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115151515151514, + "loss": 2.8474, + "theoretical_loss": 3.46567804237067, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047113131313131316, + "loss": 2.5526, + "theoretical_loss": 3.4656670964069907, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711111111111111, + "loss": 2.8303, + "theoretical_loss": 3.4656561509615846, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710909090909091, + "loss": 2.8144, + "theoretical_loss": 3.4656452060344085, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710707070707071, + "loss": 2.7062, + "theoretical_loss": 3.4656342616254197, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047105050505050504, + "loss": 2.9743, + "theoretical_loss": 3.4656233177345737, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047103030303030305, + "loss": 2.6617, + "theoretical_loss": 3.465612374361827, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471010101010101, + "loss": 2.5412, + "theoretical_loss": 3.4656014315071357, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470989898989899, + "loss": 2.716, + "theoretical_loss": 3.4655904891704563, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047096969696969697, + "loss": 2.7766, + "theoretical_loss": 3.4655795473517452, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470949494949495, + "loss": 3.0103, + "theoretical_loss": 3.465568606050958, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047092929292929294, + "loss": 2.6769, + "theoretical_loss": 3.465557665268052, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709090909090909, + "loss": 2.5385, + "theoretical_loss": 3.4655467250029837, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708888888888889, + "loss": 2.6485, + "theoretical_loss": 3.4655357852557085, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086868686868687, + "loss": 2.5561, + "theoretical_loss": 3.4655248460261827, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708484848484849, + "loss": 2.8638, + "theoretical_loss": 3.4655139073143637, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082828282828283, + "loss": 2.7157, + "theoretical_loss": 3.465502969120207, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708080808080808, + "loss": 2.6912, + "theoretical_loss": 3.4654920314436697, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707878787878788, + "loss": 2.78, + "theoretical_loss": 3.465481094284707, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707676767676768, + "loss": 2.8943, + "theoretical_loss": 3.465470157643277, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074747474747477, + "loss": 2.8289, + "theoretical_loss": 3.4654592215193345, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707272727272727, + "loss": 3.0982, + "theoretical_loss": 3.465448285912836, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1002969, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.409536123275757, + "objective/train/theoretical_loss": 3.465437350823739, + "objective/train/tokens_used": 131870176, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707070707070707, + "loss": 2.7346, + "theoretical_loss": 3.465437350823739, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706868686868687, + "loss": 2.8237, + "theoretical_loss": 3.465426416251999, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706666666666667, + "loss": 2.5994, + "theoretical_loss": 3.4654154821975727, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047064646464646466, + "loss": 2.732, + "theoretical_loss": 3.465404548660416, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706262626262626, + "loss": 2.8376, + "theoretical_loss": 3.4653936156404868, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706060606060606, + "loss": 2.9149, + "theoretical_loss": 3.4653826831377397, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047058585858585864, + "loss": 2.6145, + "theoretical_loss": 3.465371751152132, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705656565656566, + "loss": 2.9803, + "theoretical_loss": 3.46536081968362, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047054545454545455, + "loss": 2.7682, + "theoretical_loss": 3.4653498887321605, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705252525252525, + "loss": 2.7804, + "theoretical_loss": 3.465338958297709, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047050505050505047, + "loss": 2.7824, + "theoretical_loss": 3.465328028380223, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047048484848484853, + "loss": 2.9526, + "theoretical_loss": 3.465317098979659, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704646464646465, + "loss": 2.6633, + "theoretical_loss": 3.465306170095972, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044444444444445, + "loss": 2.8091, + "theoretical_loss": 3.46529524172912, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704242424242424, + "loss": 3.0607, + "theoretical_loss": 3.4652843138790583, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704040404040404, + "loss": 2.5159, + "theoretical_loss": 3.4652733865457446, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703838383838384, + "loss": 2.6239, + "theoretical_loss": 3.465262459729135, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703636363636364, + "loss": 2.9641, + "theoretical_loss": 3.4652515334291847, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047034343434343434, + "loss": 2.6504, + "theoretical_loss": 3.4652406076458515, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703232323232323, + "loss": 2.7096, + "theoretical_loss": 3.465229682379092, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703030303030303, + "loss": 2.5763, + "theoretical_loss": 3.465218757628862, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702828282828283, + "loss": 2.7635, + "theoretical_loss": 3.4652078333951186, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702626262626263, + "loss": 2.7962, + "theoretical_loss": 3.4651969096778177, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047024242424242423, + "loss": 2.5108, + "theoretical_loss": 3.465185986476916, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022222222222224, + "loss": 2.5891, + "theoretical_loss": 3.465175063792371, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1004180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6892433166503906, + "objective/train/theoretical_loss": 3.465164141624137, + "objective/train/tokens_used": 133508576, + "theoretical_loss": 3.465164141624137, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702020202020202, + "loss": 2.8761, + "theoretical_loss": 3.465164141624137, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701818181818182, + "loss": 2.8233, + "theoretical_loss": 3.465153219972173, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047016161616161617, + "loss": 2.7882, + "theoretical_loss": 3.4651422988364335, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701414141414141, + "loss": 2.6957, + "theoretical_loss": 3.4651313782168764, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012121212121214, + "loss": 2.814, + "theoretical_loss": 3.465120458113458, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701010101010101, + "loss": 2.7846, + "theoretical_loss": 3.4651095385261343, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700808080808081, + "loss": 2.9251, + "theoretical_loss": 3.4650986194548623, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006060606060606, + "loss": 2.73, + "theoretical_loss": 3.4650877008995984, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004040404040407, + "loss": 2.7338, + "theoretical_loss": 3.4650767828602995, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047002020202020203, + "loss": 2.739, + "theoretical_loss": 3.4650658653369213, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 2.626, + "theoretical_loss": 3.4650549483294215, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469979797979798, + "loss": 2.9396, + "theoretical_loss": 3.4650440318377562, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995959595959595, + "loss": 2.6442, + "theoretical_loss": 3.465033115861882, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046993939393939396, + "loss": 2.5763, + "theoretical_loss": 3.465022200401755, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699191919191919, + "loss": 2.8594, + "theoretical_loss": 3.465011285457332, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698989898989899, + "loss": 2.842, + "theoretical_loss": 3.4650003710285704, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698787878787879, + "loss": 2.8244, + "theoretical_loss": 3.4649894571154265, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698585858585859, + "loss": 2.6777, + "theoretical_loss": 3.464978543717856, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983838383838386, + "loss": 2.9383, + "theoretical_loss": 3.464967630835816, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698181818181818, + "loss": 2.5925, + "theoretical_loss": 3.464956718469264, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697979797979798, + "loss": 2.8484, + "theoretical_loss": 3.4649458066181555, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697777777777778, + "loss": 2.8149, + "theoretical_loss": 3.4649348952824477, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697575757575758, + "loss": 2.7442, + "theoretical_loss": 3.4649239844620974, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046973737373737375, + "loss": 2.4768, + "theoretical_loss": 3.46491307415706, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697171717171717, + "loss": 2.7716, + "theoretical_loss": 3.464902164367294, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1004852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1573808193206787, + "objective/train/theoretical_loss": 3.4648912550927546, + "objective/train/tokens_used": 135146976, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696969696969697, + "loss": 2.6002, + "theoretical_loss": 3.4648912550927546, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967676767676773, + "loss": 2.7412, + "theoretical_loss": 3.4648803463333993, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696565656565657, + "loss": 2.6396, + "theoretical_loss": 3.4648694380891847, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046963636363636364, + "loss": 2.7974, + "theoretical_loss": 3.4648585303600665, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696161616161616, + "loss": 2.7612, + "theoretical_loss": 3.464847623146003, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695959595959596, + "loss": 2.9008, + "theoretical_loss": 3.464836716446949, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695757575757576, + "loss": 2.7469, + "theoretical_loss": 3.464825810262863, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695555555555556, + "loss": 2.4257, + "theoretical_loss": 3.4648149045937005, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953535353535354, + "loss": 2.8676, + "theoretical_loss": 3.464803999439418, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695151515151515, + "loss": 2.9688, + "theoretical_loss": 3.4647930947999743, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694949494949495, + "loss": 2.8084, + "theoretical_loss": 3.464782190675323, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694747474747475, + "loss": 2.9967, + "theoretical_loss": 3.4647712870654237, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945454545454547, + "loss": 2.8732, + "theoretical_loss": 3.464760383970231, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046943434343434343, + "loss": 2.6916, + "theoretical_loss": 3.4647494813897026, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694141414141414, + "loss": 2.5639, + "theoretical_loss": 3.464738579323795, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046939393939393945, + "loss": 2.9633, + "theoretical_loss": 3.4647276777724647, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693737373737374, + "loss": 2.6007, + "theoretical_loss": 3.4647167767356692, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046935353535353536, + "loss": 2.8483, + "theoretical_loss": 3.4647058762133645, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693333333333333, + "loss": 2.5303, + "theoretical_loss": 3.4646949762055077, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693131313131313, + "loss": 2.6525, + "theoretical_loss": 3.4646840767120555, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929292929292934, + "loss": 2.7351, + "theoretical_loss": 3.4646731777329647, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692727272727273, + "loss": 2.7693, + "theoretical_loss": 3.4646622792681923, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046925252525252526, + "loss": 2.8001, + "theoretical_loss": 3.4646513813176942, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692323232323232, + "loss": 2.8073, + "theoretical_loss": 3.464640483881428, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692121212121212, + "loss": 2.6543, + "theoretical_loss": 3.46462958695935, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1006253, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.731198310852051, + "objective/train/theoretical_loss": 3.4646186905514176, + "objective/train/tokens_used": 136785376, + "theoretical_loss": 3.4646186905514176, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046919191919191924, + "loss": 2.7954, + "theoretical_loss": 3.4646186905514176, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691717171717172, + "loss": 2.6645, + "theoretical_loss": 3.464607794657587, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915151515151515, + "loss": 2.8509, + "theoretical_loss": 3.464596899277815, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691313131313131, + "loss": 2.7726, + "theoretical_loss": 3.464586004412059, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691111111111111, + "loss": 2.6649, + "theoretical_loss": 3.464575110060275, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046909090909090913, + "loss": 2.7067, + "theoretical_loss": 3.4645642162224206, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690707070707071, + "loss": 2.6392, + "theoretical_loss": 3.4645533228984515, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905050505050504, + "loss": 2.544, + "theoretical_loss": 3.464542430088326, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046903030303030305, + "loss": 2.6922, + "theoretical_loss": 3.4645315377920003, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469010101010101, + "loss": 2.5364, + "theoretical_loss": 3.46452064600943, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468989898989899, + "loss": 2.7659, + "theoretical_loss": 3.4645097547405745, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468969696969697, + "loss": 2.6646, + "theoretical_loss": 3.4644988639853884, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046894949494949494, + "loss": 2.738, + "theoretical_loss": 3.4644879737438297, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892929292929295, + "loss": 2.725, + "theoretical_loss": 3.4644770840158543, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689090909090909, + "loss": 2.675, + "theoretical_loss": 3.46446619480142, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688888888888889, + "loss": 2.6913, + "theoretical_loss": 3.4644553061004832, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046886868686868687, + "loss": 2.6766, + "theoretical_loss": 3.4644444179130014, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688484848484849, + "loss": 2.5011, + "theoretical_loss": 3.46443353023893, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046882828282828284, + "loss": 2.9316, + "theoretical_loss": 3.464422643078228, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688080808080808, + "loss": 2.8756, + "theoretical_loss": 3.4644117564308505, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687878787878788, + "loss": 2.8612, + "theoretical_loss": 3.464400870296755, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046876767676767676, + "loss": 2.8945, + "theoretical_loss": 3.4643899846758983, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687474747474748, + "loss": 2.6784, + "theoretical_loss": 3.4643790995682373, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872727272727273, + "loss": 2.8563, + "theoretical_loss": 3.46436821497373, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687070707070707, + "loss": 2.6616, + "theoretical_loss": 3.464357330892331, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1006973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7338387966156006, + "objective/train/theoretical_loss": 3.464346447323999, + "objective/train/tokens_used": 138423776, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686868686868687, + "loss": 2.5273, + "theoretical_loss": 3.464346447323999, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686666666666667, + "loss": 2.8407, + "theoretical_loss": 3.4643355642686906, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864646464646467, + "loss": 2.6013, + "theoretical_loss": 3.464324681726363, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686262626262626, + "loss": 2.5447, + "theoretical_loss": 3.4643137996969724, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686060606060606, + "loss": 2.6334, + "theoretical_loss": 3.4643029181804756, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685858585858586, + "loss": 2.6813, + "theoretical_loss": 3.464292037176831, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656565656566, + "loss": 2.576, + "theoretical_loss": 3.4642811566859937, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046854545454545456, + "loss": 2.6182, + "theoretical_loss": 3.4642702767079214, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685252525252525, + "loss": 2.6574, + "theoretical_loss": 3.4642593972425715, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685050505050505, + "loss": 2.5788, + "theoretical_loss": 3.4642485182899008, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046848484848484854, + "loss": 2.7639, + "theoretical_loss": 3.464237639849866, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684646464646465, + "loss": 2.4805, + "theoretical_loss": 3.464226761922424, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046844444444444445, + "loss": 2.6411, + "theoretical_loss": 3.464215884507532, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684242424242424, + "loss": 2.839, + "theoretical_loss": 3.464205007605147, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046840404040404037, + "loss": 2.6781, + "theoretical_loss": 3.464194131215226, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838383838383843, + "loss": 2.5844, + "theoretical_loss": 3.4641832553377254, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683636363636364, + "loss": 2.7899, + "theoretical_loss": 3.464172379972603, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834343434343435, + "loss": 2.8051, + "theoretical_loss": 3.4641615051198156, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683232323232323, + "loss": 2.6852, + "theoretical_loss": 3.4641506307793204, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046830303030303026, + "loss": 2.6864, + "theoretical_loss": 3.4641397569510737, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682828282828283, + "loss": 2.9149, + "theoretical_loss": 3.464128883635033, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682626262626263, + "loss": 2.6073, + "theoretical_loss": 3.4641180108311556, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046824242424242424, + "loss": 2.504, + "theoretical_loss": 3.4641071385393976, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682222222222222, + "loss": 2.9538, + "theoretical_loss": 3.464096266759717, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682020202020202, + "loss": 2.8252, + "theoretical_loss": 3.4640853954920705, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1008497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6411519050598145, + "objective/train/theoretical_loss": 3.4640745247364153, + "objective/train/tokens_used": 140062176, + "theoretical_loss": 3.4640745247364153, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681818181818182, + "loss": 2.84, + "theoretical_loss": 3.4640745247364153, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681616161616162, + "loss": 2.7756, + "theoretical_loss": 3.4640636544927075, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046814141414141413, + "loss": 2.8645, + "theoretical_loss": 3.4640527847609057, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681212121212121, + "loss": 2.6118, + "theoretical_loss": 3.4640419155409665, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810101010101015, + "loss": 2.6474, + "theoretical_loss": 3.4640310468328455, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680808080808081, + "loss": 2.6315, + "theoretical_loss": 3.4640201786365017, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046806060606060607, + "loss": 2.5999, + "theoretical_loss": 3.4640093109518917, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468040404040404, + "loss": 2.7819, + "theoretical_loss": 3.463998443778972, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802020202020203, + "loss": 2.6685, + "theoretical_loss": 3.4639875771177, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800000000000005, + "loss": 2.5418, + "theoretical_loss": 3.4639767109680326, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467979797979798, + "loss": 2.7933, + "theoretical_loss": 3.463965845329927, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046795959595959596, + "loss": 2.5696, + "theoretical_loss": 3.46395498020334, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679393939393939, + "loss": 2.7691, + "theoretical_loss": 3.4639441155882302, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046791919191919193, + "loss": 2.8477, + "theoretical_loss": 3.463933251484553, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789898989898994, + "loss": 2.7449, + "theoretical_loss": 3.463922387892266, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678787878787879, + "loss": 2.8766, + "theoretical_loss": 3.4639115248113264, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785858585858585, + "loss": 2.606, + "theoretical_loss": 3.4639006622416915, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046783838383838386, + "loss": 2.5004, + "theoretical_loss": 3.4638898001833183, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678181818181818, + "loss": 2.6652, + "theoretical_loss": 3.463878938636164, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046779797979797983, + "loss": 2.5869, + "theoretical_loss": 3.4638680776001856, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677777777777778, + "loss": 2.9347, + "theoretical_loss": 3.4638572170753408, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046775757575757574, + "loss": 2.6284, + "theoretical_loss": 3.4638463570615854, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046773737373737376, + "loss": 2.4615, + "theoretical_loss": 3.463835497558878, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677171717171717, + "loss": 2.9666, + "theoretical_loss": 3.463824638567175, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676969696969697, + "loss": 2.7599, + "theoretical_loss": 3.4638137800864337, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1009251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5454514026641846, + "objective/train/theoretical_loss": 3.463802922116612, + "objective/train/tokens_used": 141700576, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676767676767677, + "loss": 2.6279, + "theoretical_loss": 3.463802922116612, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676565656565657, + "loss": 2.7006, + "theoretical_loss": 3.4637920646576656, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763636363636365, + "loss": 2.6826, + "theoretical_loss": 3.463781207709553, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676161616161616, + "loss": 2.8896, + "theoretical_loss": 3.463770351272231, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675959595959596, + "loss": 2.6245, + "theoretical_loss": 3.4637594953456565, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675757575757576, + "loss": 2.8262, + "theoretical_loss": 3.463748639929787, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675555555555556, + "loss": 2.6111, + "theoretical_loss": 3.4637377850245796, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753535353535354, + "loss": 2.5511, + "theoretical_loss": 3.4637269306299916, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675151515151515, + "loss": 2.5743, + "theoretical_loss": 3.46371607674598, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674949494949495, + "loss": 2.6837, + "theoretical_loss": 3.463705223372502, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674747474747475, + "loss": 2.607, + "theoretical_loss": 3.463694370509515, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674545454545455, + "loss": 2.8854, + "theoretical_loss": 3.4636835181569765, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743434343434343, + "loss": 2.6636, + "theoretical_loss": 3.4636726663148436, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674141414141414, + "loss": 2.5508, + "theoretical_loss": 3.463661814983073, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673939393939394, + "loss": 2.9104, + "theoretical_loss": 3.463650964161623, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673737373737374, + "loss": 2.7117, + "theoretical_loss": 3.4636401138504493, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735353535353537, + "loss": 2.8102, + "theoretical_loss": 3.4636292640495103, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673333333333333, + "loss": 2.761, + "theoretical_loss": 3.4636184147587636, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673131313131313, + "loss": 2.7685, + "theoretical_loss": 3.4636075659781653, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046729292929292935, + "loss": 2.6351, + "theoretical_loss": 3.4635967177076736, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672727272727273, + "loss": 2.8262, + "theoretical_loss": 3.463585869947245, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046725252525252526, + "loss": 2.8426, + "theoretical_loss": 3.463575022696838, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672323232323232, + "loss": 2.578, + "theoretical_loss": 3.4635641759564084, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672121212121212, + "loss": 2.9209, + "theoretical_loss": 3.4635533297259142, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046719191919191924, + "loss": 2.644, + "theoretical_loss": 3.463542484005313, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 1010356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.930495262145996, + "objective/train/theoretical_loss": 3.463531638794562, + "objective/train/tokens_used": 143338976, + "theoretical_loss": 3.463531638794562, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671717171717172, + "loss": 2.749, + "theoretical_loss": 3.463531638794562, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715151515151516, + "loss": 2.991, + "theoretical_loss": 3.463520794093618, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671313131313131, + "loss": 2.8403, + "theoretical_loss": 3.4635099499024387, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671111111111111, + "loss": 2.4972, + "theoretical_loss": 3.463499106220981, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709090909090913, + "loss": 2.6309, + "theoretical_loss": 3.463488263049203, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670707070707071, + "loss": 2.7881, + "theoretical_loss": 3.4634774203870617, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705050505050505, + "loss": 2.8133, + "theoretical_loss": 3.463466578234514, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467030303030303, + "loss": 2.7233, + "theoretical_loss": 3.463455736591518, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467010101010101, + "loss": 2.9342, + "theoretical_loss": 3.4634448954580304, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466989898989899, + "loss": 2.417, + "theoretical_loss": 3.463434054834009, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466969696969697, + "loss": 2.8858, + "theoretical_loss": 3.463423214719411, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694949494949494, + "loss": 2.7873, + "theoretical_loss": 3.463412375114193, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046692929292929295, + "loss": 2.9604, + "theoretical_loss": 3.4634015360183135, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669090909090909, + "loss": 2.8986, + "theoretical_loss": 3.4633906974317292, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668888888888889, + "loss": 2.3352, + "theoretical_loss": 3.463379859354398, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668686868686869, + "loss": 2.3947, + "theoretical_loss": 3.463369021786277, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046684848484848483, + "loss": 2.7602, + "theoretical_loss": 3.463358184727323, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682828282828284, + "loss": 2.6178, + "theoretical_loss": 3.4633473481774946, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668080808080808, + "loss": 2.6711, + "theoretical_loss": 3.4633365121367485, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667878787878788, + "loss": 2.9619, + "theoretical_loss": 3.463325676605042, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046676767676767677, + "loss": 2.739, + "theoretical_loss": 3.4633148415823323, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667474747474748, + "loss": 2.8322, + "theoretical_loss": 3.4633040070685777, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046672727272727274, + "loss": 2.7512, + "theoretical_loss": 3.463293173063735, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667070707070707, + "loss": 2.6775, + "theoretical_loss": 3.463282339567762, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666868686868687, + "loss": 2.6827, + "theoretical_loss": 3.4632715065806154, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1010805, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.406250238418579, + "objective/train/theoretical_loss": 3.463260674102253, + "objective/train/tokens_used": 144977376, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046666666666666666, + "loss": 2.946, + "theoretical_loss": 3.463260674102253, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666464646464647, + "loss": 2.8226, + "theoretical_loss": 3.4632498421326328, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662626262626263, + "loss": 2.4439, + "theoretical_loss": 3.4632390106717112, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666060606060606, + "loss": 3.0341, + "theoretical_loss": 3.4632281797194464, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665858585858586, + "loss": 2.739, + "theoretical_loss": 3.463217349275796, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665656565656566, + "loss": 2.848, + "theoretical_loss": 3.4632065193407167, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654545454545457, + "loss": 2.7128, + "theoretical_loss": 3.4631956899141665, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665252525252525, + "loss": 2.5013, + "theoretical_loss": 3.463184860996103, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665050505050505, + "loss": 2.8272, + "theoretical_loss": 3.4631740325864833, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664848484848485, + "loss": 2.7579, + "theoretical_loss": 3.463163204685265, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664646464646465, + "loss": 2.7442, + "theoretical_loss": 3.4631523772924053, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644444444444446, + "loss": 2.9351, + "theoretical_loss": 3.463141550407862, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664242424242424, + "loss": 2.5088, + "theoretical_loss": 3.4631307240315934, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640404040404037, + "loss": 2.9525, + "theoretical_loss": 3.463119898163555, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663838383838384, + "loss": 2.7689, + "theoretical_loss": 3.463109072803706, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663636363636364, + "loss": 2.8355, + "theoretical_loss": 3.4630982479520034, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046634343434343435, + "loss": 2.825, + "theoretical_loss": 3.4630874236084046, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663232323232323, + "loss": 2.6632, + "theoretical_loss": 3.4630765997728674, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663030303030303, + "loss": 2.6517, + "theoretical_loss": 3.463065776445349, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046628282828282833, + "loss": 3.0371, + "theoretical_loss": 3.463054953625807, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662626262626263, + "loss": 2.7771, + "theoretical_loss": 3.4630441313141986, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624242424242424, + "loss": 2.8047, + "theoretical_loss": 3.4630333095104824, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662222222222222, + "loss": 2.6366, + "theoretical_loss": 3.463022488214615, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662020202020202, + "loss": 2.7071, + "theoretical_loss": 3.4630116674265548, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661818181818182, + "loss": 2.8657, + "theoretical_loss": 3.463000847146258, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1011887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5470471382141113, + "objective/train/theoretical_loss": 3.4629900273736833, + "objective/train/tokens_used": 146615776, + "theoretical_loss": 3.4629900273736833, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661616161616162, + "loss": 2.6649, + "theoretical_loss": 3.4629900273736833, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614141414141414, + "loss": 2.4893, + "theoretical_loss": 3.462979208108788, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661212121212121, + "loss": 2.8209, + "theoretical_loss": 3.462968389351529, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610101010101016, + "loss": 2.7423, + "theoretical_loss": 3.4629575711018648, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660808080808081, + "loss": 2.8264, + "theoretical_loss": 3.462946753359753, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606060606060607, + "loss": 2.5454, + "theoretical_loss": 3.4629359361251506, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046604040404040403, + "loss": 2.7781, + "theoretical_loss": 3.4629251193980153, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466020202020202, + "loss": 2.7413, + "theoretical_loss": 3.462914303178305, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046600000000000005, + "loss": 2.6839, + "theoretical_loss": 3.4629034874659768, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465979797979798, + "loss": 3.0989, + "theoretical_loss": 3.4628926722609887, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595959595959597, + "loss": 2.641, + "theoretical_loss": 3.4628818575632985, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659393939393939, + "loss": 2.7021, + "theoretical_loss": 3.4628710433728633, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591919191919193, + "loss": 2.6398, + "theoretical_loss": 3.462860229689641, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046589898989898994, + "loss": 2.8992, + "theoretical_loss": 3.4628494165135892, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658787878787879, + "loss": 2.6784, + "theoretical_loss": 3.4628386038446655, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046585858585858586, + "loss": 2.7633, + "theoretical_loss": 3.4628277916828276, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658383838383838, + "loss": 2.7998, + "theoretical_loss": 3.462816980028033, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658181818181818, + "loss": 2.8293, + "theoretical_loss": 3.46280616888024, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046579797979797984, + "loss": 2.5696, + "theoretical_loss": 3.462795358239405, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657777777777778, + "loss": 2.9313, + "theoretical_loss": 3.462784548105487, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575757575757575, + "loss": 2.6595, + "theoretical_loss": 3.4627737384784423, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046573737373737376, + "loss": 2.7146, + "theoretical_loss": 3.46276292935823, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171717171717, + "loss": 2.7205, + "theoretical_loss": 3.4627521207448066, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046569696969696973, + "loss": 2.6487, + "theoretical_loss": 3.4627413126381303, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656767676767677, + "loss": 2.5697, + "theoretical_loss": 3.462730505038159, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1012559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.73884916305542, + "objective/train/theoretical_loss": 3.46271969794485, + "objective/train/tokens_used": 148254176, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565656565656564, + "loss": 2.6804, + "theoretical_loss": 3.46271969794485, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046563636363636365, + "loss": 2.8008, + "theoretical_loss": 3.4627088913581607, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656161616161616, + "loss": 2.7929, + "theoretical_loss": 3.4626980852780496, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655959595959596, + "loss": 2.6424, + "theoretical_loss": 3.4626872797044737, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655757575757576, + "loss": 2.6583, + "theoretical_loss": 3.4626764746373913, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655555555555556, + "loss": 3.0106, + "theoretical_loss": 3.4626656700767597, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553535353535355, + "loss": 2.5664, + "theoretical_loss": 3.462654866022537, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655151515151515, + "loss": 2.6954, + "theoretical_loss": 3.4626440624746797, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654949494949495, + "loss": 2.7787, + "theoretical_loss": 3.4626332594331473, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046547474747474747, + "loss": 2.753, + "theoretical_loss": 3.462622456897897, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654545454545455, + "loss": 2.5064, + "theoretical_loss": 3.4626116548688857, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046543434343434344, + "loss": 2.744, + "theoretical_loss": 3.462600853346072, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654141414141414, + "loss": 2.7535, + "theoretical_loss": 3.4625900523294133, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653939393939394, + "loss": 2.6721, + "theoretical_loss": 3.462579251818867, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653737373737374, + "loss": 2.7118, + "theoretical_loss": 3.4625684518143918, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653535353535354, + "loss": 2.7511, + "theoretical_loss": 3.4625576523159447, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533333333333333, + "loss": 2.7749, + "theoretical_loss": 3.4625468533234836, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653131313131313, + "loss": 2.7252, + "theoretical_loss": 3.4625360548369666, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652929292929293, + "loss": 2.799, + "theoretical_loss": 3.462525256856351, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652727272727273, + "loss": 2.6464, + "theoretical_loss": 3.462514459381595, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525252525252527, + "loss": 2.6659, + "theoretical_loss": 3.462503662412656, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652323232323232, + "loss": 2.9181, + "theoretical_loss": 3.4624928659494927, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652121212121212, + "loss": 2.5982, + "theoretical_loss": 3.4624820699920615, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519191919191925, + "loss": 2.9492, + "theoretical_loss": 3.462471274540321, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651717171717172, + "loss": 2.7015, + "theoretical_loss": 3.4624604795942293, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1013169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3495373725891113, + "objective/train/theoretical_loss": 3.462449685153744, + "objective/train/tokens_used": 149892576, + "theoretical_loss": 3.462449685153744, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046515151515151516, + "loss": 2.8181, + "theoretical_loss": 3.462449685153744, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651313131313131, + "loss": 2.8172, + "theoretical_loss": 3.462438891218822, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651111111111111, + "loss": 2.9925, + "theoretical_loss": 3.4624280977894224, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046509090909090914, + "loss": 2.8042, + "theoretical_loss": 3.462417304865502, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650707070707071, + "loss": 2.8083, + "theoretical_loss": 3.4624065124470196, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046505050505050505, + "loss": 2.8893, + "theoretical_loss": 3.462395720533933, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465030303030303, + "loss": 2.8295, + "theoretical_loss": 3.462384929126199, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046501010101010097, + "loss": 2.4364, + "theoretical_loss": 3.462374138223776, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046498989898989903, + "loss": 2.7849, + "theoretical_loss": 3.4623633478266225, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464969696969697, + "loss": 2.6153, + "theoretical_loss": 3.4623525579346954, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046494949494949495, + "loss": 2.7682, + "theoretical_loss": 3.462341768547953, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649292929292929, + "loss": 2.6638, + "theoretical_loss": 3.462330979666353, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649090909090909, + "loss": 2.8313, + "theoretical_loss": 3.462320191289854, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648888888888889, + "loss": 2.5548, + "theoretical_loss": 3.4623094034184128, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648686868686869, + "loss": 2.6759, + "theoretical_loss": 3.4622986160519877, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484848484848484, + "loss": 2.702, + "theoretical_loss": 3.4622878291905366, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648282828282828, + "loss": 2.734, + "theoretical_loss": 3.4622770428340184, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648080808080808, + "loss": 2.7279, + "theoretical_loss": 3.4622662569823888, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647878787878788, + "loss": 2.7667, + "theoretical_loss": 3.4622554716356078, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647676767676768, + "loss": 2.848, + "theoretical_loss": 3.462244686793632, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046474747474747473, + "loss": 2.6142, + "theoretical_loss": 3.46223390245642, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472727272727274, + "loss": 2.6433, + "theoretical_loss": 3.4622231186239296, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647070707070707, + "loss": 2.5263, + "theoretical_loss": 3.4622123352961185, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646868686868687, + "loss": 2.744, + "theoretical_loss": 3.462201552472945, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046466666666666667, + "loss": 2.7105, + "theoretical_loss": 3.4621907701543666, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1014342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.994889974594116, + "objective/train/theoretical_loss": 3.462179988340341, + "objective/train/tokens_used": 151530976, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646464646464646, + "loss": 2.763, + "theoretical_loss": 3.462179988340341, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462626262626264, + "loss": 2.4245, + "theoretical_loss": 3.4621692070308274, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046460606060606065, + "loss": 2.6451, + "theoretical_loss": 3.4621584262257823, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645858585858586, + "loss": 2.7631, + "theoretical_loss": 3.4621476459251648, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046456565656565656, + "loss": 2.8297, + "theoretical_loss": 3.4621368661289322, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454545454545457, + "loss": 2.8989, + "theoretical_loss": 3.462126086837042, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046452525252525253, + "loss": 2.4307, + "theoretical_loss": 3.4621153080494538, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450505050505054, + "loss": 2.704, + "theoretical_loss": 3.462104529766124, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644848484848485, + "loss": 2.6389, + "theoretical_loss": 3.462093751987011, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446464646464645, + "loss": 2.7716, + "theoretical_loss": 3.4620829747120734, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046444444444444446, + "loss": 2.8243, + "theoretical_loss": 3.4620721979412687, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644242424242424, + "loss": 2.6771, + "theoretical_loss": 3.462061421674554, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046440404040404043, + "loss": 2.7131, + "theoretical_loss": 3.46205064591189, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643838383838384, + "loss": 2.9494, + "theoretical_loss": 3.462039870653231, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643636363636364, + "loss": 2.872, + "theoretical_loss": 3.4620290958985382, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434343434343436, + "loss": 2.9053, + "theoretical_loss": 3.462018321647768, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643232323232323, + "loss": 2.604, + "theoretical_loss": 3.462007547900879, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643030303030303, + "loss": 2.9024, + "theoretical_loss": 3.4619967746578286, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828282828283, + "loss": 2.6515, + "theoretical_loss": 3.4619860019185755, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642626262626263, + "loss": 2.6749, + "theoretical_loss": 3.4619752296830772, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424242424242425, + "loss": 2.5952, + "theoretical_loss": 3.461964457951292, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642222222222222, + "loss": 2.7708, + "theoretical_loss": 3.461953686723178, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642020202020202, + "loss": 2.6752, + "theoretical_loss": 3.461942915998694, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046418181818181823, + "loss": 2.7009, + "theoretical_loss": 3.4619321457777965, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641616161616162, + "loss": 2.6824, + "theoretical_loss": 3.4619213760604444, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1015045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.449554204940796, + "objective/train/theoretical_loss": 3.4619106068465957, + "objective/train/tokens_used": 153169376, + "theoretical_loss": 3.4619106068465957, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046414141414141414, + "loss": 2.7771, + "theoretical_loss": 3.4619106068465957, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641212121212121, + "loss": 2.6351, + "theoretical_loss": 3.4618998381362087, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641010101010101, + "loss": 2.5257, + "theoretical_loss": 3.4618890699292413, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640808080808081, + "loss": 2.4698, + "theoretical_loss": 3.461878302225651, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640606060606061, + "loss": 2.9304, + "theoretical_loss": 3.4618675350253967, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046404040404040403, + "loss": 2.7184, + "theoretical_loss": 3.4618567683284365, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464020202020202, + "loss": 2.7699, + "theoretical_loss": 3.4618460021347275, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400000000000006, + "loss": 2.7353, + "theoretical_loss": 3.4618352364442293, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463979797979798, + "loss": 2.6306, + "theoretical_loss": 3.4618244712568984, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046395959595959597, + "loss": 2.5673, + "theoretical_loss": 3.4618137065726944, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393939393939393, + "loss": 2.704, + "theoretical_loss": 3.4618029423915746, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639191919191919, + "loss": 2.7585, + "theoretical_loss": 3.461792178713497, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389898989898995, + "loss": 2.6957, + "theoretical_loss": 3.46178141553842, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638787878787879, + "loss": 2.5954, + "theoretical_loss": 3.461770652866302, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385858585858586, + "loss": 2.4982, + "theoretical_loss": 3.4617598906971008, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638383838383838, + "loss": 2.4389, + "theoretical_loss": 3.4617491290307743, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381818181818183, + "loss": 2.918, + "theoretical_loss": 3.461738367867281, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046379797979797984, + "loss": 2.6122, + "theoretical_loss": 3.461727607206579, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637777777777778, + "loss": 2.7881, + "theoretical_loss": 3.461716847048627, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375757575757576, + "loss": 2.8287, + "theoretical_loss": 3.461706087393382, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637373737373737, + "loss": 3.0034, + "theoretical_loss": 3.461695328240803, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637171717171717, + "loss": 2.5419, + "theoretical_loss": 3.461684569590848, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046369696969696974, + "loss": 2.7493, + "theoretical_loss": 3.461673811443475, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636767676767677, + "loss": 2.662, + "theoretical_loss": 3.4616630537986426, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046365656565656565, + "loss": 2.7129, + "theoretical_loss": 3.4616522966563084, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1016473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8136777877807617, + "objective/train/theoretical_loss": 3.461641540016431, + "objective/train/tokens_used": 154807776, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046363636363636366, + "loss": 2.8604, + "theoretical_loss": 3.461641540016431, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636161616161616, + "loss": 2.7494, + "theoretical_loss": 3.4616307838789684, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359595959595963, + "loss": 2.7422, + "theoretical_loss": 3.4616200282438783, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635757575757576, + "loss": 2.7958, + "theoretical_loss": 3.4616092731111205, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046355555555555554, + "loss": 2.5943, + "theoretical_loss": 3.4615985184806517, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046353535353535355, + "loss": 2.5225, + "theoretical_loss": 3.461587764352431, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635151515151515, + "loss": 2.7002, + "theoretical_loss": 3.4615770107264154, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634949494949495, + "loss": 2.5169, + "theoretical_loss": 3.461566257602565, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634747474747475, + "loss": 2.7956, + "theoretical_loss": 3.4615555049808364, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634545454545455, + "loss": 2.8636, + "theoretical_loss": 3.4615447528611885, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343434343434345, + "loss": 2.6231, + "theoretical_loss": 3.46153400124358, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634141414141414, + "loss": 2.6944, + "theoretical_loss": 3.461523250127968, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633939393939394, + "loss": 2.5458, + "theoretical_loss": 3.4615124995143116, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046337373737373737, + "loss": 2.6413, + "theoretical_loss": 3.4615017494025686, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633535353535354, + "loss": 2.961, + "theoretical_loss": 3.461490999792698, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046333333333333334, + "loss": 2.994, + "theoretical_loss": 3.461480250684657, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633131313131313, + "loss": 2.8472, + "theoretical_loss": 3.4614695020784048, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632929292929293, + "loss": 2.6766, + "theoretical_loss": 3.4614587539738992, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632727272727273, + "loss": 2.6869, + "theoretical_loss": 3.4614480063710986, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632525252525253, + "loss": 2.696, + "theoretical_loss": 3.461437259269961, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323232323232323, + "loss": 2.7113, + "theoretical_loss": 3.4614265126704455, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632121212121212, + "loss": 2.6412, + "theoretical_loss": 3.46141576657251, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631919191919192, + "loss": 2.7324, + "theoretical_loss": 3.461405020976112, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631717171717172, + "loss": 2.6534, + "theoretical_loss": 3.4613942758812106, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046315151515151517, + "loss": 2.7589, + "theoretical_loss": 3.4613835312877645, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1017203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4702134132385254, + "objective/train/theoretical_loss": 3.461372787195731, + "objective/train/tokens_used": 156446176, + "theoretical_loss": 3.461372787195731, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631313131313131, + "loss": 2.727, + "theoretical_loss": 3.461372787195731, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631111111111111, + "loss": 2.5825, + "theoretical_loss": 3.461362043605069, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630909090909091, + "loss": 2.7447, + "theoretical_loss": 3.4613513005157373, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630707070707071, + "loss": 2.8931, + "theoretical_loss": 3.4613405579276932, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046305050505050506, + "loss": 2.7087, + "theoretical_loss": 3.4613298158408954, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463030303030303, + "loss": 2.6264, + "theoretical_loss": 3.461319074255303, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301010101010097, + "loss": 2.7158, + "theoretical_loss": 3.461308333170873, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298989898989904, + "loss": 2.7337, + "theoretical_loss": 3.461297592587565, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462969696969697, + "loss": 2.6001, + "theoretical_loss": 3.461286852505336, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294949494949495, + "loss": 2.7455, + "theoretical_loss": 3.4612761129241463, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629292929292929, + "loss": 2.8433, + "theoretical_loss": 3.4612653738439523, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629090909090909, + "loss": 2.5658, + "theoretical_loss": 3.4612546352647136, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046288888888888893, + "loss": 2.6168, + "theoretical_loss": 3.461243897186388, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628686868686869, + "loss": 2.475, + "theoretical_loss": 3.461233159608934, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284848484848484, + "loss": 2.7468, + "theoretical_loss": 3.4612224225323103, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628282828282828, + "loss": 2.5965, + "theoretical_loss": 3.461211685956475, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046280808080808087, + "loss": 2.7695, + "theoretical_loss": 3.4612009498813867, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627878787878788, + "loss": 2.6748, + "theoretical_loss": 3.4611902143070035, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627676767676768, + "loss": 2.5764, + "theoretical_loss": 3.4611794792332837, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274747474747474, + "loss": 2.5994, + "theoretical_loss": 3.4611687446601866, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627272727272727, + "loss": 2.505, + "theoretical_loss": 3.4611580105876696, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046270707070707076, + "loss": 2.628, + "theoretical_loss": 3.4611472770156917, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626868686868687, + "loss": 2.7478, + "theoretical_loss": 3.4611365439442103, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626666666666667, + "loss": 2.6562, + "theoretical_loss": 3.4611258113731855, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046264646464646463, + "loss": 2.8, + "theoretical_loss": 3.4611150793025747, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1018428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466665267944336, + "objective/train/theoretical_loss": 3.461104347732337, + "objective/train/tokens_used": 158084576, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046262626262626264, + "loss": 2.6652, + "theoretical_loss": 3.461104347732337, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260606060606065, + "loss": 2.5963, + "theoretical_loss": 3.4610936166624295, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625858585858586, + "loss": 2.6544, + "theoretical_loss": 3.4610828860928122, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046256565656565657, + "loss": 2.5933, + "theoretical_loss": 3.4610721560234428, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625454545454545, + "loss": 2.5568, + "theoretical_loss": 3.46106142645428, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252525252525253, + "loss": 2.6386, + "theoretical_loss": 3.4610506973852813, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046250505050505054, + "loss": 2.6948, + "theoretical_loss": 3.4610399688164066, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624848484848485, + "loss": 2.681, + "theoretical_loss": 3.4610292407476138, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046246464646464646, + "loss": 2.5926, + "theoretical_loss": 3.4610185131788613, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244444444444447, + "loss": 2.6342, + "theoretical_loss": 3.461007786110108, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624242424242424, + "loss": 2.6162, + "theoretical_loss": 3.4609970595413113, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046240404040404044, + "loss": 2.681, + "theoretical_loss": 3.4609863334724307, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623838383838384, + "loss": 2.7124, + "theoretical_loss": 3.460975607903425, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236363636363635, + "loss": 2.6313, + "theoretical_loss": 3.460964882834251, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046234343434343436, + "loss": 2.7182, + "theoretical_loss": 3.460954158264869, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623232323232323, + "loss": 2.7154, + "theoretical_loss": 3.460943434195237, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230303030303033, + "loss": 2.8216, + "theoretical_loss": 3.460932710625313, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622828282828283, + "loss": 2.8098, + "theoretical_loss": 3.4609219875550563, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622626262626263, + "loss": 2.6422, + "theoretical_loss": 3.460911264984425, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046224242424242425, + "loss": 2.6852, + "theoretical_loss": 3.460900542913377, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622222222222222, + "loss": 2.3605, + "theoretical_loss": 3.460889821341872, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622020202020202, + "loss": 2.9078, + "theoretical_loss": 3.4608791002698682, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621818181818182, + "loss": 2.5778, + "theoretical_loss": 3.460868379697324, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621616161616162, + "loss": 2.6897, + "theoretical_loss": 3.460857659624198, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214141414141415, + "loss": 2.6651, + "theoretical_loss": 3.4608469400504482, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 1018922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.611227035522461, + "objective/train/theoretical_loss": 3.4608362209760335, + "objective/train/tokens_used": 159722976, + "theoretical_loss": 3.4608362209760335, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621212121212121, + "loss": 2.6863, + "theoretical_loss": 3.4608362209760335, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621010101010101, + "loss": 2.9061, + "theoretical_loss": 3.4608255024009136, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620808080808081, + "loss": 2.9291, + "theoretical_loss": 3.4608147843250454, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620606060606061, + "loss": 2.7639, + "theoretical_loss": 3.4608040667483886, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046204040404040404, + "loss": 2.7102, + "theoretical_loss": 3.460793349670901, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462020202020202, + "loss": 2.8882, + "theoretical_loss": 3.460782633092542, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462, + "loss": 2.7257, + "theoretical_loss": 3.4607719170132696, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461979797979798, + "loss": 2.6129, + "theoretical_loss": 3.4607612014330424, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461959595959596, + "loss": 2.8061, + "theoretical_loss": 3.4607504863518193, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046193939393939393, + "loss": 2.8155, + "theoretical_loss": 3.460739771769559, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619191919191919, + "loss": 2.4622, + "theoretical_loss": 3.46072905768622, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046189898989898996, + "loss": 2.6425, + "theoretical_loss": 3.4607183441017604, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618787878787879, + "loss": 2.6035, + "theoretical_loss": 3.460707631016139, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046185858585858587, + "loss": 2.7073, + "theoretical_loss": 3.4606969184293153, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618383838383838, + "loss": 2.7954, + "theoretical_loss": 3.460686206341247, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618181818181818, + "loss": 2.7, + "theoretical_loss": 3.4606754947518934, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179797979797985, + "loss": 2.5735, + "theoretical_loss": 3.4606647836612128, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617777777777778, + "loss": 2.6491, + "theoretical_loss": 3.460654073069163, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046175757575757576, + "loss": 2.809, + "theoretical_loss": 3.4606433629757043, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617373737373737, + "loss": 2.7535, + "theoretical_loss": 3.4606326533807943, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617171717171717, + "loss": 2.7769, + "theoretical_loss": 3.4606219442843917, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046169696969696974, + "loss": 2.7841, + "theoretical_loss": 3.460611235686456, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616767676767677, + "loss": 2.8651, + "theoretical_loss": 3.4606005275869447, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165656565656565, + "loss": 2.9124, + "theoretical_loss": 3.460589819985817, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616363636363636, + "loss": 2.7824, + "theoretical_loss": 3.460579112883032, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1019593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63952898979187, + "objective/train/theoretical_loss": 3.460568406278548, + "objective/train/tokens_used": 161361376, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616161616161616, + "loss": 2.7077, + "theoretical_loss": 3.460568406278548, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046159595959595963, + "loss": 2.8167, + "theoretical_loss": 3.4605577001723233, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615757575757576, + "loss": 2.7481, + "theoretical_loss": 3.460546994564317, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046155555555555555, + "loss": 2.7599, + "theoretical_loss": 3.460536289454488, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615353535353535, + "loss": 2.6566, + "theoretical_loss": 3.460525584842795, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615151515151515, + "loss": 2.4966, + "theoretical_loss": 3.4605148807291966, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614949494949495, + "loss": 2.8364, + "theoretical_loss": 3.460504177113651, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614747474747475, + "loss": 2.6459, + "theoretical_loss": 3.4604934739961175, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145454545454544, + "loss": 2.8458, + "theoretical_loss": 3.460482771376555, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046143434343434345, + "loss": 3.0219, + "theoretical_loss": 3.460472069254922, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614141414141414, + "loss": 2.8236, + "theoretical_loss": 3.4604613676311766, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613939393939394, + "loss": 2.5348, + "theoretical_loss": 3.4604506665052783, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613737373737374, + "loss": 2.7465, + "theoretical_loss": 3.4604399658771863, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046135353535353533, + "loss": 2.4844, + "theoretical_loss": 3.460429265746858, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133333333333334, + "loss": 2.7915, + "theoretical_loss": 3.460418566114253, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613131313131313, + "loss": 2.696, + "theoretical_loss": 3.4604078669793297, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612929292929293, + "loss": 2.5781, + "theoretical_loss": 3.4603971683420474, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046127272727272727, + "loss": 2.672, + "theoretical_loss": 3.4603864702023643, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612525252525253, + "loss": 2.8061, + "theoretical_loss": 3.4603757725602398, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046123232323232324, + "loss": 2.797, + "theoretical_loss": 3.4603650754156323, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612121212121212, + "loss": 2.7203, + "theoretical_loss": 3.4603543787685, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611919191919192, + "loss": 2.6554, + "theoretical_loss": 3.460343682618803, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117171717171716, + "loss": 2.7389, + "theoretical_loss": 3.460332986966499, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046115151515151517, + "loss": 2.5564, + "theoretical_loss": 3.4603222918115475, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046113131313131313, + "loss": 2.7158, + "theoretical_loss": 3.4603115971539067, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1020148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5186383724212646, + "objective/train/theoretical_loss": 3.460300902993536, + "objective/train/tokens_used": 162999776, + "theoretical_loss": 3.460300902993536, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111111111111114, + "loss": 2.8404, + "theoretical_loss": 3.460300902993536, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610909090909091, + "loss": 2.7754, + "theoretical_loss": 3.4602902093303936, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610707070707071, + "loss": 3.0004, + "theoretical_loss": 3.460279516164439, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046105050505050506, + "loss": 2.6974, + "theoretical_loss": 3.460268823495631, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461030303030303, + "loss": 2.86, + "theoretical_loss": 3.4602581313239273, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046101010101010103, + "loss": 2.7812, + "theoretical_loss": 3.460247439649288, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460989898989899, + "loss": 2.7085, + "theoretical_loss": 3.4602367484716714, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460969696969697, + "loss": 2.9051, + "theoretical_loss": 3.4602260577910364, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046094949494949496, + "loss": 2.6695, + "theoretical_loss": 3.4602153676073417, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609292929292929, + "loss": 2.6876, + "theoretical_loss": 3.4602046779205464, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609090909090909, + "loss": 2.8946, + "theoretical_loss": 3.4601939887306097, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046088888888888894, + "loss": 2.6602, + "theoretical_loss": 3.4601833000374898, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608686868686869, + "loss": 2.8106, + "theoretical_loss": 3.4601726118411458, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084848484848485, + "loss": 2.7228, + "theoretical_loss": 3.460161924141537, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608282828282828, + "loss": 2.632, + "theoretical_loss": 3.4601512369386214, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608080808080808, + "loss": 2.7028, + "theoretical_loss": 3.4601405502323583, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046078787878787883, + "loss": 2.5609, + "theoretical_loss": 3.460129864022707, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607676767676768, + "loss": 2.5489, + "theoretical_loss": 3.460119178309626, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046074747474747474, + "loss": 2.9247, + "theoretical_loss": 3.4601084930930743, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607272727272727, + "loss": 2.7837, + "theoretical_loss": 3.4600978083730105, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070707070707077, + "loss": 2.8433, + "theoretical_loss": 3.460087124149394, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606868686868687, + "loss": 2.6663, + "theoretical_loss": 3.4600764404221835, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606666666666667, + "loss": 2.8822, + "theoretical_loss": 3.460065757191338, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046064646464646464, + "loss": 2.5902, + "theoretical_loss": 3.460055074456816, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606262626262626, + "loss": 2.7039, + "theoretical_loss": 3.4600443922185766, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1021470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7786316871643066, + "objective/train/theoretical_loss": 3.4600337104765795, + "objective/train/tokens_used": 164638176, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060606060606066, + "loss": 2.8081, + "theoretical_loss": 3.4600337104765795, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605858585858586, + "loss": 2.7807, + "theoretical_loss": 3.4600230292307828, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046056565656565657, + "loss": 2.7275, + "theoretical_loss": 3.4600123484811456, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054545454545453, + "loss": 2.8439, + "theoretical_loss": 3.460001668227627, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046052525252525254, + "loss": 2.4894, + "theoretical_loss": 3.459990988470186, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050505050505055, + "loss": 2.6306, + "theoretical_loss": 3.459980309208781, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604848484848485, + "loss": 2.6925, + "theoretical_loss": 3.459969630443372, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046464646464646, + "loss": 2.922, + "theoretical_loss": 3.459958952173917, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604444444444444, + "loss": 2.8044, + "theoretical_loss": 3.459948274400375, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042424242424243, + "loss": 2.857, + "theoretical_loss": 3.459937597122706, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046040404040404044, + "loss": 2.5852, + "theoretical_loss": 3.459926920340868, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603838383838384, + "loss": 2.6252, + "theoretical_loss": 3.4599162440548197, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046036363636363636, + "loss": 2.6304, + "theoretical_loss": 3.4599055682645217, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046034343434343437, + "loss": 2.8092, + "theoretical_loss": 3.459894892969931, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603232323232323, + "loss": 2.7712, + "theoretical_loss": 3.459884218171008, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030303030303034, + "loss": 2.6412, + "theoretical_loss": 3.4598735438677117, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602828282828283, + "loss": 2.7084, + "theoretical_loss": 3.4598628700600003, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046026262626262625, + "loss": 2.6916, + "theoretical_loss": 3.4598521967478337, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046024242424242426, + "loss": 2.7032, + "theoretical_loss": 3.4598415239311695, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602222222222222, + "loss": 2.6248, + "theoretical_loss": 3.4598308516099685, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046020202020202023, + "loss": 2.5597, + "theoretical_loss": 3.4598201797841885, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601818181818182, + "loss": 2.8847, + "theoretical_loss": 3.459809508453789, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601616161616162, + "loss": 2.7305, + "theoretical_loss": 3.459798837618729, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046014141414141415, + "loss": 2.6421, + "theoretical_loss": 3.4597881672789677, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601212121212121, + "loss": 2.7503, + "theoretical_loss": 3.4597774974344637, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1022105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2993433475494385, + "objective/train/theoretical_loss": 3.4597668280851765, + "objective/train/tokens_used": 166276576, + "theoretical_loss": 3.4597668280851765, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601010101010101, + "loss": 2.7519, + "theoretical_loss": 3.4597668280851765, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600808080808081, + "loss": 2.7846, + "theoretical_loss": 3.4597561592310644, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600606060606061, + "loss": 2.7839, + "theoretical_loss": 3.4597454908720877, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046004040404040405, + "loss": 3.1233, + "theoretical_loss": 3.459734823008205, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460020202020202, + "loss": 2.3595, + "theoretical_loss": 3.4597241556393747, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 2.7129, + "theoretical_loss": 3.4597134887655567, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459979797979798, + "loss": 2.8135, + "theoretical_loss": 3.459702822386709, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459959595959596, + "loss": 2.7543, + "theoretical_loss": 3.4596921565027925, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993939393939394, + "loss": 2.7411, + "theoretical_loss": 3.4596814911137646, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599191919191919, + "loss": 2.6561, + "theoretical_loss": 3.459670826219585, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598989898989899, + "loss": 2.6021, + "theoretical_loss": 3.459660161820213, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598787878787879, + "loss": 2.8831, + "theoretical_loss": 3.459649497915608, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598585858585859, + "loss": 2.3374, + "theoretical_loss": 3.459638834505728, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045983838383838383, + "loss": 2.7788, + "theoretical_loss": 3.4596281715905333, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598181818181818, + "loss": 2.6551, + "theoretical_loss": 3.4596175091699823, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597979797979798, + "loss": 2.6238, + "theoretical_loss": 3.459606847244034, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597777777777778, + "loss": 2.668, + "theoretical_loss": 3.4595961858126483, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045975757575757577, + "loss": 2.6607, + "theoretical_loss": 3.459585524875784, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597373737373737, + "loss": 2.7879, + "theoretical_loss": 3.4595748644334, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597171717171717, + "loss": 2.9108, + "theoretical_loss": 3.4595642044854555, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969696969696975, + "loss": 2.7249, + "theoretical_loss": 3.45955354503191, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596767676767677, + "loss": 2.4827, + "theoretical_loss": 3.4595428860727218, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045965656565656566, + "loss": 2.781, + "theoretical_loss": 3.4595322276078515, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596363636363636, + "loss": 2.8503, + "theoretical_loss": 3.459521569637257, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596161616161616, + "loss": 2.873, + "theoretical_loss": 3.459510912160898, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1022607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8745276927948, + "objective/train/theoretical_loss": 3.4595002551787335, + "objective/train/tokens_used": 167914976, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959595959595964, + "loss": 2.6407, + "theoretical_loss": 3.4595002551787335, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595757575757576, + "loss": 2.7935, + "theoretical_loss": 3.4594895986907233, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955555555555555, + "loss": 2.5537, + "theoretical_loss": 3.4594789426968253, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595353535353535, + "loss": 2.846, + "theoretical_loss": 3.4594682871969997, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595151515151515, + "loss": 2.9611, + "theoretical_loss": 3.4594576321912056, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045949494949494953, + "loss": 2.7946, + "theoretical_loss": 3.459446977679402, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594747474747475, + "loss": 2.847, + "theoretical_loss": 3.4594363236615484, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045945454545454545, + "loss": 2.6249, + "theoretical_loss": 3.4594256701376036, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594343434343434, + "loss": 2.8331, + "theoretical_loss": 3.459415017107527, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045941414141414147, + "loss": 2.6638, + "theoretical_loss": 3.4594043645712773, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593939393939394, + "loss": 2.6927, + "theoretical_loss": 3.459393712528815, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593737373737374, + "loss": 2.5214, + "theoretical_loss": 3.459383060980098, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045935353535353534, + "loss": 2.8331, + "theoretical_loss": 3.459372409925087, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045933333333333335, + "loss": 2.8768, + "theoretical_loss": 3.4593617593637394, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931313131313136, + "loss": 2.6665, + "theoretical_loss": 3.4593511092960156, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592929292929293, + "loss": 2.9336, + "theoretical_loss": 3.459340459721875, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592727272727273, + "loss": 2.7247, + "theoretical_loss": 3.459329810641276, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045925252525252523, + "loss": 2.8262, + "theoretical_loss": 3.4593191620541783, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923232323232324, + "loss": 2.8846, + "theoretical_loss": 3.4593085139605417, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921212121212125, + "loss": 2.8352, + "theoretical_loss": 3.459297866360325, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591919191919192, + "loss": 2.9307, + "theoretical_loss": 3.459287219253487, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917171717171717, + "loss": 2.751, + "theoretical_loss": 3.459276572639988, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591515151515152, + "loss": 2.6462, + "theoretical_loss": 3.459265926519786, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913131313131313, + "loss": 2.7831, + "theoretical_loss": 3.459255280892841, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045911111111111115, + "loss": 2.728, + "theoretical_loss": 3.459244635759113, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1024104, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4631869792938232, + "objective/train/theoretical_loss": 3.459233991118561, + "objective/train/tokens_used": 169553376, + "theoretical_loss": 3.459233991118561, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590909090909091, + "loss": 2.7345, + "theoretical_loss": 3.459233991118561, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045907070707070706, + "loss": 2.7417, + "theoretical_loss": 3.4592233469711426, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905050505050507, + "loss": 2.577, + "theoretical_loss": 3.459212703316819, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045903030303030303, + "loss": 2.7466, + "theoretical_loss": 3.4592020601555493, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901010101010104, + "loss": 2.6525, + "theoretical_loss": 3.4591914174872915, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458989898989899, + "loss": 2.6407, + "theoretical_loss": 3.4591807753120065, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458969696969697, + "loss": 2.8483, + "theoretical_loss": 3.4591701336296534, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894949494949496, + "loss": 2.827, + "theoretical_loss": 3.4591594924401905, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589292929292929, + "loss": 2.7865, + "theoretical_loss": 3.459148851743578, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890909090909093, + "loss": 2.7844, + "theoretical_loss": 3.459138211539775, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588888888888889, + "loss": 2.7864, + "theoretical_loss": 3.4591275718287404, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588686868686869, + "loss": 2.7121, + "theoretical_loss": 3.459116932610435, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045884848484848486, + "loss": 2.7344, + "theoretical_loss": 3.4591062938848163, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588282828282828, + "loss": 2.6589, + "theoretical_loss": 3.4590956556518444, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588080808080808, + "loss": 2.9284, + "theoretical_loss": 3.4590850179114794, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045878787878787883, + "loss": 2.4579, + "theoretical_loss": 3.45907438066368, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587676767676768, + "loss": 2.7925, + "theoretical_loss": 3.4590637439084055, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874747474747475, + "loss": 2.639, + "theoretical_loss": 3.459053107645615, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587272727272727, + "loss": 2.6596, + "theoretical_loss": 3.459042471875269, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587070707070707, + "loss": 2.4519, + "theoretical_loss": 3.4590318365973256, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868686868686873, + "loss": 2.5492, + "theoretical_loss": 3.459021201811745, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586666666666667, + "loss": 2.7058, + "theoretical_loss": 3.4590105675184866, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864646464646464, + "loss": 2.8016, + "theoretical_loss": 3.458999933717509, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586262626262626, + "loss": 2.5911, + "theoretical_loss": 3.4589893004087733, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860606060606066, + "loss": 2.8291, + "theoretical_loss": 3.4589786675922367, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1024786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.423832893371582, + "objective/train/theoretical_loss": 3.4589680352678602, + "objective/train/tokens_used": 171191776, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585858585858586, + "loss": 2.3708, + "theoretical_loss": 3.4589680352678602, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585656565656566, + "loss": 2.5868, + "theoretical_loss": 3.4589574034356025, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045854545454545453, + "loss": 2.8749, + "theoretical_loss": 3.4589467720954232, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585252525252525, + "loss": 2.6718, + "theoretical_loss": 3.4589361412472828, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045850505050505056, + "loss": 2.8781, + "theoretical_loss": 3.4589255108911385, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584848484848485, + "loss": 2.7955, + "theoretical_loss": 3.458914881026952, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045846464646464647, + "loss": 2.728, + "theoretical_loss": 3.458904251654681, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584444444444444, + "loss": 2.55, + "theoretical_loss": 3.458893622774286, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584242424242424, + "loss": 2.7166, + "theoretical_loss": 3.4588829943857258, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840404040404045, + "loss": 2.9026, + "theoretical_loss": 3.4588723664889605, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583838383838384, + "loss": 2.7166, + "theoretical_loss": 3.4588617390839493, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836363636363636, + "loss": 2.8551, + "theoretical_loss": 3.4588511121706516, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583434343434343, + "loss": 2.7041, + "theoretical_loss": 3.458840485749027, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832323232323233, + "loss": 2.6589, + "theoretical_loss": 3.4588298598190352, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045830303030303034, + "loss": 2.6504, + "theoretical_loss": 3.458819234380635, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582828282828283, + "loss": 2.7475, + "theoretical_loss": 3.4588086094337864, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826262626262626, + "loss": 2.7237, + "theoretical_loss": 3.4587979849784487, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582424242424242, + "loss": 2.6961, + "theoretical_loss": 3.458787361014582, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582222222222222, + "loss": 2.9232, + "theoretical_loss": 3.4587767375421445, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820202020202023, + "loss": 2.6333, + "theoretical_loss": 3.458766114561097, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581818181818182, + "loss": 2.753, + "theoretical_loss": 3.458755492071398, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045816161616161615, + "loss": 2.7348, + "theoretical_loss": 3.458744870073008, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045814141414141416, + "loss": 2.5678, + "theoretical_loss": 3.458734248565886, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581212121212121, + "loss": 2.6481, + "theoretical_loss": 3.4587236275499915, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581010101010101, + "loss": 2.5859, + "theoretical_loss": 3.458713007025284, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1026014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9905080795288086, + "objective/train/theoretical_loss": 3.4587023869917237, + "objective/train/tokens_used": 172830176, + "theoretical_loss": 3.4587023869917237, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580808080808081, + "loss": 2.6292, + "theoretical_loss": 3.4587023869917237, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806060606060604, + "loss": 2.73, + "theoretical_loss": 3.458691767449269, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045804040404040405, + "loss": 2.5536, + "theoretical_loss": 3.45868114839788, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458020202020202, + "loss": 2.5646, + "theoretical_loss": 3.4586705298375167, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458, + "loss": 2.7652, + "theoretical_loss": 3.4586599117681383, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457979797979798, + "loss": 2.9956, + "theoretical_loss": 3.458649294189704, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457959595959596, + "loss": 2.7514, + "theoretical_loss": 3.458638677102174, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045793939393939394, + "loss": 2.796, + "theoretical_loss": 3.4586280605055073, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579191919191919, + "loss": 2.7835, + "theoretical_loss": 3.458617444399664, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578989898989899, + "loss": 2.5738, + "theoretical_loss": 3.4586068287846032, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787878787878787, + "loss": 2.8933, + "theoretical_loss": 3.458596213660285, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578585858585859, + "loss": 2.5159, + "theoretical_loss": 3.4585855990266685, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783838383838384, + "loss": 2.8074, + "theoretical_loss": 3.4585749848837133, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578181818181818, + "loss": 2.767, + "theoretical_loss": 3.45856437123138, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577979797979798, + "loss": 2.6562, + "theoretical_loss": 3.4585537580696264, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577777777777778, + "loss": 3.0138, + "theoretical_loss": 3.458543145398414, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045775757575757577, + "loss": 2.7923, + "theoretical_loss": 3.458532533217701, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045773737373737373, + "loss": 2.8577, + "theoretical_loss": 3.4585219215274474, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045771717171717174, + "loss": 2.7327, + "theoretical_loss": 3.458511310327613, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576969696969697, + "loss": 2.6893, + "theoretical_loss": 3.458500699618158, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576767676767677, + "loss": 2.7187, + "theoretical_loss": 3.458490089399041, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045765656565656567, + "loss": 2.8272, + "theoretical_loss": 3.4584794796702223, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576363636363636, + "loss": 2.5984, + "theoretical_loss": 3.458468870431661, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761616161616163, + "loss": 2.7403, + "theoretical_loss": 3.4584582616833175, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045759595959595964, + "loss": 2.5738, + "theoretical_loss": 3.458447653425151, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1026828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.785205841064453, + "objective/train/theoretical_loss": 3.4584370456571207, + "objective/train/tokens_used": 174468576, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575757575757576, + "loss": 2.8435, + "theoretical_loss": 3.4584370456571207, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045755555555555556, + "loss": 2.6289, + "theoretical_loss": 3.4584264383791874, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575353535353535, + "loss": 2.8496, + "theoretical_loss": 3.4584158315913096, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575151515151515, + "loss": 2.8828, + "theoretical_loss": 3.458405225293448, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749494949494954, + "loss": 2.6929, + "theoretical_loss": 3.458394619485561, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574747474747475, + "loss": 2.8342, + "theoretical_loss": 3.4583840141676094, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745454545454545, + "loss": 2.6944, + "theoretical_loss": 3.458373409339553, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574343434343434, + "loss": 2.9315, + "theoretical_loss": 3.4583628050013506, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574141414141415, + "loss": 2.7609, + "theoretical_loss": 3.4583522011529624, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739393939393943, + "loss": 2.7821, + "theoretical_loss": 3.458341597794348, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573737373737374, + "loss": 2.7627, + "theoretical_loss": 3.4583309949254675, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045735353535353534, + "loss": 2.7485, + "theoretical_loss": 3.4583203925462795, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573333333333333, + "loss": 2.6619, + "theoretical_loss": 3.4583097906567453, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731313131313137, + "loss": 2.9983, + "theoretical_loss": 3.458299189256823, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572929292929293, + "loss": 2.9038, + "theoretical_loss": 3.458288588346474, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572727272727273, + "loss": 2.7941, + "theoretical_loss": 3.4582779879256567, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045725252525252524, + "loss": 2.7035, + "theoretical_loss": 3.458267387994331, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723232323232325, + "loss": 2.7648, + "theoretical_loss": 3.458256788552457, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045721212121212126, + "loss": 2.6228, + "theoretical_loss": 3.4582461895999947, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571919191919192, + "loss": 2.9447, + "theoretical_loss": 3.458235591136903, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717171717171717, + "loss": 2.9458, + "theoretical_loss": 3.458224993163143, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045715151515151513, + "loss": 2.7362, + "theoretical_loss": 3.458214395678673, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713131313131314, + "loss": 2.8063, + "theoretical_loss": 3.4582037986834537, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045711111111111115, + "loss": 2.9608, + "theoretical_loss": 3.458193202177444, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004570909090909091, + "loss": 2.5834, + "theoretical_loss": 3.4581826061606047, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 1028029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.948615550994873, + "objective/train/theoretical_loss": 3.458172010632895, + "objective/train/tokens_used": 176106976, + "theoretical_loss": 3.458172010632895, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045707070707070706, + "loss": 2.6524, + "theoretical_loss": 3.458172010632895, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570505050505051, + "loss": 2.9222, + "theoretical_loss": 3.458161415594275, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045703030303030303, + "loss": 2.7792, + "theoretical_loss": 3.458150821044704, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701010101010104, + "loss": 2.8863, + "theoretical_loss": 3.458140226984142, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456989898989899, + "loss": 2.815, + "theoretical_loss": 3.458129633412549, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696969696969696, + "loss": 3.0687, + "theoretical_loss": 3.4581190403298847, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045694949494949497, + "loss": 2.9307, + "theoretical_loss": 3.458108447736109, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569292929292929, + "loss": 2.6997, + "theoretical_loss": 3.4580978556311814, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690909090909094, + "loss": 2.6606, + "theoretical_loss": 3.458087264015062, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568888888888889, + "loss": 3.05, + "theoretical_loss": 3.45807667288771, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568686868686869, + "loss": 2.8669, + "theoretical_loss": 3.4580660822490863, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684848484848486, + "loss": 2.6636, + "theoretical_loss": 3.45805549209915, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568282828282828, + "loss": 2.7062, + "theoretical_loss": 3.4580449024378614, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680808080808083, + "loss": 2.6352, + "theoretical_loss": 3.4580343132651796, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567878787878788, + "loss": 2.5521, + "theoretical_loss": 3.458023724581065, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567676767676768, + "loss": 2.7404, + "theoretical_loss": 3.4580131363854782, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674747474747475, + "loss": 2.7452, + "theoretical_loss": 3.458002548678377, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567272727272727, + "loss": 2.7767, + "theoretical_loss": 3.457991961459723, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567070707070707, + "loss": 2.8785, + "theoretical_loss": 3.4579813747294756, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045668686868686873, + "loss": 2.6842, + "theoretical_loss": 3.4579707884875948, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566666666666667, + "loss": 2.6801, + "theoretical_loss": 3.4579602027340393, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045664646464646465, + "loss": 2.6236, + "theoretical_loss": 3.457949617468771, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566262626262626, + "loss": 2.6719, + "theoretical_loss": 3.457939032691748, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566060606060606, + "loss": 2.6363, + "theoretical_loss": 3.4579284484029316, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565858585858586, + "loss": 2.5759, + "theoretical_loss": 3.45791786460228, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1028792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2022347450256348, + "objective/train/theoretical_loss": 3.4579072812897547, + "objective/train/tokens_used": 177745376, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565656565656566, + "loss": 2.7865, + "theoretical_loss": 3.4579072812897547, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654545454545454, + "loss": 2.7154, + "theoretical_loss": 3.457896698465315, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565252525252525, + "loss": 2.7121, + "theoretical_loss": 3.457886116128921, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565050505050505, + "loss": 2.7662, + "theoretical_loss": 3.4578755342805323, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564848484848485, + "loss": 2.8467, + "theoretical_loss": 3.457864952920109, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564646464646465, + "loss": 2.9555, + "theoretical_loss": 3.4578543720476103, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045644444444444443, + "loss": 2.9755, + "theoretical_loss": 3.4578437916629974, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564242424242424, + "loss": 2.6065, + "theoretical_loss": 3.4578332117662294, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640404040404045, + "loss": 2.8788, + "theoretical_loss": 3.4578226323572663, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563838383838384, + "loss": 2.6361, + "theoretical_loss": 3.4578120534360686, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045636363636363637, + "loss": 2.796, + "theoretical_loss": 3.4578014750025954, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563434343434343, + "loss": 2.6042, + "theoretical_loss": 3.457790897056807, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563232323232323, + "loss": 2.8136, + "theoretical_loss": 3.4577803195986636, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630303030303035, + "loss": 2.7466, + "theoretical_loss": 3.457769742628125, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562828282828283, + "loss": 2.8045, + "theoretical_loss": 3.4577591661451517, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045626262626262626, + "loss": 2.8097, + "theoretical_loss": 3.457748590149703, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562424242424242, + "loss": 2.9414, + "theoretical_loss": 3.457738014641738, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045622222222222223, + "loss": 2.7751, + "theoretical_loss": 3.4577274396212183, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045620202020202024, + "loss": 2.7572, + "theoretical_loss": 3.457716865088103, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561818181818182, + "loss": 2.7608, + "theoretical_loss": 3.457706291042353, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616161616161615, + "loss": 2.7217, + "theoretical_loss": 3.4576957174839267, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561414141414141, + "loss": 2.695, + "theoretical_loss": 3.4576851444127854, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561212121212121, + "loss": 2.5052, + "theoretical_loss": 3.457674571828889, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045610101010101013, + "loss": 2.7035, + "theoretical_loss": 3.457663999732197, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560808080808081, + "loss": 2.8835, + "theoretical_loss": 3.4576534281226694, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1029550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0437440872192383, + "objective/train/theoretical_loss": 3.457642857000267, + "objective/train/tokens_used": 179383776, + "theoretical_loss": 3.457642857000267, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045606060606060605, + "loss": 2.8068, + "theoretical_loss": 3.457642857000267, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604040404040406, + "loss": 2.9178, + "theoretical_loss": 3.457632286364949, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045602020202020207, + "loss": 3.0388, + "theoretical_loss": 3.4576217162166754, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456, + "loss": 2.7146, + "theoretical_loss": 3.457611146555407, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455979797979798, + "loss": 2.8108, + "theoretical_loss": 3.457600577381103, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045595959595959594, + "loss": 2.849, + "theoretical_loss": 3.4575900086937237, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593939393939395, + "loss": 2.6997, + "theoretical_loss": 3.4575794404932294, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591919191919196, + "loss": 2.8487, + "theoretical_loss": 3.4575688727795804, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558989898989899, + "loss": 2.7506, + "theoretical_loss": 3.4575583055527357, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558787878787879, + "loss": 2.7141, + "theoretical_loss": 3.4575477388126563, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558585858585859, + "loss": 2.5869, + "theoretical_loss": 3.457537172559302, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583838383838384, + "loss": 2.8941, + "theoretical_loss": 3.4575266067926327, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581818181818185, + "loss": 3.0471, + "theoretical_loss": 3.4575160415126085, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557979797979798, + "loss": 2.7158, + "theoretical_loss": 3.4575054767191897, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045577777777777777, + "loss": 2.9494, + "theoretical_loss": 3.4574949124123364, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557575757575758, + "loss": 2.7361, + "theoretical_loss": 3.457484348592008, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045573737373737374, + "loss": 2.9178, + "theoretical_loss": 3.4574737852581654, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045571717171717175, + "loss": 2.7656, + "theoretical_loss": 3.4574632224107686, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556969696969697, + "loss": 2.8451, + "theoretical_loss": 3.4574526600497775, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556767676767677, + "loss": 2.8523, + "theoretical_loss": 3.457442098175152, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045565656565656567, + "loss": 2.9012, + "theoretical_loss": 3.457431536786853, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045563636363636363, + "loss": 2.883, + "theoretical_loss": 3.4574209758848395, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045561616161616164, + "loss": 2.6552, + "theoretical_loss": 3.457410415469072, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555959595959596, + "loss": 2.8173, + "theoretical_loss": 3.4573998555395113, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555757575757576, + "loss": 2.7659, + "theoretical_loss": 3.4573892960961166, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1030259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5049033164978027, + "objective/train/theoretical_loss": 3.457378737138849, + "objective/train/tokens_used": 181022176, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555555555555556, + "loss": 2.7223, + "theoretical_loss": 3.457378737138849, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555353535353535, + "loss": 2.6485, + "theoretical_loss": 3.4573681786676675, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551515151515153, + "loss": 2.7364, + "theoretical_loss": 3.457357620682533, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045549494949494954, + "loss": 2.6497, + "theoretical_loss": 3.4573470631834056, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554747474747475, + "loss": 2.6301, + "theoretical_loss": 3.457336506170245, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045545454545454546, + "loss": 2.82, + "theoretical_loss": 3.457325949643012, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554343434343434, + "loss": 2.6727, + "theoretical_loss": 3.4573153936016663, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554141414141414, + "loss": 2.5298, + "theoretical_loss": 3.4573048380461686, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045539393939393944, + "loss": 2.6992, + "theoretical_loss": 3.4572942829764783, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553737373737374, + "loss": 2.6018, + "theoretical_loss": 3.4572837283925564, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535353535353535, + "loss": 2.8452, + "theoretical_loss": 3.457273174294362, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553333333333333, + "loss": 2.8835, + "theoretical_loss": 3.457262620681856, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045531313131313137, + "loss": 2.7719, + "theoretical_loss": 3.4572520675549985, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529292929292933, + "loss": 2.6711, + "theoretical_loss": 3.45724151491375, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552727272727273, + "loss": 2.7379, + "theoretical_loss": 3.4572309627580706, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045525252525252524, + "loss": 2.7252, + "theoretical_loss": 3.4572204110879197, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552323232323232, + "loss": 2.6214, + "theoretical_loss": 3.4572098599032586, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521212121212126, + "loss": 2.8895, + "theoretical_loss": 3.4571993092040465, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551919191919192, + "loss": 2.8812, + "theoretical_loss": 3.4571887589902444, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551717171717172, + "loss": 2.583, + "theoretical_loss": 3.457178209261812, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045515151515151513, + "loss": 2.6921, + "theoretical_loss": 3.457167660018711, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551313131313131, + "loss": 2.6511, + "theoretical_loss": 3.4571571112608988, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045511111111111116, + "loss": 2.9865, + "theoretical_loss": 3.4571465629883384, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550909090909091, + "loss": 2.6777, + "theoretical_loss": 3.4571360152009882, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045507070707070707, + "loss": 2.8989, + "theoretical_loss": 3.4571254678988095, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1031478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.003403425216675, + "objective/train/theoretical_loss": 3.4571149210817618, + "objective/train/tokens_used": 182660576, + "theoretical_loss": 3.4571149210817618, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505050505050503, + "loss": 2.769, + "theoretical_loss": 3.4571149210817618, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045503030303030304, + "loss": 2.8009, + "theoretical_loss": 3.4571043747498065, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501010101010105, + "loss": 2.7386, + "theoretical_loss": 3.4570938289029023, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454989898989899, + "loss": 2.8968, + "theoretical_loss": 3.457083283541011, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045496969696969696, + "loss": 2.5428, + "theoretical_loss": 3.4570727386640914, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549494949494949, + "loss": 2.5718, + "theoretical_loss": 3.4570621942721047, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045492929292929293, + "loss": 2.6708, + "theoretical_loss": 3.4570516503650115, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045490909090909094, + "loss": 2.5928, + "theoretical_loss": 3.4570411069427713, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548888888888889, + "loss": 2.7112, + "theoretical_loss": 3.4570305640053443, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486868686868686, + "loss": 2.7203, + "theoretical_loss": 3.457020021552691, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045484848484848487, + "loss": 2.6355, + "theoretical_loss": 3.4570094795847726, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548282828282828, + "loss": 2.6633, + "theoretical_loss": 3.456998938101548, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480808080808083, + "loss": 2.5711, + "theoretical_loss": 3.456988397102979, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547878787878788, + "loss": 2.5718, + "theoretical_loss": 3.4569778565890243, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045476767676767675, + "loss": 2.7613, + "theoretical_loss": 3.456967316559645, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045474747474747476, + "loss": 2.83, + "theoretical_loss": 3.4569567770148018, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547272727272727, + "loss": 3.0262, + "theoretical_loss": 3.4569462379544547, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045470707070707073, + "loss": 2.8845, + "theoretical_loss": 3.4569356993785636, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546868686868687, + "loss": 2.7995, + "theoretical_loss": 3.456925161287089, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546666666666667, + "loss": 2.5765, + "theoretical_loss": 3.456914623679992, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464646464646465, + "loss": 3.0724, + "theoretical_loss": 3.4569040865572322, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546262626262626, + "loss": 2.9716, + "theoretical_loss": 3.4568935499187696, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546060606060606, + "loss": 2.6274, + "theoretical_loss": 3.456883013764566, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545858585858586, + "loss": 2.9424, + "theoretical_loss": 3.4568724780945796, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545656565656566, + "loss": 2.9468, + "theoretical_loss": 3.456861942908773, + "tokens_seen": 1825374208 + }, + { + "debugging/Self-BLEU-5": 0.383412367211192, + "debugging/distinct-1-grams": 0.7526443544193546, + "debugging/distinct-2-grams": 0.9328595808568966, + "debugging/entropy-1-grams": 5.657827037788948, + "debugging/entropy-2-grams": 6.397750202501998, + "debugging/length": 484.77777777777777, + "debugging/num_segments": 9, + "epoch": 0.1, + "objective/train/docs_used": 1032081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5285487174987793, + "objective/train/theoretical_loss": 3.456851408207105, + "objective/train/tokens_used": 184298976, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045454545454545455, + "loss": 2.8721, + "theoretical_loss": 3.456851408207105, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545252525252525, + "loss": 2.481, + "theoretical_loss": 3.456840873989537, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545050505050505, + "loss": 2.4851, + "theoretical_loss": 3.4568303402560288, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544848484848485, + "loss": 2.7403, + "theoretical_loss": 3.4568198070065406, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544646464646465, + "loss": 2.8656, + "theoretical_loss": 3.4568092742410332, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045444444444444444, + "loss": 2.5863, + "theoretical_loss": 3.456798741959467, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544242424242424, + "loss": 2.6413, + "theoretical_loss": 3.456788210161802, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544040404040404, + "loss": 2.6993, + "theoretical_loss": 3.456777678847999, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543838383838384, + "loss": 2.7035, + "theoretical_loss": 3.4567671480180184, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543636363636364, + "loss": 2.6861, + "theoretical_loss": 3.4567566176718203, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045434343434343433, + "loss": 2.6218, + "theoretical_loss": 3.456746087809365, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543232323232323, + "loss": 2.7763, + "theoretical_loss": 3.456735558430614, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045430303030303035, + "loss": 2.487, + "theoretical_loss": 3.4567250295355265, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828282828283, + "loss": 2.8302, + "theoretical_loss": 3.456714501124063, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426262626262627, + "loss": 2.4841, + "theoretical_loss": 3.4567039731961846, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542424242424242, + "loss": 2.8058, + "theoretical_loss": 3.4566934457518514, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045422222222222223, + "loss": 2.8221, + "theoretical_loss": 3.456682918791024, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420202020202025, + "loss": 2.8255, + "theoretical_loss": 3.4566723923136626, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541818181818182, + "loss": 2.7826, + "theoretical_loss": 3.4566618663197275, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045416161616161616, + "loss": 2.8136, + "theoretical_loss": 3.45665134080918, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541414141414141, + "loss": 2.6064, + "theoretical_loss": 3.4566408157819795, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541212121212122, + "loss": 2.6452, + "theoretical_loss": 3.456630291238087, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045410101010101014, + "loss": 2.7091, + "theoretical_loss": 3.4566197671774628, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540808080808081, + "loss": 2.753, + "theoretical_loss": 3.456609243600068, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045406060606060605, + "loss": 2.6462, + "theoretical_loss": 3.456598720505862, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1033277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3679161071777344, + "objective/train/theoretical_loss": 3.4565881978948063, + "objective/train/tokens_used": 185937376, + "theoretical_loss": 3.4565881978948063, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454040404040404, + "loss": 2.4956, + "theoretical_loss": 3.4565881978948063, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540202020202021, + "loss": 2.5966, + "theoretical_loss": 3.4565776757668605, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400000000000003, + "loss": 2.6165, + "theoretical_loss": 3.456567154121986, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453979797979798, + "loss": 2.8525, + "theoretical_loss": 3.4565566329601425, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045395959595959594, + "loss": 2.7701, + "theoretical_loss": 3.4565461122812904, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045393939393939396, + "loss": 2.5275, + "theoretical_loss": 3.4565355920853915, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391919191919197, + "loss": 2.8853, + "theoretical_loss": 3.456525072372405, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538989898989899, + "loss": 2.4489, + "theoretical_loss": 3.456514553142292, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538787878787879, + "loss": 2.7814, + "theoretical_loss": 3.456504034395013, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045385858585858584, + "loss": 2.739, + "theoretical_loss": 3.4564935161305277, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383838383838385, + "loss": 2.9733, + "theoretical_loss": 3.4564829983487977, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045381818181818186, + "loss": 2.6506, + "theoretical_loss": 3.4564724810497838, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537979797979798, + "loss": 2.7338, + "theoretical_loss": 3.456461964233445, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537777777777778, + "loss": 2.8285, + "theoretical_loss": 3.4564514478997435, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537575757575758, + "loss": 2.7575, + "theoretical_loss": 3.456440932048639, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045373737373737374, + "loss": 2.6726, + "theoretical_loss": 3.4564304166800914, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045371717171717175, + "loss": 2.686, + "theoretical_loss": 3.456419901794063, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536969696969697, + "loss": 2.7443, + "theoretical_loss": 3.4564093873905124, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367676767676767, + "loss": 2.6844, + "theoretical_loss": 3.4563988734694018, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536565656565657, + "loss": 2.5547, + "theoretical_loss": 3.456388360030691, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045363636363636363, + "loss": 2.8119, + "theoretical_loss": 3.4563778470743403, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361616161616164, + "loss": 2.5855, + "theoretical_loss": 3.456367334600311, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535959595959596, + "loss": 2.6677, + "theoretical_loss": 3.4563568226085635, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535757575757576, + "loss": 2.8127, + "theoretical_loss": 3.456346311099058, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045355555555555557, + "loss": 2.5376, + "theoretical_loss": 3.4563358000717552, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1034021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7584831714630127, + "objective/train/theoretical_loss": 3.456325289526616, + "objective/train/tokens_used": 187575776, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535353535353535, + "loss": 2.9362, + "theoretical_loss": 3.456325289526616, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045351515151515154, + "loss": 2.6903, + "theoretical_loss": 3.456314779463601, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534949494949495, + "loss": 2.7164, + "theoretical_loss": 3.4563042698826707, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534747474747475, + "loss": 2.8571, + "theoretical_loss": 3.456293760783786, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045345454545454546, + "loss": 2.9708, + "theoretical_loss": 3.456283252166906, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534343434343434, + "loss": 2.6743, + "theoretical_loss": 3.4562727440319936, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341414141414143, + "loss": 2.657, + "theoretical_loss": 3.4562622363790076, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339393939393944, + "loss": 2.737, + "theoretical_loss": 3.45625172920791, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533737373737374, + "loss": 2.7241, + "theoretical_loss": 3.45624122251866, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045335353535353535, + "loss": 2.7485, + "theoretical_loss": 3.45623071631122, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533333333333333, + "loss": 2.8172, + "theoretical_loss": 3.456220210585549, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533131313131313, + "loss": 2.4831, + "theoretical_loss": 3.4562097053416085, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045329292929292933, + "loss": 2.7984, + "theoretical_loss": 3.456199200579359, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532727272727273, + "loss": 2.7636, + "theoretical_loss": 3.456188696298761, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325252525252525, + "loss": 2.7041, + "theoretical_loss": 3.4561781924997756, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532323232323232, + "loss": 2.5928, + "theoretical_loss": 3.456167689182363, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532121212121212, + "loss": 2.8728, + "theoretical_loss": 3.456157186346484, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531919191919192, + "loss": 2.7414, + "theoretical_loss": 3.4561466839921, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531717171717172, + "loss": 2.8615, + "theoretical_loss": 3.4561361821191703, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315151515151514, + "loss": 2.8741, + "theoretical_loss": 3.456125680727656, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531313131313131, + "loss": 2.94, + "theoretical_loss": 3.4561151798175187, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311111111111116, + "loss": 2.6731, + "theoretical_loss": 3.4561046793887185, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530909090909091, + "loss": 2.6412, + "theoretical_loss": 3.456094179441216, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530707070707071, + "loss": 2.578, + "theoretical_loss": 3.456083679974972, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045305050505050503, + "loss": 2.7961, + "theoretical_loss": 3.456073180989947, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1035168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6923987865448, + "objective/train/theoretical_loss": 3.456062682486102, + "objective/train/tokens_used": 189214176, + "theoretical_loss": 3.456062682486102, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453030303030303, + "loss": 2.7259, + "theoretical_loss": 3.456062682486102, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045301010101010106, + "loss": 2.742, + "theoretical_loss": 3.4560521844633976, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452989898989899, + "loss": 2.8285, + "theoretical_loss": 3.456041686921795, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045296969696969697, + "loss": 2.7108, + "theoretical_loss": 3.456031189861254, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529494949494949, + "loss": 2.6907, + "theoretical_loss": 3.4560206932817357, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292929292929294, + "loss": 2.8064, + "theoretical_loss": 3.4560101971832013, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045290909090909095, + "loss": 2.3048, + "theoretical_loss": 3.455999701565611, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528888888888889, + "loss": 2.5728, + "theoretical_loss": 3.455989206428926, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045286868686868686, + "loss": 2.6463, + "theoretical_loss": 3.455978711773107, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528484848484848, + "loss": 2.734, + "theoretical_loss": 3.455968217598114, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045282828282828283, + "loss": 2.8513, + "theoretical_loss": 3.455957723903909, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045280808080808084, + "loss": 2.6367, + "theoretical_loss": 3.4559472306904517, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527878787878788, + "loss": 2.7211, + "theoretical_loss": 3.4559367379577033, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276767676767675, + "loss": 2.7202, + "theoretical_loss": 3.4559262457056246, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045274747474747477, + "loss": 2.8859, + "theoretical_loss": 3.4559157539341765, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527272727272727, + "loss": 2.6553, + "theoretical_loss": 3.455905262643319, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270707070707073, + "loss": 3.0149, + "theoretical_loss": 3.4558947718330137, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526868686868687, + "loss": 2.7415, + "theoretical_loss": 3.4558842815032214, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266666666666665, + "loss": 2.9856, + "theoretical_loss": 3.455873791653903, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045264646464646466, + "loss": 2.7616, + "theoretical_loss": 3.4558633022850183, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526262626262626, + "loss": 2.4641, + "theoretical_loss": 3.455852813396529, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526060606060606, + "loss": 2.7974, + "theoretical_loss": 3.455842324988396, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525858585858586, + "loss": 2.9701, + "theoretical_loss": 3.455831837060579, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525656565656566, + "loss": 2.8074, + "theoretical_loss": 3.45582134961304, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254545454545455, + "loss": 2.6105, + "theoretical_loss": 3.4558108626457402, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1035908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.223759412765503, + "objective/train/theoretical_loss": 3.455800376158639, + "objective/train/tokens_used": 190852576, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045252525252525256, + "loss": 2.8248, + "theoretical_loss": 3.455800376158639, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525050505050505, + "loss": 2.7729, + "theoretical_loss": 3.4557898901516984, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524848484848485, + "loss": 2.8977, + "theoretical_loss": 3.455779404624878, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524646464646465, + "loss": 2.7569, + "theoretical_loss": 3.45576891957814, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045244444444444444, + "loss": 2.9215, + "theoretical_loss": 3.4557584350114445, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045242424242424245, + "loss": 2.688, + "theoretical_loss": 3.4557479509247524, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524040404040404, + "loss": 3.0079, + "theoretical_loss": 3.455737467318025, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523838383838384, + "loss": 2.7317, + "theoretical_loss": 3.4557269841912226, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523636363636364, + "loss": 2.6184, + "theoretical_loss": 3.4557165015443063, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045234343434343434, + "loss": 2.4079, + "theoretical_loss": 3.455706019377237, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045232323232323235, + "loss": 2.5802, + "theoretical_loss": 3.4556955376899756, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523030303030303, + "loss": 2.7047, + "theoretical_loss": 3.4556850564824826, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522828282828283, + "loss": 2.8047, + "theoretical_loss": 3.4556745757547196, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045226262626262627, + "loss": 2.7982, + "theoretical_loss": 3.455664095506647, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224242424242423, + "loss": 2.8402, + "theoretical_loss": 3.4556536157382256, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222222222222224, + "loss": 2.5908, + "theoretical_loss": 3.455643136449417, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220202020202025, + "loss": 2.818, + "theoretical_loss": 3.455632657640181, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521818181818182, + "loss": 2.5081, + "theoretical_loss": 3.4556221793104793, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216161616161616, + "loss": 2.7472, + "theoretical_loss": 3.4556117014602727, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521414141414141, + "loss": 2.7309, + "theoretical_loss": 3.455601224089522, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212121212121213, + "loss": 3.1213, + "theoretical_loss": 3.4555907471981886, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045210101010101014, + "loss": 2.6728, + "theoretical_loss": 3.455580270786232, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520808080808081, + "loss": 2.9695, + "theoretical_loss": 3.455569794853615, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045206060606060606, + "loss": 2.862, + "theoretical_loss": 3.4555593194002974, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452040404040404, + "loss": 2.4657, + "theoretical_loss": 3.4555488444262403, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 1036404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5966298580169678, + "objective/train/theoretical_loss": 3.4555383699314044, + "objective/train/tokens_used": 192490976, + "theoretical_loss": 3.4555383699314044, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520202020202021, + "loss": 2.6121, + "theoretical_loss": 3.4555383699314044, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200000000000004, + "loss": 2.8645, + "theoretical_loss": 3.4555278959157514, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451979797979798, + "loss": 2.7986, + "theoretical_loss": 3.455517422379242, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045195959595959595, + "loss": 2.8923, + "theoretical_loss": 3.455506949321837, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519393939393939, + "loss": 2.7054, + "theoretical_loss": 3.455496476743497, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045191919191919197, + "loss": 2.8595, + "theoretical_loss": 3.4554860046441833, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189898989898993, + "loss": 2.8207, + "theoretical_loss": 3.455475533023857, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518787878787879, + "loss": 2.6514, + "theoretical_loss": 3.455465061882479, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045185858585858584, + "loss": 2.8002, + "theoretical_loss": 3.4554545912200103, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518383838383838, + "loss": 2.6166, + "theoretical_loss": 3.455444121036412, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181818181818186, + "loss": 2.7315, + "theoretical_loss": 3.455433651331645, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517979797979798, + "loss": 2.772, + "theoretical_loss": 3.45542318210567, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517777777777778, + "loss": 2.7918, + "theoretical_loss": 3.455412713358448, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175757575757574, + "loss": 2.7234, + "theoretical_loss": 3.45540224508994, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173737373737375, + "loss": 2.728, + "theoretical_loss": 3.455391777300108, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171717171717176, + "loss": 2.859, + "theoretical_loss": 3.455381309988912, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516969696969697, + "loss": 2.9261, + "theoretical_loss": 3.455370843156313, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167676767676767, + "loss": 2.6963, + "theoretical_loss": 3.4553603768022727, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165656565656563, + "loss": 2.8501, + "theoretical_loss": 3.4553499109267514, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163636363636364, + "loss": 2.5744, + "theoretical_loss": 3.4553394455297104, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045161616161616165, + "loss": 2.6265, + "theoretical_loss": 3.455328980611111, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515959595959596, + "loss": 2.8088, + "theoretical_loss": 3.455318516170914, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045157575757575756, + "loss": 2.8294, + "theoretical_loss": 3.4553080522090807, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515555555555556, + "loss": 2.4038, + "theoretical_loss": 3.4552975887255717, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045153535353535353, + "loss": 2.5655, + "theoretical_loss": 3.455287125720348, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1037817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.05637788772583, + "objective/train/theoretical_loss": 3.455276663193372, + "objective/train/tokens_used": 194129376, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151515151515154, + "loss": 2.7749, + "theoretical_loss": 3.455276663193372, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514949494949495, + "loss": 2.6157, + "theoretical_loss": 3.4552662011446023, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147474747474746, + "loss": 2.7462, + "theoretical_loss": 3.4552557395740022, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045145454545454547, + "loss": 2.6743, + "theoretical_loss": 3.4552452784815317, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514343434343434, + "loss": 2.7419, + "theoretical_loss": 3.455234817867152, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141414141414144, + "loss": 2.78, + "theoretical_loss": 3.455224357730825, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513939393939394, + "loss": 2.7739, + "theoretical_loss": 3.4552138980725102, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513737373737374, + "loss": 2.7352, + "theoretical_loss": 3.4552034388921697, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135353535353536, + "loss": 2.7885, + "theoretical_loss": 3.4551929801897647, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513333333333333, + "loss": 2.9663, + "theoretical_loss": 3.455182521965256, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045131313131313133, + "loss": 2.7933, + "theoretical_loss": 3.455172064218605, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512929292929293, + "loss": 2.7499, + "theoretical_loss": 3.4551616069497726, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512727272727273, + "loss": 2.7672, + "theoretical_loss": 3.4551511501587195, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125252525252525, + "loss": 2.8145, + "theoretical_loss": 3.4551406938454075, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512323232323232, + "loss": 2.9202, + "theoretical_loss": 3.4551302380097972, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512121212121212, + "loss": 2.8165, + "theoretical_loss": 3.4551197826518507, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119191919191923, + "loss": 2.5778, + "theoretical_loss": 3.4551093277715275, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511717171717172, + "loss": 2.6501, + "theoretical_loss": 3.45509887336879, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115151515151515, + "loss": 2.9076, + "theoretical_loss": 3.455088419443599, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511313131313131, + "loss": 2.6364, + "theoretical_loss": 3.4550779659959154, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511111111111111, + "loss": 2.6785, + "theoretical_loss": 3.455067513025701, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510909090909091, + "loss": 2.992, + "theoretical_loss": 3.4550570605329165, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510707070707071, + "loss": 2.7593, + "theoretical_loss": 3.455046608517523, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045105050505050504, + "loss": 2.8443, + "theoretical_loss": 3.4550361569794816, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451030303030303, + "loss": 2.9233, + "theoretical_loss": 3.4550257059187537, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1038484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0293540954589844, + "objective/train/theoretical_loss": 3.4550152553353, + "objective/train/tokens_used": 195767776, + "theoretical_loss": 3.4550152553353, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045101010101010106, + "loss": 2.7728, + "theoretical_loss": 3.4550152553353, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450989898989899, + "loss": 2.7474, + "theoretical_loss": 3.4550048052290823, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450969696969697, + "loss": 2.6559, + "theoretical_loss": 3.454994355600062, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094949494949493, + "loss": 2.6298, + "theoretical_loss": 3.4549839064481995, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509292929292929, + "loss": 2.9199, + "theoretical_loss": 3.4549734577734563, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090909090909095, + "loss": 2.8747, + "theoretical_loss": 3.4549630095757937, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508888888888889, + "loss": 2.85, + "theoretical_loss": 3.4549525618551726, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086868686868687, + "loss": 2.8365, + "theoretical_loss": 3.454942114611555, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508484848484848, + "loss": 2.859, + "theoretical_loss": 3.454931667844901, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508282828282829, + "loss": 2.5456, + "theoretical_loss": 3.4549212215551726, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045080808080808085, + "loss": 2.7341, + "theoretical_loss": 3.4549107757423307, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507878787878788, + "loss": 2.6449, + "theoretical_loss": 3.4549003304063364, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076767676767676, + "loss": 2.7697, + "theoretical_loss": 3.4548898855471517, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507474747474747, + "loss": 2.6985, + "theoretical_loss": 3.4548794411647368, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507272727272728, + "loss": 2.8064, + "theoretical_loss": 3.4548689972590534, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045070707070707074, + "loss": 3.08, + "theoretical_loss": 3.4548585538300625, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506868686868687, + "loss": 2.9138, + "theoretical_loss": 3.454848110877726, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045066666666666665, + "loss": 2.6963, + "theoretical_loss": 3.4548376684020043, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064646464646466, + "loss": 2.8484, + "theoretical_loss": 3.45482722640286, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506262626262627, + "loss": 2.863, + "theoretical_loss": 3.4548167848802525, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060606060606063, + "loss": 2.6864, + "theoretical_loss": 3.454806343834144, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505858585858586, + "loss": 2.6592, + "theoretical_loss": 3.454795903264496, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056565656565655, + "loss": 2.977, + "theoretical_loss": 3.4547854631712696, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045054545454545456, + "loss": 2.9276, + "theoretical_loss": 3.454775023554426, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052525252525257, + "loss": 2.8782, + "theoretical_loss": 3.454764584413927, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1039769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9331185817718506, + "objective/train/theoretical_loss": 3.4547541457497326, + "objective/train/tokens_used": 197406176, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505050505050505, + "loss": 2.7239, + "theoretical_loss": 3.4547541457497326, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504848484848485, + "loss": 2.5853, + "theoretical_loss": 3.4547437075618053, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504646464646465, + "loss": 2.5352, + "theoretical_loss": 3.454733269850106, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045044444444444445, + "loss": 2.6486, + "theoretical_loss": 3.4547228326145953, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042424242424246, + "loss": 2.9982, + "theoretical_loss": 3.454712395855236, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504040404040404, + "loss": 2.8552, + "theoretical_loss": 3.454701959571988, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503838383838384, + "loss": 2.8642, + "theoretical_loss": 3.4546915237648137, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503636363636364, + "loss": 2.6347, + "theoretical_loss": 3.4546810884336736, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034343434343434, + "loss": 2.7538, + "theoretical_loss": 3.454670653578529, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032323232323235, + "loss": 2.82, + "theoretical_loss": 3.4546602191993427, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503030303030303, + "loss": 2.7994, + "theoretical_loss": 3.4546497852960742, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502828282828283, + "loss": 2.7817, + "theoretical_loss": 3.4546393518686855, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502626262626263, + "loss": 2.8347, + "theoretical_loss": 3.4546289189171375, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024242424242423, + "loss": 2.4303, + "theoretical_loss": 3.454618486441393, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045022222222222225, + "loss": 2.8937, + "theoretical_loss": 3.4546080544414117, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502020202020202, + "loss": 2.7481, + "theoretical_loss": 3.4545976229171558, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501818181818182, + "loss": 2.5923, + "theoretical_loss": 3.454587191868587, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016161616161617, + "loss": 2.8479, + "theoretical_loss": 3.454576761295665, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501414141414141, + "loss": 2.7934, + "theoretical_loss": 3.454566331198353, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012121212121214, + "loss": 2.8326, + "theoretical_loss": 3.4545559015766116, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045010101010101015, + "loss": 2.867, + "theoretical_loss": 3.4545454724304028, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500808080808081, + "loss": 2.7015, + "theoretical_loss": 3.4545350437596865, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045006060606060606, + "loss": 2.7364, + "theoretical_loss": 3.4545246155644254, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450040404040404, + "loss": 2.7326, + "theoretical_loss": 3.4545141878445804, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045002020202020203, + "loss": 2.7559, + "theoretical_loss": 3.4545037606001134, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1040587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.242511749267578, + "objective/train/theoretical_loss": 3.4544933338309853, + "objective/train/tokens_used": 199044576, + "theoretical_loss": 3.4544933338309853, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 2.7415, + "theoretical_loss": 3.4544933338309853, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449979797979798, + "loss": 2.7863, + "theoretical_loss": 3.4544829075371575, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995959595959596, + "loss": 2.8131, + "theoretical_loss": 3.4544724817185912, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499393939393939, + "loss": 2.7401, + "theoretical_loss": 3.454462056375249, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499191919191919, + "loss": 2.8743, + "theoretical_loss": 3.454451631507091, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044989898989898993, + "loss": 2.662, + "theoretical_loss": 3.454441207114079, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498787878787879, + "loss": 2.6488, + "theoretical_loss": 3.454430783196174, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985858585858585, + "loss": 2.8648, + "theoretical_loss": 3.4544203597533385, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498383838383838, + "loss": 2.8004, + "theoretical_loss": 3.454409936785533, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044981818181818187, + "loss": 2.6073, + "theoretical_loss": 3.45439951429272, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979797979797983, + "loss": 2.6904, + "theoretical_loss": 3.4543890922748597, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497777777777778, + "loss": 3.0477, + "theoretical_loss": 3.454378670731914, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044975757575757574, + "loss": 2.7516, + "theoretical_loss": 3.454368249663845, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497373737373737, + "loss": 2.7504, + "theoretical_loss": 3.454357829070613, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044971717171717176, + "loss": 2.5669, + "theoretical_loss": 3.4543474089521804, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496969696969697, + "loss": 2.8246, + "theoretical_loss": 3.4543369893085085, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496767676767677, + "loss": 2.7134, + "theoretical_loss": 3.4543265701395582, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965656565656563, + "loss": 2.7917, + "theoretical_loss": 3.4543161514452914, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044963636363636364, + "loss": 2.5886, + "theoretical_loss": 3.45430573322567, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961616161616166, + "loss": 2.5896, + "theoretical_loss": 3.4542953154806546, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495959595959596, + "loss": 2.7591, + "theoretical_loss": 3.4542848982102075, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957575757575757, + "loss": 2.7047, + "theoretical_loss": 3.4542744814142896, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495555555555555, + "loss": 2.8835, + "theoretical_loss": 3.454264065092863, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953535353535354, + "loss": 2.6635, + "theoretical_loss": 3.454253649245888, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044951515151515155, + "loss": 2.7854, + "theoretical_loss": 3.4542432338733278, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1041999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.937293767929077, + "objective/train/theoretical_loss": 3.4542328189751426, + "objective/train/tokens_used": 200682976, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494949494949495, + "loss": 2.593, + "theoretical_loss": 3.4542328189751426, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947474747474746, + "loss": 2.7933, + "theoretical_loss": 3.4542224045512944, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494545454545455, + "loss": 2.7946, + "theoretical_loss": 3.4542119906017446, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044943434343434343, + "loss": 2.6794, + "theoretical_loss": 3.4542015771264554, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941414141414144, + "loss": 2.7665, + "theoretical_loss": 3.454191164125387, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493939393939394, + "loss": 2.95, + "theoretical_loss": 3.4541807515985017, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937373737373735, + "loss": 2.8299, + "theoretical_loss": 3.4541703395457617, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044935353535353537, + "loss": 2.8046, + "theoretical_loss": 3.4541599279671273, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493333333333333, + "loss": 2.8173, + "theoretical_loss": 3.454149516862561, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931313131313133, + "loss": 2.5899, + "theoretical_loss": 3.4541391062320237, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492929292929293, + "loss": 2.9247, + "theoretical_loss": 3.454128696075477, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492727272727273, + "loss": 2.7234, + "theoretical_loss": 3.454118286392883, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044925252525252526, + "loss": 2.7411, + "theoretical_loss": 3.454107877184203, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492323232323232, + "loss": 2.6785, + "theoretical_loss": 3.4540974684493984, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492121212121212, + "loss": 2.9529, + "theoretical_loss": 3.4540870601884306, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491919191919192, + "loss": 2.63, + "theoretical_loss": 3.4540766524012616, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491717171717172, + "loss": 2.8365, + "theoretical_loss": 3.454066245087853, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915151515151515, + "loss": 2.6317, + "theoretical_loss": 3.4540558382481663, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044913131313131316, + "loss": 2.5956, + "theoretical_loss": 3.4540454318821627, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491111111111111, + "loss": 2.4991, + "theoretical_loss": 3.4540350259898043, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044909090909090913, + "loss": 2.7602, + "theoretical_loss": 3.4540246205710528, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490707070707071, + "loss": 2.8354, + "theoretical_loss": 3.454014215625869, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044905050505050504, + "loss": 2.8201, + "theoretical_loss": 3.4540038111542155, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903030303030306, + "loss": 2.7807, + "theoretical_loss": 3.4539934071560534, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449010101010101, + "loss": 2.7007, + "theoretical_loss": 3.453983003631344, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1042423, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3816003799438477, + "objective/train/theoretical_loss": 3.45397260058005, + "objective/train/tokens_used": 202321376, + "theoretical_loss": 3.45397260058005, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448989898989899, + "loss": 2.7819, + "theoretical_loss": 3.45397260058005, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448969696969697, + "loss": 2.7296, + "theoretical_loss": 3.453962198002132, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044894949494949494, + "loss": 2.7357, + "theoretical_loss": 3.4539517958975514, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892929292929295, + "loss": 2.8821, + "theoretical_loss": 3.4539413942662707, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890909090909096, + "loss": 2.8082, + "theoretical_loss": 3.4539309931082514, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488888888888889, + "loss": 2.6764, + "theoretical_loss": 3.4539205924234553, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044886868686868687, + "loss": 2.6435, + "theoretical_loss": 3.4539101922118434, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044884848484848483, + "loss": 2.7416, + "theoretical_loss": 3.4538997924733774, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882828282828284, + "loss": 2.7244, + "theoretical_loss": 3.4538893932080197, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044880808080808085, + "loss": 2.6938, + "theoretical_loss": 3.4538789944157315, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487878787878788, + "loss": 2.6852, + "theoretical_loss": 3.4538685960964743, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044876767676767677, + "loss": 2.8495, + "theoretical_loss": 3.45385819825021, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487474747474747, + "loss": 2.7925, + "theoretical_loss": 3.4538478008769, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487272727272728, + "loss": 2.4651, + "theoretical_loss": 3.4538374039765065, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044870707070707074, + "loss": 2.859, + "theoretical_loss": 3.4538270075489907, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486868686868687, + "loss": 2.6411, + "theoretical_loss": 3.4538166115943145, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866666666666666, + "loss": 2.6933, + "theoretical_loss": 3.45380621611244, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486464646464646, + "loss": 2.6706, + "theoretical_loss": 3.453795821103328, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486262626262627, + "loss": 2.6051, + "theoretical_loss": 3.4537854265669408, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044860606060606064, + "loss": 2.7884, + "theoretical_loss": 3.4537750325032404, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485858585858586, + "loss": 2.6922, + "theoretical_loss": 3.4537646389121877, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044856565656565655, + "loss": 2.6397, + "theoretical_loss": 3.4537542457937453, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485454545454545, + "loss": 2.6908, + "theoretical_loss": 3.4537438531478735, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485252525252526, + "loss": 2.7295, + "theoretical_loss": 3.4537334609745356, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850505050505053, + "loss": 2.9012, + "theoretical_loss": 3.4537230692736927, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1043833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.584007740020752, + "objective/train/theoretical_loss": 3.4537126780453065, + "objective/train/tokens_used": 203959776, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484848484848485, + "loss": 2.7326, + "theoretical_loss": 3.4537126780453065, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044846464646464644, + "loss": 2.7066, + "theoretical_loss": 3.453702287289339, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844444444444445, + "loss": 2.7772, + "theoretical_loss": 3.4536918970057515, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842424242424247, + "loss": 2.8463, + "theoretical_loss": 3.453681507194506, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484040404040404, + "loss": 2.735, + "theoretical_loss": 3.4536711178555644, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483838383838384, + "loss": 2.7291, + "theoretical_loss": 3.453660728988888, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836363636363634, + "loss": 2.7237, + "theoretical_loss": 3.453650340594439, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834343434343435, + "loss": 2.8545, + "theoretical_loss": 3.453639952672179, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044832323232323236, + "loss": 2.8892, + "theoretical_loss": 3.4536295652220694, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483030303030303, + "loss": 2.6493, + "theoretical_loss": 3.453619178244073, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828282828282827, + "loss": 2.703, + "theoretical_loss": 3.4536087917381506, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482626262626263, + "loss": 2.9192, + "theoretical_loss": 3.4535984057042644, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044824242424242424, + "loss": 2.7044, + "theoretical_loss": 3.4535880201423765, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044822222222222225, + "loss": 2.6079, + "theoretical_loss": 3.4535776350524476, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482020202020202, + "loss": 2.8396, + "theoretical_loss": 3.453567250434441, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818181818181816, + "loss": 2.8414, + "theoretical_loss": 3.453556866288317, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481616161616162, + "loss": 3.0126, + "theoretical_loss": 3.453546482614038, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814141414141413, + "loss": 2.7239, + "theoretical_loss": 3.4535360994115667, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044812121212121214, + "loss": 2.8923, + "theoretical_loss": 3.453525716680863, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481010101010101, + "loss": 2.9057, + "theoretical_loss": 3.453515334421891, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480808080808081, + "loss": 2.5546, + "theoretical_loss": 3.453504952634611, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806060606060607, + "loss": 2.7031, + "theoretical_loss": 3.453494571318985, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448040404040404, + "loss": 2.9167, + "theoretical_loss": 3.453484190474976, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802020202020204, + "loss": 2.8575, + "theoretical_loss": 3.4534738101025435, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448, + "loss": 2.6198, + "theoretical_loss": 3.4534634302016514, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1044447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.853868007659912, + "objective/train/theoretical_loss": 3.4534530507722607, + "objective/train/tokens_used": 205598176, + "theoretical_loss": 3.4534530507722607, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447979797979798, + "loss": 2.7887, + "theoretical_loss": 3.4534530507722607, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795959595959596, + "loss": 2.9349, + "theoretical_loss": 3.453442671814334, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479393939393939, + "loss": 2.8864, + "theoretical_loss": 3.4534322933278316, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044791919191919193, + "loss": 2.6109, + "theoretical_loss": 3.453421915312717, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044789898989898994, + "loss": 2.7262, + "theoretical_loss": 3.453411537768951, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478787878787879, + "loss": 2.6512, + "theoretical_loss": 3.4534011606964965, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785858585858585, + "loss": 2.8101, + "theoretical_loss": 3.4533907840953146, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478383838383838, + "loss": 2.7451, + "theoretical_loss": 3.453380407965367, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478181818181818, + "loss": 2.8169, + "theoretical_loss": 3.4533700323066157, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779797979797983, + "loss": 2.6043, + "theoretical_loss": 3.4533596571190235, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477777777777778, + "loss": 2.6044, + "theoretical_loss": 3.4533492824025513, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044775757575757575, + "loss": 2.6643, + "theoretical_loss": 3.453338908157161, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477373737373737, + "loss": 2.7256, + "theoretical_loss": 3.4533285343828153, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771717171717177, + "loss": 2.6265, + "theoretical_loss": 3.453318161079475, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476969696969697, + "loss": 2.7167, + "theoretical_loss": 3.453307788247103, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476767676767677, + "loss": 2.8601, + "theoretical_loss": 3.45329741588566, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044765656565656564, + "loss": 2.7338, + "theoretical_loss": 3.4532870439951098, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476363636363636, + "loss": 2.7701, + "theoretical_loss": 3.453276672575413, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044761616161616166, + "loss": 2.7614, + "theoretical_loss": 3.4532663016265315, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475959595959596, + "loss": 2.8528, + "theoretical_loss": 3.4532559311484277, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475757575757576, + "loss": 2.7737, + "theoretical_loss": 3.4532455611410633, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755555555555553, + "loss": 2.5977, + "theoretical_loss": 3.4532351916044, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044753535353535354, + "loss": 2.6602, + "theoretical_loss": 3.4532248225384006, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044751515151515155, + "loss": 2.6128, + "theoretical_loss": 3.453214453943026, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474949494949495, + "loss": 2.6234, + "theoretical_loss": 3.4532040858182387, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1045763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.294877052307129, + "objective/train/theoretical_loss": 3.453193718164001, + "objective/train/tokens_used": 207236576, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747474747474747, + "loss": 2.6171, + "theoretical_loss": 3.453193718164001, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474545454545454, + "loss": 2.5522, + "theoretical_loss": 3.4531833509802734, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743434343434344, + "loss": 2.562, + "theoretical_loss": 3.45317298426702, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741414141414145, + "loss": 2.5335, + "theoretical_loss": 3.453162618024201, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473939393939394, + "loss": 2.7062, + "theoretical_loss": 3.4531522522517797, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044737373737373736, + "loss": 2.8025, + "theoretical_loss": 3.453141886949717, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735353535353537, + "loss": 2.6407, + "theoretical_loss": 3.453131522117975, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473333333333334, + "loss": 2.8705, + "theoretical_loss": 3.4531211577565166, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044731313131313134, + "loss": 2.9645, + "theoretical_loss": 3.453110793865303, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472929292929293, + "loss": 2.7202, + "theoretical_loss": 3.4531004304442963, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727272727272725, + "loss": 2.8856, + "theoretical_loss": 3.4530900674934584, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044725252525252526, + "loss": 2.8725, + "theoretical_loss": 3.453079705012752, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472323232323233, + "loss": 2.8119, + "theoretical_loss": 3.4530693430021384, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044721212121212123, + "loss": 2.7535, + "theoretical_loss": 3.45305898146158, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471919191919192, + "loss": 2.6924, + "theoretical_loss": 3.453048620391038, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471717171717172, + "loss": 2.5326, + "theoretical_loss": 3.4530382597904756, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715151515151516, + "loss": 2.5854, + "theoretical_loss": 3.4530278996598547, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044713131313131317, + "loss": 2.5726, + "theoretical_loss": 3.453017539999136, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471111111111111, + "loss": 2.8325, + "theoretical_loss": 3.4530071808082834, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470909090909091, + "loss": 2.8497, + "theoretical_loss": 3.4529968220872576, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470707070707071, + "loss": 2.655, + "theoretical_loss": 3.4529864638360213, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044705050505050505, + "loss": 2.7491, + "theoretical_loss": 3.452976106054536, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703030303030306, + "loss": 3.0076, + "theoretical_loss": 3.4529657487427636, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447010101010101, + "loss": 2.5165, + "theoretical_loss": 3.4529553919006672, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698989898989903, + "loss": 2.6784, + "theoretical_loss": 3.4529450355282085, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 1046397, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6954104900360107, + "objective/train/theoretical_loss": 3.452934679625349, + "objective/train/tokens_used": 208874976, + "theoretical_loss": 3.452934679625349, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446969696969697, + "loss": 2.695, + "theoretical_loss": 3.452934679625349, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044694949494949494, + "loss": 2.6491, + "theoretical_loss": 3.4529243241920513, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044692929292929295, + "loss": 2.8544, + "theoretical_loss": 3.452913969228277, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469090909090909, + "loss": 2.7055, + "theoretical_loss": 3.4529036147339887, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468888888888889, + "loss": 2.7715, + "theoretical_loss": 3.452893260709148, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468686868686869, + "loss": 2.8593, + "theoretical_loss": 3.4528829071537177, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684848484848484, + "loss": 2.6011, + "theoretical_loss": 3.4528725540676595, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682828282828285, + "loss": 2.8535, + "theoretical_loss": 3.452862201450935, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468080808080808, + "loss": 3.0192, + "theoretical_loss": 3.452851849303507, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467878787878788, + "loss": 2.834, + "theoretical_loss": 3.452841497625337, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676767676767677, + "loss": 2.6329, + "theoretical_loss": 3.452831146416388, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674747474747473, + "loss": 2.8201, + "theoretical_loss": 3.4528207956766215, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672727272727274, + "loss": 2.9282, + "theoretical_loss": 3.4528104454059996, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044670707070707075, + "loss": 2.7681, + "theoretical_loss": 3.4528000956044846, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466868686868687, + "loss": 2.3523, + "theoretical_loss": 3.4527897462720385, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044666666666666666, + "loss": 2.5514, + "theoretical_loss": 3.452779397408624, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466464646464646, + "loss": 2.9287, + "theoretical_loss": 3.452769049014202, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044662626262626263, + "loss": 2.6313, + "theoretical_loss": 3.4527587010887357, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660606060606064, + "loss": 2.7763, + "theoretical_loss": 3.4527483536321864, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465858585858586, + "loss": 2.7668, + "theoretical_loss": 3.4527380066445175, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656565656565656, + "loss": 2.6097, + "theoretical_loss": 3.4527276601256904, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465454545454545, + "loss": 2.6688, + "theoretical_loss": 3.4527173140756666, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465252525252526, + "loss": 2.8219, + "theoretical_loss": 3.4527069684944096, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044650505050505054, + "loss": 2.603, + "theoretical_loss": 3.4526966233818808, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464848484848485, + "loss": 2.704, + "theoretical_loss": 3.4526862787380423, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1047383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7329440116882324, + "objective/train/theoretical_loss": 3.4526759345628566, + "objective/train/tokens_used": 210513376, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044646464646464645, + "loss": 2.7305, + "theoretical_loss": 3.4526759345628566, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464444444444444, + "loss": 2.7622, + "theoretical_loss": 3.452665590856286, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044642424242424247, + "loss": 2.5636, + "theoretical_loss": 3.452655247618292, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640404040404043, + "loss": 2.6177, + "theoretical_loss": 3.452644904848838, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463838383838384, + "loss": 2.8558, + "theoretical_loss": 3.4526345625478845, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044636363636363634, + "loss": 2.8706, + "theoretical_loss": 3.4526242207153954, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634343434343435, + "loss": 2.7634, + "theoretical_loss": 3.4526138793513317, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632323232323236, + "loss": 2.9884, + "theoretical_loss": 3.452603538455656, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463030303030303, + "loss": 2.5586, + "theoretical_loss": 3.452593198028331, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462828282828283, + "loss": 2.6976, + "theoretical_loss": 3.4525828580693183, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626262626262623, + "loss": 2.7232, + "theoretical_loss": 3.45257251857858, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624242424242425, + "loss": 2.8818, + "theoretical_loss": 3.452562179556079, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622222222222226, + "loss": 2.6422, + "theoretical_loss": 3.452551841001777, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462020202020202, + "loss": 2.6963, + "theoretical_loss": 3.4525415029156363, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618181818181817, + "loss": 2.3642, + "theoretical_loss": 3.452531165297619, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461616161616162, + "loss": 2.5992, + "theoretical_loss": 3.452520828147688, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614141414141414, + "loss": 2.7142, + "theoretical_loss": 3.452510491465805, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044612121212121215, + "loss": 2.6749, + "theoretical_loss": 3.4525001552519323, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461010101010101, + "loss": 2.7101, + "theoretical_loss": 3.4524898195060323, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044608080808080806, + "loss": 2.6496, + "theoretical_loss": 3.4524794842280673, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460606060606061, + "loss": 2.6907, + "theoretical_loss": 3.4524691494179995, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044604040404040403, + "loss": 2.886, + "theoretical_loss": 3.452458815075791, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602020202020204, + "loss": 2.699, + "theoretical_loss": 3.452448481201404, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446, + "loss": 2.7617, + "theoretical_loss": 3.452438147794801, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445979797979798, + "loss": 2.6425, + "theoretical_loss": 3.4524278148559446, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1048172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8061985969543457, + "objective/train/theoretical_loss": 3.4524174823847966, + "objective/train/tokens_used": 212151776, + "theoretical_loss": 3.4524174823847966, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044595959595959597, + "loss": 2.6553, + "theoretical_loss": 3.4524174823847966, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459393939393939, + "loss": 2.5941, + "theoretical_loss": 3.452407150381319, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591919191919193, + "loss": 2.7949, + "theoretical_loss": 3.4523968188454752, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458989898989899, + "loss": 2.8215, + "theoretical_loss": 3.452386487777227, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458787878787879, + "loss": 2.8149, + "theoretical_loss": 3.452376157176536, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585858585858586, + "loss": 2.8937, + "theoretical_loss": 3.452365827043365, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458383838383838, + "loss": 2.9624, + "theoretical_loss": 3.4523554973776767, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044581818181818183, + "loss": 2.7514, + "theoretical_loss": 3.452345168179433, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044579797979797984, + "loss": 2.7052, + "theoretical_loss": 3.4523348394485964, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457777777777778, + "loss": 2.5312, + "theoretical_loss": 3.452324511185129, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575757575757575, + "loss": 2.8883, + "theoretical_loss": 3.452314183388993, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457373737373737, + "loss": 2.6577, + "theoretical_loss": 3.4523038560601513, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171717171717, + "loss": 2.7302, + "theoretical_loss": 3.452293529198566, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569696969696973, + "loss": 2.6482, + "theoretical_loss": 3.4522832028041988, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456767676767677, + "loss": 2.7209, + "theoretical_loss": 3.4522728768770135, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565656565656564, + "loss": 2.733, + "theoretical_loss": 3.452262551416971, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044563636363636366, + "loss": 2.7799, + "theoretical_loss": 3.4522522264240347, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561616161616167, + "loss": 2.9549, + "theoretical_loss": 3.452241901898166, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455959595959596, + "loss": 2.879, + "theoretical_loss": 3.452231577839328, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455757575757576, + "loss": 2.787, + "theoretical_loss": 3.452221254247483, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044555555555555554, + "loss": 2.6501, + "theoretical_loss": 3.4522109311225933, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553535353535355, + "loss": 2.559, + "theoretical_loss": 3.4522006084646204, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044551515151515156, + "loss": 2.7236, + "theoretical_loss": 3.4521902862735283, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454949494949495, + "loss": 2.7109, + "theoretical_loss": 3.4521799645492783, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454747474747475, + "loss": 2.713, + "theoretical_loss": 3.452169643291833, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1049323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6239025592803955, + "objective/train/theoretical_loss": 3.452159322501155, + "objective/train/tokens_used": 213790176, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545454545454543, + "loss": 2.706, + "theoretical_loss": 3.452159322501155, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454343434343435, + "loss": 2.7307, + "theoretical_loss": 3.4521490021772063, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044541414141414145, + "loss": 2.8314, + "theoretical_loss": 3.45213868231995, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453939393939394, + "loss": 2.5119, + "theoretical_loss": 3.452128362929348, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537373737373737, + "loss": 2.6978, + "theoretical_loss": 3.4521180440053625, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453535353535353, + "loss": 2.809, + "theoretical_loss": 3.452107725547956, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453333333333334, + "loss": 2.5285, + "theoretical_loss": 3.4520974075570914, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531313131313135, + "loss": 2.8153, + "theoretical_loss": 3.452087090032731, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452929292929293, + "loss": 2.6924, + "theoretical_loss": 3.452076772974837, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527272727272726, + "loss": 2.6262, + "theoretical_loss": 3.4520664563833723, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452525252525252, + "loss": 2.8656, + "theoretical_loss": 3.452056140258298, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452323232323233, + "loss": 2.6025, + "theoretical_loss": 3.4520458245995784, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521212121212124, + "loss": 2.7464, + "theoretical_loss": 3.4520355094071746, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451919191919192, + "loss": 2.5115, + "theoretical_loss": 3.45202519468105, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517171717171715, + "loss": 2.8408, + "theoretical_loss": 3.452014880421166, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044515151515151516, + "loss": 2.788, + "theoretical_loss": 3.452004566627486, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451313131313132, + "loss": 2.6661, + "theoretical_loss": 3.4519942532999712, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044511111111111113, + "loss": 2.7606, + "theoretical_loss": 3.451983940438586, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450909090909091, + "loss": 2.6389, + "theoretical_loss": 3.4519736280432918, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044507070707070704, + "loss": 2.6303, + "theoretical_loss": 3.4519633161140506, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044505050505050506, + "loss": 2.7242, + "theoretical_loss": 3.4519530046508256, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044503030303030307, + "loss": 2.6052, + "theoretical_loss": 3.451942693653579, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445010101010101, + "loss": 2.7507, + "theoretical_loss": 3.4519323831222737, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444989898989899, + "loss": 2.6374, + "theoretical_loss": 3.4519220730568714, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444969696969697, + "loss": 2.6468, + "theoretical_loss": 3.4519117634573355, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1049929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4704978466033936, + "objective/train/theoretical_loss": 3.451901454323628, + "objective/train/tokens_used": 215428576, + "theoretical_loss": 3.451901454323628, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044494949494949495, + "loss": 2.682, + "theoretical_loss": 3.451901454323628, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044492929292929296, + "loss": 2.7669, + "theoretical_loss": 3.4518911456557113, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449090909090909, + "loss": 2.7552, + "theoretical_loss": 3.4518808374535483, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488888888888887, + "loss": 2.7886, + "theoretical_loss": 3.4518705297171013, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448686868686869, + "loss": 2.6318, + "theoretical_loss": 3.4518602224463324, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044484848484848484, + "loss": 2.8501, + "theoretical_loss": 3.4518499156412052, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482828282828285, + "loss": 2.7015, + "theoretical_loss": 3.4518396093016817, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448080808080808, + "loss": 2.7465, + "theoretical_loss": 3.451829303427724, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447878787878788, + "loss": 2.8551, + "theoretical_loss": 3.451818998019295, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447676767676768, + "loss": 2.6703, + "theoretical_loss": 3.451808693076357, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474747474747473, + "loss": 2.7308, + "theoretical_loss": 3.4517983885988732, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472727272727274, + "loss": 2.6342, + "theoretical_loss": 3.4517880845868056, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447070707070707, + "loss": 2.9306, + "theoretical_loss": 3.4517777810401165, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446868686868687, + "loss": 2.6936, + "theoretical_loss": 3.4517674779587693, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466666666666667, + "loss": 2.8203, + "theoretical_loss": 3.451757175342726, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446464646464646, + "loss": 2.7962, + "theoretical_loss": 3.451746873191949, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462626262626264, + "loss": 2.5618, + "theoretical_loss": 3.4517365715064017, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044460606060606065, + "loss": 2.798, + "theoretical_loss": 3.451726270286046, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445858585858586, + "loss": 2.8126, + "theoretical_loss": 3.451715969530844, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044456565656565656, + "loss": 2.8469, + "theoretical_loss": 3.4517056692407597, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445454545454545, + "loss": 2.84, + "theoretical_loss": 3.451695369415755, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044452525252525253, + "loss": 2.6725, + "theoretical_loss": 3.4516850700557917, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450505050505054, + "loss": 2.8752, + "theoretical_loss": 3.451674771160833, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444848484848485, + "loss": 2.7244, + "theoretical_loss": 3.451664472730842, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044446464646464645, + "loss": 2.7359, + "theoretical_loss": 3.451654174765781, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1050597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.185572862625122, + "objective/train/theoretical_loss": 3.451643877265613, + "objective/train/tokens_used": 217066976, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444444444444444, + "loss": 2.9036, + "theoretical_loss": 3.451643877265613, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444242424242425, + "loss": 2.7681, + "theoretical_loss": 3.451633580230299, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044440404040404043, + "loss": 2.8137, + "theoretical_loss": 3.4516232836598033, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443838383838384, + "loss": 2.5401, + "theoretical_loss": 3.451612987554088, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436363636363635, + "loss": 2.8946, + "theoretical_loss": 3.451602691913116, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443434343434343, + "loss": 2.7779, + "theoretical_loss": 3.4515923967368494, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044432323232323237, + "loss": 2.7997, + "theoretical_loss": 3.4515821020252506, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443030303030303, + "loss": 2.5496, + "theoretical_loss": 3.4515718077782838, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442828282828283, + "loss": 2.6267, + "theoretical_loss": 3.4515615139959097, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426262626262624, + "loss": 2.7195, + "theoretical_loss": 3.451551220678092, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044424242424242425, + "loss": 2.8553, + "theoretical_loss": 3.4515409278247935, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422222222222226, + "loss": 2.3626, + "theoretical_loss": 3.4515306354359767, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442020202020202, + "loss": 2.8699, + "theoretical_loss": 3.4515203435116035, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441818181818182, + "loss": 2.7496, + "theoretical_loss": 3.4515100520516375, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044416161616161613, + "loss": 2.7206, + "theoretical_loss": 3.4514997610560414, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414141414141414, + "loss": 2.679, + "theoretical_loss": 3.451489470524777, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044412121212121215, + "loss": 2.6295, + "theoretical_loss": 3.4514791804578078, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441010101010101, + "loss": 3.0401, + "theoretical_loss": 3.4514688908550966, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044408080808080807, + "loss": 2.6389, + "theoretical_loss": 3.451458601716605, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440606060606061, + "loss": 2.944, + "theoretical_loss": 3.451448313042297, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044404040404040404, + "loss": 2.6815, + "theoretical_loss": 3.451438024832134, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044402020202020205, + "loss": 2.6059, + "theoretical_loss": 3.45142773708608, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444, + "loss": 2.7036, + "theoretical_loss": 3.451417449804097, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044397979797979796, + "loss": 2.4589, + "theoretical_loss": 3.451407162986148, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044395959595959597, + "loss": 2.7425, + "theoretical_loss": 3.451396876632195, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1051522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.380699634552002, + "objective/train/theoretical_loss": 3.4513865907422017, + "objective/train/tokens_used": 218705376, + "theoretical_loss": 3.4513865907422017, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443939393939394, + "loss": 2.7935, + "theoretical_loss": 3.4513865907422017, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391919191919194, + "loss": 2.6264, + "theoretical_loss": 3.4513763053161304, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438989898989899, + "loss": 2.8886, + "theoretical_loss": 3.4513660203539436, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438787878787879, + "loss": 2.7372, + "theoretical_loss": 3.4513557358556044, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385858585858587, + "loss": 2.7386, + "theoretical_loss": 3.4513454518210755, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438383838383839, + "loss": 2.521, + "theoretical_loss": 3.4513351682503193, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044381818181818183, + "loss": 2.7777, + "theoretical_loss": 3.4513248851432987, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437979797979798, + "loss": 2.5528, + "theoretical_loss": 3.451314602499977, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437777777777778, + "loss": 2.6142, + "theoretical_loss": 3.451304320320316, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375757575757576, + "loss": 2.6724, + "theoretical_loss": 3.451294038604279, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044373737373737377, + "loss": 2.9276, + "theoretical_loss": 3.4512837573518294, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437171717171717, + "loss": 2.623, + "theoretical_loss": 3.4512734765629283, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369696969696974, + "loss": 2.9157, + "theoretical_loss": 3.45126319623754, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436767676767677, + "loss": 2.5669, + "theoretical_loss": 3.451252916375627, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365656565656565, + "loss": 2.9155, + "theoretical_loss": 3.4512426369771516, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044363636363636366, + "loss": 2.6364, + "theoretical_loss": 3.4512323580420765, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436161616161616, + "loss": 2.7681, + "theoretical_loss": 3.451222079570365, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044359595959595963, + "loss": 2.505, + "theoretical_loss": 3.4512118015619797, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435757575757576, + "loss": 2.9545, + "theoretical_loss": 3.4512015240168834, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044355555555555554, + "loss": 2.8762, + "theoretical_loss": 3.451191246935039, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044353535353535355, + "loss": 2.6851, + "theoretical_loss": 3.4511809703164094, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435151515151515, + "loss": 2.9089, + "theoretical_loss": 3.4511706941609566, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434949494949495, + "loss": 2.5863, + "theoretical_loss": 3.4511604184686444, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434747474747475, + "loss": 2.7208, + "theoretical_loss": 3.4511501432394356, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044345454545454544, + "loss": 2.7296, + "theoretical_loss": 3.451139868473292, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1052031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3065507411956787, + "objective/train/theoretical_loss": 3.4511295941701774, + "objective/train/tokens_used": 220343776, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044343434343434345, + "loss": 2.4831, + "theoretical_loss": 3.4511295941701774, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341414141414146, + "loss": 2.7927, + "theoretical_loss": 3.4511193203300543, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433939393939394, + "loss": 2.8696, + "theoretical_loss": 3.451109046952886, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044337373737373737, + "loss": 2.7052, + "theoretical_loss": 3.451098774038634, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335353535353533, + "loss": 2.9311, + "theoretical_loss": 3.4510885015872628, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044333333333333334, + "loss": 2.4575, + "theoretical_loss": 3.4510782295987346, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044331313131313135, + "loss": 2.678, + "theoretical_loss": 3.4510679580730117, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432929292929293, + "loss": 2.8799, + "theoretical_loss": 3.4510576870100573, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044327272727272726, + "loss": 2.5043, + "theoretical_loss": 3.4510474164098346, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432525252525252, + "loss": 2.8704, + "theoretical_loss": 3.4510371462723066, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432323232323233, + "loss": 2.7391, + "theoretical_loss": 3.4510268765974357, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321212121212124, + "loss": 2.7561, + "theoretical_loss": 3.451016607385185, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431919191919192, + "loss": 2.7656, + "theoretical_loss": 3.451006338635517, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317171717171716, + "loss": 2.5423, + "theoretical_loss": 3.4509960703483946, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431515151515151, + "loss": 2.6938, + "theoretical_loss": 3.450985802523782, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431313131313132, + "loss": 2.4388, + "theoretical_loss": 3.45097553516164, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044311111111111114, + "loss": 2.7645, + "theoretical_loss": 3.4509652682619327, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430909090909091, + "loss": 2.9827, + "theoretical_loss": 3.4509550018246236, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044307070707070705, + "loss": 2.7175, + "theoretical_loss": 3.4509447358496743, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305050505050506, + "loss": 2.7346, + "theoretical_loss": 3.450934470337048, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303030303030307, + "loss": 2.5073, + "theoretical_loss": 3.450924205286708, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301010101010103, + "loss": 2.7879, + "theoretical_loss": 3.4509139406986176, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442989898989899, + "loss": 2.8311, + "theoretical_loss": 3.450903676572739, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044296969696969694, + "loss": 2.5745, + "theoretical_loss": 3.4508934129090356, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294949494949495, + "loss": 2.784, + "theoretical_loss": 3.4508831497074697, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1053498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9018161296844482, + "objective/train/theoretical_loss": 3.4508728869680043, + "objective/train/tokens_used": 221982176, + "theoretical_loss": 3.4508728869680043, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292929292929296, + "loss": 3.0838, + "theoretical_loss": 3.4508728869680043, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429090909090909, + "loss": 2.7731, + "theoretical_loss": 3.450862624690603, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428888888888889, + "loss": 2.7241, + "theoretical_loss": 3.450852362875229, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428686868686869, + "loss": 2.8578, + "theoretical_loss": 3.450842101521844, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284848484848485, + "loss": 2.7499, + "theoretical_loss": 3.4508318406304115, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044282828282828286, + "loss": 2.7221, + "theoretical_loss": 3.4508215802008952, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428080808080808, + "loss": 2.5153, + "theoretical_loss": 3.450811320233257, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044278787878787877, + "loss": 2.8814, + "theoretical_loss": 3.4508010607274606, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427676767676768, + "loss": 2.7349, + "theoretical_loss": 3.4507908016834685, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044274747474747474, + "loss": 2.6904, + "theoretical_loss": 3.4507805431012435, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272727272727275, + "loss": 2.6469, + "theoretical_loss": 3.4507702849807496, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427070707070707, + "loss": 2.6868, + "theoretical_loss": 3.450760027321948, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426868686868687, + "loss": 2.6615, + "theoretical_loss": 3.450749770124804, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426666666666667, + "loss": 2.8078, + "theoretical_loss": 3.4507395133892786, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264646464646463, + "loss": 2.6671, + "theoretical_loss": 3.450729257115336, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044262626262626264, + "loss": 2.7196, + "theoretical_loss": 3.4507190013029385, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426060606060606, + "loss": 2.8495, + "theoretical_loss": 3.45070874595205, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425858585858586, + "loss": 2.8353, + "theoretical_loss": 3.4506984910626324, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256565656565657, + "loss": 2.7421, + "theoretical_loss": 3.450688236634649, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425454545454545, + "loss": 2.6083, + "theoretical_loss": 3.450677982668063, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252525252525254, + "loss": 2.3958, + "theoretical_loss": 3.450667729162838, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044250505050505055, + "loss": 2.6759, + "theoretical_loss": 3.450657476118936, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424848484848485, + "loss": 2.8168, + "theoretical_loss": 3.450647223536321, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246464646464646, + "loss": 2.7833, + "theoretical_loss": 3.450636971414955, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424444444444444, + "loss": 2.7664, + "theoretical_loss": 3.4506267197548017, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1054143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7794628143310547, + "objective/train/theoretical_loss": 3.4506164685558245, + "objective/train/tokens_used": 223620576, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044242424242424243, + "loss": 2.8384, + "theoretical_loss": 3.4506164685558245, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044240404040404044, + "loss": 2.6512, + "theoretical_loss": 3.4506062178179855, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423838383838384, + "loss": 2.8355, + "theoretical_loss": 3.450595967541248, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044236363636363635, + "loss": 2.6564, + "theoretical_loss": 3.450585717725576, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423434343434343, + "loss": 2.8707, + "theoretical_loss": 3.450575468370931, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423232323232324, + "loss": 2.888, + "theoretical_loss": 3.4505652194772773, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230303030303033, + "loss": 2.6248, + "theoretical_loss": 3.4505549710445775, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422828282828283, + "loss": 2.5991, + "theoretical_loss": 3.450544723072795, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044226262626262625, + "loss": 2.8479, + "theoretical_loss": 3.4505344755618923, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422424242424242, + "loss": 2.7885, + "theoretical_loss": 3.450524228511833, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222222222222227, + "loss": 2.7331, + "theoretical_loss": 3.4505139819225796, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422020202020202, + "loss": 2.9298, + "theoretical_loss": 3.450503735794096, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421818181818182, + "loss": 2.8044, + "theoretical_loss": 3.4504934901263447, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216161616161614, + "loss": 2.4662, + "theoretical_loss": 3.450483244919289, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421414141414142, + "loss": 2.8921, + "theoretical_loss": 3.450473000172892, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212121212121216, + "loss": 2.6669, + "theoretical_loss": 3.450462755887117, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421010101010101, + "loss": 2.7412, + "theoretical_loss": 3.4504525120619265, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420808080808081, + "loss": 2.8152, + "theoretical_loss": 3.4504422686972838, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044206060606060603, + "loss": 2.7622, + "theoretical_loss": 3.4504320257931527, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420404040404041, + "loss": 2.9355, + "theoretical_loss": 3.450421783349496, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202020202020205, + "loss": 2.6807, + "theoretical_loss": 3.450411541366276, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442, + "loss": 3.0409, + "theoretical_loss": 3.450401299843457, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197979797979797, + "loss": 2.6968, + "theoretical_loss": 3.450391058781001, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419595959595959, + "loss": 2.5647, + "theoretical_loss": 3.450380818178872, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441939393939394, + "loss": 2.8054, + "theoretical_loss": 3.4503705780370337, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 1055457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.576772451400757, + "objective/train/theoretical_loss": 3.4503603383554475, + "objective/train/tokens_used": 225258976, + "theoretical_loss": 3.4503603383554475, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044191919191919195, + "loss": 2.5916, + "theoretical_loss": 3.4503603383554475, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418989898989899, + "loss": 2.7917, + "theoretical_loss": 3.450350099134078, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044187878787878786, + "loss": 2.7427, + "theoretical_loss": 3.450339860372888, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044185858585858587, + "loss": 2.6437, + "theoretical_loss": 3.4503296220718402, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418383838383839, + "loss": 2.6662, + "theoretical_loss": 3.4503193842308986, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181818181818184, + "loss": 2.8119, + "theoretical_loss": 3.4503091468500253, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417979797979798, + "loss": 2.5738, + "theoretical_loss": 3.4502989099291845, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044177777777777775, + "loss": 2.8838, + "theoretical_loss": 3.450288673468339, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044175757575757576, + "loss": 2.4982, + "theoretical_loss": 3.4502784374674516, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417373737373738, + "loss": 2.6966, + "theoretical_loss": 3.4502682019264856, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044171717171717173, + "loss": 2.6857, + "theoretical_loss": 3.450257966845405, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416969696969697, + "loss": 2.6504, + "theoretical_loss": 3.450247732224172, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416767676767677, + "loss": 2.6662, + "theoretical_loss": 3.4502374980627506, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165656565656566, + "loss": 2.8217, + "theoretical_loss": 3.4502272643611027, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044163636363636367, + "loss": 2.7267, + "theoretical_loss": 3.4502170311191933, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416161616161616, + "loss": 2.6704, + "theoretical_loss": 3.4502067983369846, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415959595959596, + "loss": 2.6943, + "theoretical_loss": 3.45019656601444, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415757575757576, + "loss": 2.5005, + "theoretical_loss": 3.450186334151522, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044155555555555555, + "loss": 2.8005, + "theoretical_loss": 3.4501761027481948, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044153535353535356, + "loss": 2.5185, + "theoretical_loss": 3.4501658718044217, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415151515151515, + "loss": 2.5402, + "theoretical_loss": 3.4501556413201646, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149494949494953, + "loss": 2.464, + "theoretical_loss": 3.450145411295389, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414747474747475, + "loss": 2.5489, + "theoretical_loss": 3.450135181730056, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044145454545454544, + "loss": 2.7255, + "theoretical_loss": 3.4501249526241295, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044143434343434345, + "loss": 2.7661, + "theoretical_loss": 3.450114723977573, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1056292, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8757920265197754, + "objective/train/theoretical_loss": 3.4501044957903506, + "objective/train/tokens_used": 226897376, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414141414141414, + "loss": 2.8258, + "theoretical_loss": 3.4501044957903506, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413939393939394, + "loss": 2.5358, + "theoretical_loss": 3.450094268062424, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413737373737374, + "loss": 2.6981, + "theoretical_loss": 3.4500840407937567, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044135353535353533, + "loss": 2.8689, + "theoretical_loss": 3.450073813984313, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044133333333333335, + "loss": 2.6005, + "theoretical_loss": 3.4500635876340553, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044131313131313136, + "loss": 2.7218, + "theoretical_loss": 3.450053361742947, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412929292929293, + "loss": 2.6695, + "theoretical_loss": 3.450043136310952, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044127272727272727, + "loss": 2.8102, + "theoretical_loss": 3.450032911338032, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412525252525252, + "loss": 2.9097, + "theoretical_loss": 3.4500226868241524, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123232323232324, + "loss": 2.6097, + "theoretical_loss": 3.450012462769275, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121212121212125, + "loss": 2.6656, + "theoretical_loss": 3.450002239173364, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411919191919192, + "loss": 2.6725, + "theoretical_loss": 3.449992016036382, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117171717171716, + "loss": 2.7881, + "theoretical_loss": 3.449981793358292, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411515151515151, + "loss": 2.7396, + "theoretical_loss": 3.4499715711390584, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411313131313132, + "loss": 2.8035, + "theoretical_loss": 3.4499613493786443, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044111111111111114, + "loss": 2.75, + "theoretical_loss": 3.4499511280770125, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410909090909091, + "loss": 2.8295, + "theoretical_loss": 3.4499409072341263, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107070707070706, + "loss": 2.5611, + "theoretical_loss": 3.4499306868499495, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441050505050505, + "loss": 2.7045, + "theoretical_loss": 3.449920466924445, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410303030303031, + "loss": 2.6331, + "theoretical_loss": 3.4499102474575762, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101010101010103, + "loss": 2.514, + "theoretical_loss": 3.4499000284493073, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440989898989899, + "loss": 2.832, + "theoretical_loss": 3.4498898098996, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044096969696969695, + "loss": 2.7497, + "theoretical_loss": 3.4498795918084193, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094949494949496, + "loss": 2.5607, + "theoretical_loss": 3.4498693741757274, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092929292929297, + "loss": 2.7198, + "theoretical_loss": 3.4498591570014883, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1057648, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.909499406814575, + "objective/train/theoretical_loss": 3.4498489402856647, + "objective/train/tokens_used": 228535776, + "theoretical_loss": 3.4498489402856647, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044090909090909093, + "loss": 2.8555, + "theoretical_loss": 3.4498489402856647, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408888888888889, + "loss": 2.5471, + "theoretical_loss": 3.4498387240282207, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086868686868684, + "loss": 2.7897, + "theoretical_loss": 3.4498285082291194, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044084848484848485, + "loss": 2.8596, + "theoretical_loss": 3.449818292888324, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044082828282828286, + "loss": 2.7032, + "theoretical_loss": 3.4498080780057983, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408080808080808, + "loss": 2.6185, + "theoretical_loss": 3.449797863581505, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407878787878788, + "loss": 2.7349, + "theoretical_loss": 3.4497876496154083, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407676767676768, + "loss": 2.6853, + "theoretical_loss": 3.4497774361074707, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044074747474747474, + "loss": 2.544, + "theoretical_loss": 3.4497672230576564, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044072727272727276, + "loss": 2.6251, + "theoretical_loss": 3.4497570104659285, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407070707070707, + "loss": 2.867, + "theoretical_loss": 3.4497467983322503, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068686868686867, + "loss": 2.6809, + "theoretical_loss": 3.4497365866565852, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406666666666667, + "loss": 2.6492, + "theoretical_loss": 3.4497263754388965, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064646464646464, + "loss": 2.5597, + "theoretical_loss": 3.449716164679148, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044062626262626265, + "loss": 2.8717, + "theoretical_loss": 3.449705954377303, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406060606060606, + "loss": 2.6648, + "theoretical_loss": 3.449695744533325, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405858585858586, + "loss": 2.6419, + "theoretical_loss": 3.449685535147177, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405656565656566, + "loss": 2.6933, + "theoretical_loss": 3.4496753262188227, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044054545454545453, + "loss": 2.8096, + "theoretical_loss": 3.4496651177482254, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052525252525254, + "loss": 2.619, + "theoretical_loss": 3.449654909735349, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405050505050505, + "loss": 2.5052, + "theoretical_loss": 3.449644702180156, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404848484848485, + "loss": 2.7652, + "theoretical_loss": 3.4496344950826114, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044046464646464647, + "loss": 2.6558, + "theoretical_loss": 3.4496242884426773, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404444444444445, + "loss": 2.5056, + "theoretical_loss": 3.449614082260317, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042424242424243, + "loss": 2.6324, + "theoretical_loss": 3.4496038765354955, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1058498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.454388380050659, + "objective/train/theoretical_loss": 3.449593671268175, + "objective/train/tokens_used": 230174176, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044040404040404044, + "loss": 2.4882, + "theoretical_loss": 3.449593671268175, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403838383838384, + "loss": 2.7474, + "theoretical_loss": 3.449583466458319, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044036363636363636, + "loss": 2.5742, + "theoretical_loss": 3.4495732621058917, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034343434343437, + "loss": 2.7881, + "theoretical_loss": 3.449563058210856, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403232323232323, + "loss": 2.5764, + "theoretical_loss": 3.4495528547731755, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044030303030303034, + "loss": 2.6237, + "theoretical_loss": 3.4495426517928136, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402828282828283, + "loss": 2.6606, + "theoretical_loss": 3.449532449269734, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026262626262625, + "loss": 2.5386, + "theoretical_loss": 3.4495222472039, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044024242424242426, + "loss": 2.5901, + "theoretical_loss": 3.449512045595275, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402222222222222, + "loss": 2.7852, + "theoretical_loss": 3.449501844443823, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044020202020202023, + "loss": 2.5328, + "theoretical_loss": 3.449491643749507, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401818181818182, + "loss": 2.7228, + "theoretical_loss": 3.449481443512291, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044016161616161614, + "loss": 2.7211, + "theoretical_loss": 3.449471243732138, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014141414141416, + "loss": 2.8435, + "theoretical_loss": 3.449461044409012, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044012121212121217, + "loss": 2.8524, + "theoretical_loss": 3.449450845542876, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401010101010101, + "loss": 2.8406, + "theoretical_loss": 3.4494406471336934, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400808080808081, + "loss": 2.7776, + "theoretical_loss": 3.4494304491814285, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044006060606060604, + "loss": 2.8116, + "theoretical_loss": 3.449420251686045, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004040404040405, + "loss": 2.8137, + "theoretical_loss": 3.4494100546475055, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002020202020206, + "loss": 2.9223, + "theoretical_loss": 3.4493998580657736, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 2.6325, + "theoretical_loss": 3.4493896619408138, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043997979797979797, + "loss": 2.5671, + "theoretical_loss": 3.449379466272589, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995959595959593, + "loss": 2.857, + "theoretical_loss": 3.4493692710610624, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439939393939394, + "loss": 2.7492, + "theoretical_loss": 3.449359076306198, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043991919191919195, + "loss": 2.7919, + "theoretical_loss": 3.4493488820079596, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1059136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6362321376800537, + "objective/train/theoretical_loss": 3.4493386881663106, + "objective/train/tokens_used": 231812576, + "theoretical_loss": 3.4493386881663106, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398989898989899, + "loss": 2.7305, + "theoretical_loss": 3.4493386881663106, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987878787878787, + "loss": 2.5699, + "theoretical_loss": 3.4493284947812146, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398585858585858, + "loss": 2.7299, + "theoretical_loss": 3.4493183018526348, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398383838383839, + "loss": 2.4749, + "theoretical_loss": 3.449308109380535, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043981818181818184, + "loss": 2.8044, + "theoretical_loss": 3.449297917364879, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397979797979798, + "loss": 2.7926, + "theoretical_loss": 3.44928772580563, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043977777777777776, + "loss": 2.6683, + "theoretical_loss": 3.449277534702752, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043975757575757577, + "loss": 2.812, + "theoretical_loss": 3.4492673440562083, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397373737373738, + "loss": 2.6694, + "theoretical_loss": 3.449257153865963, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971717171717174, + "loss": 2.7785, + "theoretical_loss": 3.449246964131979, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396969696969697, + "loss": 2.761, + "theoretical_loss": 3.4492367748542203, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967676767676765, + "loss": 2.8328, + "theoretical_loss": 3.4492265860326508, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043965656565656566, + "loss": 2.8822, + "theoretical_loss": 3.449216397667233, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963636363636367, + "loss": 2.6526, + "theoretical_loss": 3.449206209757932, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043961616161616163, + "loss": 2.5556, + "theoretical_loss": 3.4491960223047107, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395959595959596, + "loss": 2.5222, + "theoretical_loss": 3.4491858353075324, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395757575757576, + "loss": 2.637, + "theoretical_loss": 3.449175648766362, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955555555555555, + "loss": 2.6937, + "theoretical_loss": 3.449165462681161, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043953535353535357, + "loss": 2.7608, + "theoretical_loss": 3.4491552770518954, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395151515151515, + "loss": 2.7465, + "theoretical_loss": 3.449145091878527, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394949494949495, + "loss": 2.661, + "theoretical_loss": 3.4491349071610204, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394747474747475, + "loss": 2.6579, + "theoretical_loss": 3.449124722899339, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043945454545454545, + "loss": 2.6578, + "theoretical_loss": 3.449114539093447, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043943434343434346, + "loss": 2.5179, + "theoretical_loss": 3.449104355743307, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394141414141414, + "loss": 2.5996, + "theoretical_loss": 3.4490941728488838, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1059915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5282862186431885, + "objective/train/theoretical_loss": 3.44908399041014, + "objective/train/tokens_used": 233450976, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393939393939394, + "loss": 2.7642, + "theoretical_loss": 3.44908399041014, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393737373737374, + "loss": 2.4739, + "theoretical_loss": 3.4490738084270403, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043935353535353534, + "loss": 2.6488, + "theoretical_loss": 3.4490636268995476, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933333333333335, + "loss": 2.6753, + "theoretical_loss": 3.4490534458276256, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393131313131313, + "loss": 2.8208, + "theoretical_loss": 3.449043265211239, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392929292929293, + "loss": 2.7899, + "theoretical_loss": 3.4490330850503503, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392727272727273, + "loss": 2.6833, + "theoretical_loss": 3.4490229053449237, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925252525252523, + "loss": 2.5962, + "theoretical_loss": 3.449012726094923, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043923232323232324, + "loss": 2.7983, + "theoretical_loss": 3.449002547300312, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921212121212125, + "loss": 2.833, + "theoretical_loss": 3.4489923689610538, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391919191919192, + "loss": 2.6405, + "theoretical_loss": 3.4489821910771123, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043917171717171717, + "loss": 2.6869, + "theoretical_loss": 3.448972013648452, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391515151515151, + "loss": 2.6155, + "theoretical_loss": 3.4489618366750356, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043913131313131314, + "loss": 2.8284, + "theoretical_loss": 3.448951660156828, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911111111111115, + "loss": 2.7089, + "theoretical_loss": 3.4489414840937913, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390909090909091, + "loss": 2.7488, + "theoretical_loss": 3.448931308485891, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043907070707070706, + "loss": 3.0305, + "theoretical_loss": 3.4489211333330894, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439050505050505, + "loss": 2.6922, + "theoretical_loss": 3.4489109586353512, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390303030303031, + "loss": 2.5978, + "theoretical_loss": 3.44890078439264, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043901010101010104, + "loss": 2.5284, + "theoretical_loss": 3.448890610604918, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438989898989899, + "loss": 2.6912, + "theoretical_loss": 3.4488804372721518, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043896969696969695, + "loss": 2.8593, + "theoretical_loss": 3.448870264394303, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389494949494949, + "loss": 2.9581, + "theoretical_loss": 3.448860091971336, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438929292929293, + "loss": 2.7545, + "theoretical_loss": 3.448849920003215, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890909090909093, + "loss": 2.6821, + "theoretical_loss": 3.4488397484899034, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1060624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.365621566772461, + "objective/train/theoretical_loss": 3.448829577431365, + "objective/train/tokens_used": 235089376, + "theoretical_loss": 3.448829577431365, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388888888888889, + "loss": 2.656, + "theoretical_loss": 3.448829577431365, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043886868686868685, + "loss": 2.8536, + "theoretical_loss": 3.448819406827563, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388484848484848, + "loss": 2.6756, + "theoretical_loss": 3.448809236678462, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882828282828287, + "loss": 2.5616, + "theoretical_loss": 3.4487990669840256, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388080808080808, + "loss": 2.7942, + "theoretical_loss": 3.448788897744217, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387878787878788, + "loss": 2.5658, + "theoretical_loss": 3.4487787289590015, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876767676767674, + "loss": 2.6889, + "theoretical_loss": 3.4487685606283414, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387474747474748, + "loss": 2.9163, + "theoretical_loss": 3.4487583927522008, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872727272727276, + "loss": 2.8597, + "theoretical_loss": 3.4487482253305437, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387070707070707, + "loss": 2.7193, + "theoretical_loss": 3.4487380583633342, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386868686868687, + "loss": 2.8507, + "theoretical_loss": 3.4487278918505364, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043866666666666663, + "loss": 2.7237, + "theoretical_loss": 3.448717725792113, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386464646464647, + "loss": 2.7634, + "theoretical_loss": 3.448707560188028, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043862626262626265, + "loss": 2.785, + "theoretical_loss": 3.4486973950382467, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386060606060606, + "loss": 2.8518, + "theoretical_loss": 3.448687230342731, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043858585858585857, + "loss": 2.6201, + "theoretical_loss": 3.448677066101446, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656565656566, + "loss": 2.5509, + "theoretical_loss": 3.448666902314355, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385454545454546, + "loss": 2.8245, + "theoretical_loss": 3.4486567389814224, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852525252525255, + "loss": 2.8998, + "theoretical_loss": 3.4486465761026115, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385050505050505, + "loss": 2.6801, + "theoretical_loss": 3.448636413677886, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043848484848484846, + "loss": 2.6001, + "theoretical_loss": 3.4486262517072106, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043846464646464647, + "loss": 2.734, + "theoretical_loss": 3.4486160901905487, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384444444444445, + "loss": 2.5, + "theoretical_loss": 3.448605929127864, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043842424242424244, + "loss": 2.7242, + "theoretical_loss": 3.4485957685191204, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384040404040404, + "loss": 2.7524, + "theoretical_loss": 3.448585608364282, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1062058, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4810831546783447, + "objective/train/theoretical_loss": 3.4485754486633127, + "objective/train/tokens_used": 236727776, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383838383838384, + "loss": 2.5655, + "theoretical_loss": 3.4485754486633127, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043836363636363636, + "loss": 2.7271, + "theoretical_loss": 3.448565289416176, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383434343434344, + "loss": 2.6832, + "theoretical_loss": 3.448555130622836, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832323232323233, + "loss": 2.7368, + "theoretical_loss": 3.4485449722832566, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383030303030303, + "loss": 2.5261, + "theoretical_loss": 3.4485348143974024, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382828282828283, + "loss": 2.5966, + "theoretical_loss": 3.448524656965236, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043826262626262626, + "loss": 2.9392, + "theoretical_loss": 3.448514499986722, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824242424242427, + "loss": 2.7046, + "theoretical_loss": 3.448504343461824, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382222222222222, + "loss": 2.6558, + "theoretical_loss": 3.448494187390507, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820202020202024, + "loss": 2.4992, + "theoretical_loss": 3.448484031772734, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381818181818182, + "loss": 2.7807, + "theoretical_loss": 3.4484738766084684, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816161616161615, + "loss": 2.9591, + "theoretical_loss": 3.4484637218976752, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043814141414141416, + "loss": 2.82, + "theoretical_loss": 3.4484535676403176, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381212121212121, + "loss": 2.7795, + "theoretical_loss": 3.44844341383636, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043810101010101013, + "loss": 2.7318, + "theoretical_loss": 3.448433260485766, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380808080808081, + "loss": 2.7346, + "theoretical_loss": 3.4484231075885, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043806060606060604, + "loss": 2.849, + "theoretical_loss": 3.4484129551445255, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043804040404040405, + "loss": 2.6799, + "theoretical_loss": 3.4484028031538063, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802020202020206, + "loss": 2.8663, + "theoretical_loss": 3.4483926516163073, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438, + "loss": 2.7284, + "theoretical_loss": 3.4483825005319915, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437979797979798, + "loss": 2.6133, + "theoretical_loss": 3.448372349900823, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795959595959593, + "loss": 2.6673, + "theoretical_loss": 3.448362199722766, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793939393939395, + "loss": 2.7643, + "theoretical_loss": 3.4483520499977844, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043791919191919196, + "loss": 2.7865, + "theoretical_loss": 3.4483419007258425, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378989898989899, + "loss": 2.5513, + "theoretical_loss": 3.4483317519069043, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1062685, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8123557567596436, + "objective/train/theoretical_loss": 3.4483216035409328, + "objective/train/tokens_used": 238366176, + "theoretical_loss": 3.4483216035409328, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787878787878787, + "loss": 2.7055, + "theoretical_loss": 3.4483216035409328, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785858585858583, + "loss": 2.5502, + "theoretical_loss": 3.4483114556278927, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378383838383839, + "loss": 2.8682, + "theoretical_loss": 3.448301308167748, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781818181818185, + "loss": 2.9879, + "theoretical_loss": 3.448291161160463, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377979797979798, + "loss": 2.6719, + "theoretical_loss": 3.4482810146060014, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777777777777776, + "loss": 2.9228, + "theoretical_loss": 3.4482708685043266, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377575757575757, + "loss": 2.6664, + "theoretical_loss": 3.4482607228554034, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377373737373738, + "loss": 2.7722, + "theoretical_loss": 3.448250577659196, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043771717171717174, + "loss": 2.6311, + "theoretical_loss": 3.448240432915667, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376969696969697, + "loss": 2.6578, + "theoretical_loss": 3.4482302886247824, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767676767676766, + "loss": 2.7735, + "theoretical_loss": 3.448220144786505, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043765656565656567, + "loss": 2.7577, + "theoretical_loss": 3.448210001400799, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376363636363637, + "loss": 2.576, + "theoretical_loss": 3.4481998584676288, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043761616161616164, + "loss": 2.7648, + "theoretical_loss": 3.448189715986958, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375959595959596, + "loss": 2.7249, + "theoretical_loss": 3.4481795739587504, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043757575757575755, + "loss": 2.6364, + "theoretical_loss": 3.448169432382971, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043755555555555556, + "loss": 2.8826, + "theoretical_loss": 3.448159291259583, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043753535353535357, + "loss": 2.8159, + "theoretical_loss": 3.448149150588551, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751515151515153, + "loss": 2.7824, + "theoretical_loss": 3.4481390103698386, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374949494949495, + "loss": 2.6268, + "theoretical_loss": 3.44812887060341, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374747474747475, + "loss": 2.7807, + "theoretical_loss": 3.448118731289229, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745454545454545, + "loss": 2.6639, + "theoretical_loss": 3.448108592427261, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743434343434346, + "loss": 2.7484, + "theoretical_loss": 3.448098454017469, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374141414141414, + "loss": 2.7943, + "theoretical_loss": 3.4480883160598164, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373939393939394, + "loss": 2.8943, + "theoretical_loss": 3.448078178554268, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1064316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0120768547058105, + "objective/train/theoretical_loss": 3.4480680415007887, + "objective/train/tokens_used": 240004576, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373737373737374, + "loss": 2.6438, + "theoretical_loss": 3.4480680415007887, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735353535353535, + "loss": 2.6145, + "theoretical_loss": 3.4480579048993416, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733333333333336, + "loss": 2.5821, + "theoretical_loss": 3.448047768749891, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373131313131313, + "loss": 2.8976, + "theoretical_loss": 3.448037633052401, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372929292929293, + "loss": 2.7997, + "theoretical_loss": 3.4480274978068355, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372727272727273, + "loss": 2.5558, + "theoretical_loss": 3.448017363013159, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725252525252524, + "loss": 2.6181, + "theoretical_loss": 3.4480072286713357, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043723232323232325, + "loss": 2.5932, + "theoretical_loss": 3.4479970947813294, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372121212121212, + "loss": 2.7511, + "theoretical_loss": 3.447986961343104, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371919191919192, + "loss": 2.6566, + "theoretical_loss": 3.4479768283566243, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371717171717172, + "loss": 2.8153, + "theoretical_loss": 3.447966695821854, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043715151515151513, + "loss": 2.8218, + "theoretical_loss": 3.447956563738757, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713131313131314, + "loss": 2.9349, + "theoretical_loss": 3.4479464321072983, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043711111111111115, + "loss": 2.8099, + "theoretical_loss": 3.4479363009274406, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370909090909091, + "loss": 2.6767, + "theoretical_loss": 3.4479261701991497, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043707070707070707, + "loss": 2.5844, + "theoretical_loss": 3.4479160399223887, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370505050505051, + "loss": 2.6705, + "theoretical_loss": 3.4479059100971217, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043703030303030303, + "loss": 2.6515, + "theoretical_loss": 3.4478957807233135, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043701010101010105, + "loss": 2.7583, + "theoretical_loss": 3.447885651800928, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436989898989899, + "loss": 2.6205, + "theoretical_loss": 3.4478755233299294, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043696969696969696, + "loss": 2.664, + "theoretical_loss": 3.4478653953102816, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043694949494949497, + "loss": 2.4867, + "theoretical_loss": 3.4478552677419487, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043692929292929293, + "loss": 2.6708, + "theoretical_loss": 3.4478451406248953, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043690909090909094, + "loss": 2.8433, + "theoretical_loss": 3.4478350139590854, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004368888888888889, + "loss": 2.6959, + "theoretical_loss": 3.447824887744483, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 1064917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0615880489349365, + "objective/train/theoretical_loss": 3.447814761981053, + "objective/train/tokens_used": 241642976, + "theoretical_loss": 3.447814761981053, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686868686868685, + "loss": 2.8483, + "theoretical_loss": 3.447814761981053, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684848484848486, + "loss": 2.6425, + "theoretical_loss": 3.447804636668759, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368282828282829, + "loss": 2.7452, + "theoretical_loss": 3.4477945118075652, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680808080808083, + "loss": 2.6797, + "theoretical_loss": 3.4477843873974354, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367878787878788, + "loss": 2.5593, + "theoretical_loss": 3.447774263438335, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676767676767674, + "loss": 2.4923, + "theoretical_loss": 3.447764139930227, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043674747474747476, + "loss": 2.7182, + "theoretical_loss": 3.4477540168730765, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672727272727277, + "loss": 2.3769, + "theoretical_loss": 3.4477438942668472, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367070707070707, + "loss": 2.6939, + "theoretical_loss": 3.447733772111503, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366868686868687, + "loss": 2.6878, + "theoretical_loss": 3.447723650407009, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043666666666666664, + "loss": 2.5994, + "theoretical_loss": 3.447713529153329, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366464646464647, + "loss": 2.6525, + "theoretical_loss": 3.447703408350427, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662626262626266, + "loss": 2.7098, + "theoretical_loss": 3.447693287998268, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366060606060606, + "loss": 2.6988, + "theoretical_loss": 3.447683168096815, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365858585858586, + "loss": 2.7252, + "theoretical_loss": 3.447673048646034, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656565656565653, + "loss": 2.7116, + "theoretical_loss": 3.4476629296458876, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365454545454546, + "loss": 2.5895, + "theoretical_loss": 3.447652811096341, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043652525252525255, + "loss": 2.737, + "theoretical_loss": 3.447642692997358, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365050505050505, + "loss": 2.7158, + "theoretical_loss": 3.4476325753489028, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648484848484847, + "loss": 2.7265, + "theoretical_loss": 3.4476224581509403, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364646464646465, + "loss": 2.8233, + "theoretical_loss": 3.447612341403434, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364444444444445, + "loss": 2.7179, + "theoretical_loss": 3.4476022251063485, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642424242424244, + "loss": 2.4931, + "theoretical_loss": 3.447592109259648, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364040404040404, + "loss": 2.6365, + "theoretical_loss": 3.447581993863297, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043638383838383836, + "loss": 2.6176, + "theoretical_loss": 3.44757187891726, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1065907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8918585777282715, + "objective/train/theoretical_loss": 3.4475617644215006, + "objective/train/tokens_used": 243281376, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043636363636363637, + "loss": 2.5669, + "theoretical_loss": 3.4475617644215006, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363434343434344, + "loss": 2.4341, + "theoretical_loss": 3.447551650375984, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632323232323234, + "loss": 2.6919, + "theoretical_loss": 3.4475415367806734, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363030303030303, + "loss": 2.751, + "theoretical_loss": 3.4475314236355343, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362828282828283, + "loss": 2.942, + "theoretical_loss": 3.44752131094053, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043626262626262626, + "loss": 2.8027, + "theoretical_loss": 3.447511198695625, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362424242424243, + "loss": 2.7431, + "theoretical_loss": 3.4475010869007843, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043622222222222223, + "loss": 2.6807, + "theoretical_loss": 3.4474909755559713, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362020202020202, + "loss": 2.7703, + "theoretical_loss": 3.4474808646611512, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361818181818182, + "loss": 2.5818, + "theoretical_loss": 3.4474707542162877, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043616161616161616, + "loss": 2.5474, + "theoretical_loss": 3.447460644221345, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614141414141417, + "loss": 2.8644, + "theoretical_loss": 3.447450534676288, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361212121212121, + "loss": 2.7175, + "theoretical_loss": 3.447440425581081, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043610101010101013, + "loss": 2.8522, + "theoretical_loss": 3.447430316935688, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360808080808081, + "loss": 2.5067, + "theoretical_loss": 3.4474202087400734, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606060606060605, + "loss": 2.8428, + "theoretical_loss": 3.447410100994202, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043604040404040406, + "loss": 2.7062, + "theoretical_loss": 3.4473999936980375, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436020202020202, + "loss": 2.7189, + "theoretical_loss": 3.4473898868515445, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436, + "loss": 2.5164, + "theoretical_loss": 3.447379780454688, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435979797979798, + "loss": 2.9322, + "theoretical_loss": 3.4473696745074314, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043595959595959594, + "loss": 2.9066, + "theoretical_loss": 3.4473595690097394, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593939393939395, + "loss": 2.7151, + "theoretical_loss": 3.4473494639615767, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591919191919196, + "loss": 2.8117, + "theoretical_loss": 3.447339359362907, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358989898989899, + "loss": 2.8175, + "theoretical_loss": 3.447329255213696, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358787878787879, + "loss": 2.6957, + "theoretical_loss": 3.4473191515139066, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1066425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.749819278717041, + "objective/train/theoretical_loss": 3.447309048263504, + "objective/train/tokens_used": 244919776, + "theoretical_loss": 3.447309048263504, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043585858585858583, + "loss": 2.6145, + "theoretical_loss": 3.447309048263504, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583838383838384, + "loss": 2.4759, + "theoretical_loss": 3.4472989454624523, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043581818181818186, + "loss": 2.7213, + "theoretical_loss": 3.447288843110716, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357979797979798, + "loss": 2.8457, + "theoretical_loss": 3.4472787412082595, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043577777777777777, + "loss": 2.7856, + "theoretical_loss": 3.4472686397550474, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357575757575757, + "loss": 2.4388, + "theoretical_loss": 3.4472585387510435, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357373737373738, + "loss": 2.612, + "theoretical_loss": 3.447248438196213, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571717171717175, + "loss": 2.7182, + "theoretical_loss": 3.4472383380905205, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356969696969697, + "loss": 2.6214, + "theoretical_loss": 3.4472282384339294, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567676767676766, + "loss": 2.5399, + "theoretical_loss": 3.4472181392264045, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356565656565656, + "loss": 2.5266, + "theoretical_loss": 3.4472080404679106, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356363636363637, + "loss": 2.6301, + "theoretical_loss": 3.447197942158412, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561616161616164, + "loss": 2.7636, + "theoretical_loss": 3.447187844297873, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355959595959596, + "loss": 2.8787, + "theoretical_loss": 3.447177746886258, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043557575757575755, + "loss": 2.5692, + "theoretical_loss": 3.4471676499235318, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355555555555555, + "loss": 2.7474, + "theoretical_loss": 3.4471575534096583, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355353535353536, + "loss": 2.6425, + "theoretical_loss": 3.4471474573446024, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551515151515153, + "loss": 2.7879, + "theoretical_loss": 3.4471373617283287, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354949494949495, + "loss": 2.6096, + "theoretical_loss": 3.447127266560801, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043547474747474745, + "loss": 2.6842, + "theoretical_loss": 3.4471171718419846, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545454545454546, + "loss": 2.6711, + "theoretical_loss": 3.447107077571843, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043543434343434347, + "loss": 2.6342, + "theoretical_loss": 3.447096983750342, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354141414141414, + "loss": 2.6866, + "theoretical_loss": 3.447086890377445, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353939393939394, + "loss": 2.7505, + "theoretical_loss": 3.447076797453117, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043537373737373734, + "loss": 3.0324, + "theoretical_loss": 3.4470667049773214, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1067085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5449960231781006, + "objective/train/theoretical_loss": 3.4470566129500244, + "objective/train/tokens_used": 246558176, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535353535353535, + "loss": 2.6128, + "theoretical_loss": 3.4470566129500244, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533333333333336, + "loss": 2.6959, + "theoretical_loss": 3.4470465213711896, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353131313131313, + "loss": 2.7237, + "theoretical_loss": 3.4470364302407814, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352929292929293, + "loss": 2.6424, + "theoretical_loss": 3.447026339558765, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352727272727273, + "loss": 2.6558, + "theoretical_loss": 3.4470162493251038, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352525252525253, + "loss": 2.6625, + "theoretical_loss": 3.4470061595397627, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523232323232325, + "loss": 2.9305, + "theoretical_loss": 3.446996070202707, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352121212121212, + "loss": 2.7227, + "theoretical_loss": 3.446985981313901, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519191919191917, + "loss": 2.7155, + "theoretical_loss": 3.4469758928733083, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351717171717172, + "loss": 2.66, + "theoretical_loss": 3.446965804880894, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351515151515152, + "loss": 2.4458, + "theoretical_loss": 3.446955717336623, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043513131313131315, + "loss": 2.6543, + "theoretical_loss": 3.4469456302404597, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351111111111111, + "loss": 2.9044, + "theoretical_loss": 3.4469355435923683, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350909090909091, + "loss": 2.991, + "theoretical_loss": 3.4469254573923136, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043507070707070707, + "loss": 2.7941, + "theoretical_loss": 3.4469153716402596, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350505050505051, + "loss": 2.9618, + "theoretical_loss": 3.4469052863361713, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503030303030304, + "loss": 2.6343, + "theoretical_loss": 3.4468952014800136, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435010101010101, + "loss": 2.6961, + "theoretical_loss": 3.4468851170717505, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434989898989899, + "loss": 2.6708, + "theoretical_loss": 3.446875033111347, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496969696969696, + "loss": 2.7318, + "theoretical_loss": 3.4468649495987678, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434949494949495, + "loss": 2.628, + "theoretical_loss": 3.4468548665339767, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043492929292929293, + "loss": 2.6674, + "theoretical_loss": 3.446844783916939, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043490909090909094, + "loss": 2.7995, + "theoretical_loss": 3.4468347017476186, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348888888888889, + "loss": 2.7184, + "theoretical_loss": 3.446824620025981, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043486868686868686, + "loss": 2.609, + "theoretical_loss": 3.44681453875199, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1068281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.476545810699463, + "objective/train/theoretical_loss": 3.4468044579256105, + "objective/train/tokens_used": 248196576, + "theoretical_loss": 3.4468044579256105, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043484848484848487, + "loss": 2.8055, + "theoretical_loss": 3.4468044579256105, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348282828282828, + "loss": 2.7732, + "theoretical_loss": 3.446794377546807, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480808080808084, + "loss": 2.7115, + "theoretical_loss": 3.4467842976155447, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347878787878788, + "loss": 2.9332, + "theoretical_loss": 3.4467742181317873, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476767676767675, + "loss": 2.9765, + "theoretical_loss": 3.4467641390955, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043474747474747476, + "loss": 2.78, + "theoretical_loss": 3.446754060506647, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472727272727277, + "loss": 2.9347, + "theoretical_loss": 3.4467439823651933, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043470707070707073, + "loss": 2.6642, + "theoretical_loss": 3.4467339046711034, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346868686868687, + "loss": 2.7112, + "theoretical_loss": 3.446723827424342, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466666666666664, + "loss": 2.7962, + "theoretical_loss": 3.4467137506248733, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043464646464646465, + "loss": 2.7397, + "theoretical_loss": 3.446703674272663, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462626262626267, + "loss": 2.7429, + "theoretical_loss": 3.4466935983676743, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346060606060606, + "loss": 2.8701, + "theoretical_loss": 3.446683522909873, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345858585858586, + "loss": 2.7257, + "theoretical_loss": 3.446673447899223, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043456565656565654, + "loss": 2.764, + "theoretical_loss": 3.4466633733356895, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345454545454546, + "loss": 2.7603, + "theoretical_loss": 3.4466532992192365, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452525252525256, + "loss": 2.577, + "theoretical_loss": 3.4466432255498294, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345050505050505, + "loss": 2.5705, + "theoretical_loss": 3.4466331523274327, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043448484848484847, + "loss": 2.7355, + "theoretical_loss": 3.446623079552011, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446464646464643, + "loss": 2.7225, + "theoretical_loss": 3.4466130072235286, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344444444444445, + "loss": 2.911, + "theoretical_loss": 3.4466029353419505, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043442424242424245, + "loss": 2.7512, + "theoretical_loss": 3.4465928639072416, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344040404040404, + "loss": 2.8163, + "theoretical_loss": 3.446582792919366, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043438383838383836, + "loss": 2.7919, + "theoretical_loss": 3.446572722378289, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343636363636364, + "loss": 2.5169, + "theoretical_loss": 3.4465626522839745, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1069021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.568247079849243, + "objective/train/theoretical_loss": 3.4465525826363885, + "objective/train/tokens_used": 249834976, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343434343434344, + "loss": 2.731, + "theoretical_loss": 3.4465525826363885, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432323232323234, + "loss": 2.6317, + "theoretical_loss": 3.4465425134354946, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343030303030303, + "loss": 2.3406, + "theoretical_loss": 3.4465324446812575, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428282828282826, + "loss": 2.584, + "theoretical_loss": 3.4465223763736423, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043426262626262627, + "loss": 2.6658, + "theoretical_loss": 3.446512308512614, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342424242424243, + "loss": 2.9166, + "theoretical_loss": 3.446502241098137, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043422222222222224, + "loss": 2.8404, + "theoretical_loss": 3.4464921741301757, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342020202020202, + "loss": 2.8458, + "theoretical_loss": 3.4464821076086953, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341818181818182, + "loss": 2.6229, + "theoretical_loss": 3.44647204153366, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416161616161616, + "loss": 2.3028, + "theoretical_loss": 3.4464619759050352, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414141414141417, + "loss": 2.6855, + "theoretical_loss": 3.4464519107227853, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412121212121213, + "loss": 2.642, + "theoretical_loss": 3.4464418459868753, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341010101010101, + "loss": 2.825, + "theoretical_loss": 3.446431781697269, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340808080808081, + "loss": 2.5787, + "theoretical_loss": 3.4464217178539327, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406060606060605, + "loss": 2.4574, + "theoretical_loss": 3.4464116544568295, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404040404040406, + "loss": 2.6793, + "theoretical_loss": 3.446401591505925, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434020202020202, + "loss": 2.8695, + "theoretical_loss": 3.4463915290011844, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043400000000000003, + "loss": 2.6604, + "theoretical_loss": 3.446381466942572, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433979797979798, + "loss": 2.5884, + "theoretical_loss": 3.4463714053300523, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043395959595959595, + "loss": 2.6641, + "theoretical_loss": 3.44636134416359, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043393939393939396, + "loss": 2.6219, + "theoretical_loss": 3.4463512834431507, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339191919191919, + "loss": 2.5315, + "theoretical_loss": 3.446341223168698, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338989898989899, + "loss": 2.6097, + "theoretical_loss": 3.446331163340198, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338787878787879, + "loss": 2.6247, + "theoretical_loss": 3.446321103957615, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043385858585858584, + "loss": 2.5369, + "theoretical_loss": 3.4463110450209133, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1070317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.482539653778076, + "objective/train/theoretical_loss": 3.4463009865300585, + "objective/train/tokens_used": 251473376, + "theoretical_loss": 3.4463009865300585, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383838383838385, + "loss": 2.6317, + "theoretical_loss": 3.4463009865300585, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043381818181818186, + "loss": 2.6243, + "theoretical_loss": 3.4462909284850145, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337979797979798, + "loss": 2.7515, + "theoretical_loss": 3.4462808708857464, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337777777777778, + "loss": 2.6561, + "theoretical_loss": 3.4462708137322196, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375757575757573, + "loss": 2.8158, + "theoretical_loss": 3.446260757024398, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043373737373737374, + "loss": 2.7285, + "theoretical_loss": 3.4462507007622474, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371717171717175, + "loss": 2.7233, + "theoretical_loss": 3.4462406449457315, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336969696969697, + "loss": 2.7187, + "theoretical_loss": 3.4462305895748164, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367676767676767, + "loss": 2.5958, + "theoretical_loss": 3.4462205346494654, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336565656565656, + "loss": 2.8754, + "theoretical_loss": 3.446210480169645, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363636363636364, + "loss": 2.5836, + "theoretical_loss": 3.446200426135319, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043361616161616165, + "loss": 2.8075, + "theoretical_loss": 3.4461903725464524, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335959595959596, + "loss": 2.4467, + "theoretical_loss": 3.4461803194030103, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357575757575756, + "loss": 2.7178, + "theoretical_loss": 3.446170266704957, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355555555555557, + "loss": 2.6568, + "theoretical_loss": 3.446160214452258, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335353535353536, + "loss": 2.8153, + "theoretical_loss": 3.4461501626448783, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043351515151515154, + "loss": 2.7827, + "theoretical_loss": 3.446140111282782, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334949494949495, + "loss": 2.92, + "theoretical_loss": 3.4461300603659337, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043347474747474745, + "loss": 2.9315, + "theoretical_loss": 3.4461200098942997, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043345454545454546, + "loss": 2.5178, + "theoretical_loss": 3.4461099598678437, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334343434343435, + "loss": 2.9703, + "theoretical_loss": 3.446099910286531, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341414141414143, + "loss": 2.9402, + "theoretical_loss": 3.446089861150326, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333939393939394, + "loss": 2.6976, + "theoretical_loss": 3.4460798124591947, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043337373737373735, + "loss": 2.6612, + "theoretical_loss": 3.446069764213101, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333535353535354, + "loss": 2.5298, + "theoretical_loss": 3.4460597164120097, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1070823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4670941829681396, + "objective/train/theoretical_loss": 3.4460496690558866, + "objective/train/tokens_used": 253111776, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043333333333333337, + "loss": 2.5597, + "theoretical_loss": 3.4460496690558866, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333131313131313, + "loss": 2.59, + "theoretical_loss": 3.4460396221446956, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332929292929293, + "loss": 2.65, + "theoretical_loss": 3.4460295756784025, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327272727272724, + "loss": 2.5597, + "theoretical_loss": 3.446019529656972, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332525252525253, + "loss": 2.7512, + "theoretical_loss": 3.446009484080368, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323232323232326, + "loss": 2.7317, + "theoretical_loss": 3.445999438948557, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332121212121212, + "loss": 2.8081, + "theoretical_loss": 3.4459893942615025, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331919191919192, + "loss": 2.5551, + "theoretical_loss": 3.4459793500191704, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331717171717172, + "loss": 2.8269, + "theoretical_loss": 3.4459693062215253, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331515151515152, + "loss": 2.7809, + "theoretical_loss": 3.445959262868532, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313131313131315, + "loss": 2.4516, + "theoretical_loss": 3.4459492199601556, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331111111111111, + "loss": 2.6564, + "theoretical_loss": 3.445939177496361, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309090909090907, + "loss": 2.4535, + "theoretical_loss": 3.445929135477113, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330707070707071, + "loss": 2.6398, + "theoretical_loss": 3.445919093902377, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330505050505051, + "loss": 2.6726, + "theoretical_loss": 3.4459090527721177, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043303030303030305, + "loss": 2.7359, + "theoretical_loss": 3.4458990120863, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433010101010101, + "loss": 2.7408, + "theoretical_loss": 3.445888971844888, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432989898989899, + "loss": 2.5757, + "theoretical_loss": 3.4458789320478487, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043296969696969697, + "loss": 2.7217, + "theoretical_loss": 3.4458688926951453, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432949494949495, + "loss": 2.5871, + "theoretical_loss": 3.4458588537867434, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043292929292929294, + "loss": 2.4805, + "theoretical_loss": 3.4458488153226083, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329090909090909, + "loss": 2.5019, + "theoretical_loss": 3.445838777302704, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328888888888889, + "loss": 2.7685, + "theoretical_loss": 3.4458287397269967, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286868686868686, + "loss": 2.5512, + "theoretical_loss": 3.4458187025954503, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328484848484849, + "loss": 2.7532, + "theoretical_loss": 3.4458086659080305, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1071554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6713194847106934, + "objective/train/theoretical_loss": 3.4457986296647025, + "objective/train/tokens_used": 254750176, + "theoretical_loss": 3.4457986296647025, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043282828282828283, + "loss": 2.6613, + "theoretical_loss": 3.4457986296647025, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043280808080808084, + "loss": 2.5747, + "theoretical_loss": 3.4457885938654305, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327878787878788, + "loss": 2.5598, + "theoretical_loss": 3.44577855851018, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043276767676767676, + "loss": 2.5636, + "theoretical_loss": 3.4457685235989155, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274747474747477, + "loss": 2.6588, + "theoretical_loss": 3.445758489131603, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327272727272727, + "loss": 2.775, + "theoretical_loss": 3.4457484551082063, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270707070707073, + "loss": 2.6914, + "theoretical_loss": 3.4457384215286915, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326868686868687, + "loss": 2.54, + "theoretical_loss": 3.4457283883930234, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043266666666666665, + "loss": 2.6528, + "theoretical_loss": 3.4457183557011666, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043264646464646466, + "loss": 2.6084, + "theoretical_loss": 3.4457083234530863, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262626262626267, + "loss": 2.6185, + "theoretical_loss": 3.445698291648747, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043260606060606063, + "loss": 2.4756, + "theoretical_loss": 3.445688260288115, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325858585858586, + "loss": 2.4512, + "theoretical_loss": 3.445678229371154, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043256565656565654, + "loss": 2.6287, + "theoretical_loss": 3.4456681988978306, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254545454545455, + "loss": 2.3139, + "theoretical_loss": 3.4456581688681087, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043252525252525256, + "loss": 2.8243, + "theoretical_loss": 3.4456481392819533, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325050505050505, + "loss": 2.5888, + "theoretical_loss": 3.44563811013933, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324848484848485, + "loss": 2.7145, + "theoretical_loss": 3.445628081440203, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246464646464643, + "loss": 2.538, + "theoretical_loss": 3.445618053184539, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324444444444445, + "loss": 2.7705, + "theoretical_loss": 3.4456080253723016, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242424242424246, + "loss": 2.953, + "theoretical_loss": 3.445597998003456, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324040404040404, + "loss": 2.6659, + "theoretical_loss": 3.445587971077968, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043238383838383837, + "loss": 2.7242, + "theoretical_loss": 3.4455779445958026, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323636363636363, + "loss": 2.6194, + "theoretical_loss": 3.4455679185569243, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323434343434344, + "loss": 2.6441, + "theoretical_loss": 3.4455578929612987, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1072893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6098968982696533, + "objective/train/theoretical_loss": 3.44554786780889, + "objective/train/tokens_used": 256388576, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043232323232323235, + "loss": 2.5872, + "theoretical_loss": 3.44554786780889, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043232323232323235, + "loss": 2.5777, + "theoretical_loss": 3.4455378430996646, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323030303030303, + "loss": 2.6865, + "theoretical_loss": 3.445527818833587, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043228282828282826, + "loss": 2.7233, + "theoretical_loss": 3.4455177950106215, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322626262626262, + "loss": 2.7201, + "theoretical_loss": 3.445507771630735, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322424242424243, + "loss": 2.8348, + "theoretical_loss": 3.4454977486938914, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222222222222224, + "loss": 2.8986, + "theoretical_loss": 3.4454877262000556, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322020202020202, + "loss": 2.6766, + "theoretical_loss": 3.4454777041491935, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218181818181816, + "loss": 2.4893, + "theoretical_loss": 3.44546768254127, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043216161616161617, + "loss": 2.6269, + "theoretical_loss": 3.44545766137625, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321414141414142, + "loss": 2.7689, + "theoretical_loss": 3.4454476406540984, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212121212121213, + "loss": 2.5719, + "theoretical_loss": 3.445437620374781, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321010101010101, + "loss": 2.7092, + "theoretical_loss": 3.445427600538263, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043208080808080805, + "loss": 2.6437, + "theoretical_loss": 3.4454175811445085, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043206060606060606, + "loss": 2.7995, + "theoretical_loss": 3.445407562193484, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204040404040407, + "loss": 2.4303, + "theoretical_loss": 3.4453975436851536, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432020202020202, + "loss": 2.6503, + "theoretical_loss": 3.445387525619483, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432, + "loss": 2.3745, + "theoretical_loss": 3.445377507996437, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431979797979798, + "loss": 2.6717, + "theoretical_loss": 3.445367490815981, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043195959595959595, + "loss": 2.5656, + "theoretical_loss": 3.4453574740780804, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043193939393939396, + "loss": 2.7663, + "theoretical_loss": 3.4453474577826997, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319191919191919, + "loss": 2.6314, + "theoretical_loss": 3.445337441929805, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318989898989899, + "loss": 2.6291, + "theoretical_loss": 3.445327426519361, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318787878787879, + "loss": 2.7187, + "theoretical_loss": 3.4453174115513323, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318585858585859, + "loss": 2.8786, + "theoretical_loss": 3.445307397025685, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 1073632, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9225447177886963, + "objective/train/theoretical_loss": 3.445297382942384, + "objective/train/tokens_used": 258026976, + "theoretical_loss": 3.445297382942384, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183838383838386, + "loss": 2.809, + "theoretical_loss": 3.445297382942384, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004318181818181818, + "loss": 2.8967, + "theoretical_loss": 3.4452873693013943, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317979797979798, + "loss": 2.8723, + "theoretical_loss": 3.4452773561026815, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317777777777778, + "loss": 2.8648, + "theoretical_loss": 3.4452673433462104, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317575757575758, + "loss": 2.743, + "theoretical_loss": 3.4452573310319465, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043173737373737375, + "loss": 2.6014, + "theoretical_loss": 3.4452473191598547, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317171717171717, + "loss": 2.7664, + "theoretical_loss": 3.4452373077299003, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316969696969697, + "loss": 2.6833, + "theoretical_loss": 3.4452272967420488, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316767676767677, + "loss": 2.7623, + "theoretical_loss": 3.4452172861962653, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316565656565657, + "loss": 2.6613, + "theoretical_loss": 3.445207276092515, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163636363636364, + "loss": 2.5603, + "theoretical_loss": 3.445197266430763, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161616161616165, + "loss": 2.7279, + "theoretical_loss": 3.4451872572109745, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315959595959596, + "loss": 2.7308, + "theoretical_loss": 3.445177248433115, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043157575757575757, + "loss": 2.9899, + "theoretical_loss": 3.44516724009715, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315555555555556, + "loss": 2.663, + "theoretical_loss": 3.445157232203044, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153535353535353, + "loss": 2.6168, + "theoretical_loss": 3.4451472247507624, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043151515151515154, + "loss": 2.7339, + "theoretical_loss": 3.445137217740271, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314949494949495, + "loss": 2.4853, + "theoretical_loss": 3.4451272111715348, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147474747474746, + "loss": 2.541, + "theoretical_loss": 3.445117205044519, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043145454545454547, + "loss": 2.7185, + "theoretical_loss": 3.445107199359189, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343434343435, + "loss": 2.6942, + "theoretical_loss": 3.44509719411551, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043141414141414144, + "loss": 2.8555, + "theoretical_loss": 3.4450871893134467, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313939393939394, + "loss": 2.6386, + "theoretical_loss": 3.4450771849529653, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043137373737373735, + "loss": 2.7455, + "theoretical_loss": 3.4450671810340308, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043135353535353536, + "loss": 2.6658, + "theoretical_loss": 3.4450571775566083, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1075027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.614713668823242, + "objective/train/theoretical_loss": 3.445047174520663, + "objective/train/tokens_used": 259665376, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313333333333334, + "loss": 2.7304, + "theoretical_loss": 3.445047174520663, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131313131313133, + "loss": 2.5398, + "theoretical_loss": 3.4450371719261605, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312929292929293, + "loss": 2.8086, + "theoretical_loss": 3.4450271697730663, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127272727272724, + "loss": 2.4278, + "theoretical_loss": 3.445017168061345, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312525252525253, + "loss": 2.6343, + "theoretical_loss": 3.4450071667909623, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123232323232327, + "loss": 2.8146, + "theoretical_loss": 3.4449971659618837, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312121212121212, + "loss": 2.7642, + "theoretical_loss": 3.4449871655740743, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311919191919192, + "loss": 2.7717, + "theoretical_loss": 3.444977165627499, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043117171717171714, + "loss": 2.8654, + "theoretical_loss": 3.444967166122124, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311515151515152, + "loss": 2.5174, + "theoretical_loss": 3.444957167057914, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043113131313131316, + "loss": 2.6686, + "theoretical_loss": 3.444947168434835, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311111111111111, + "loss": 2.6287, + "theoretical_loss": 3.4449371702528513, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043109090909090907, + "loss": 2.7311, + "theoretical_loss": 3.444927172511929, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310707070707071, + "loss": 2.5892, + "theoretical_loss": 3.4449171752120336, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310505050505051, + "loss": 2.8177, + "theoretical_loss": 3.4449071783531293, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043103030303030305, + "loss": 2.6324, + "theoretical_loss": 3.4448971819351826, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431010101010101, + "loss": 2.8839, + "theoretical_loss": 3.4448871859581587, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043098989898989897, + "loss": 2.8282, + "theoretical_loss": 3.444877190422023, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430969696969697, + "loss": 2.6817, + "theoretical_loss": 3.44486719532674, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430949494949495, + "loss": 2.6844, + "theoretical_loss": 3.444857200672276, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043092929292929294, + "loss": 2.9557, + "theoretical_loss": 3.4448472064585958, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309090909090909, + "loss": 2.8264, + "theoretical_loss": 3.4448372126856652, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308888888888889, + "loss": 2.6539, + "theoretical_loss": 3.4448272193534493, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043086868686868687, + "loss": 2.863, + "theoretical_loss": 3.4448172264619137, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308484848484849, + "loss": 2.613, + "theoretical_loss": 3.444807234011024, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1075669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4522032737731934, + "objective/train/theoretical_loss": 3.4447972420007447, + "objective/train/tokens_used": 261303776, + "theoretical_loss": 3.4447972420007447, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082828282828284, + "loss": 2.6596, + "theoretical_loss": 3.4447972420007447, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308080808080808, + "loss": 2.5959, + "theoretical_loss": 3.444787250431042, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307878787878788, + "loss": 2.7244, + "theoretical_loss": 3.444777259301881, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043076767676767676, + "loss": 2.8103, + "theoretical_loss": 3.4447672686132274, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074747474747477, + "loss": 2.6817, + "theoretical_loss": 3.4447572783650457, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043072727272727273, + "loss": 2.5849, + "theoretical_loss": 3.444747288557303, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043070707070707074, + "loss": 2.7099, + "theoretical_loss": 3.444737299189963, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306868686868687, + "loss": 2.7166, + "theoretical_loss": 3.444727310262992, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066666666666665, + "loss": 2.7347, + "theoretical_loss": 3.444717321776355, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043064646464646467, + "loss": 2.5753, + "theoretical_loss": 3.444707333730018, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306262626262626, + "loss": 2.7924, + "theoretical_loss": 3.444697346123946, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043060606060606063, + "loss": 2.5358, + "theoretical_loss": 3.444687358958104, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305858585858586, + "loss": 2.6528, + "theoretical_loss": 3.4446773722324586, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043056565656565655, + "loss": 2.7749, + "theoretical_loss": 3.4446673859469747, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043054545454545456, + "loss": 2.6544, + "theoretical_loss": 3.444657400101617, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052525252525257, + "loss": 2.7302, + "theoretical_loss": 3.444647414696352, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305050505050505, + "loss": 2.583, + "theoretical_loss": 3.444637429731145, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304848484848485, + "loss": 2.8319, + "theoretical_loss": 3.4446274452059606, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043046464646464644, + "loss": 2.6831, + "theoretical_loss": 3.444617461120765, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044444444444445, + "loss": 2.5649, + "theoretical_loss": 3.444607477475524, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042424242424246, + "loss": 2.7912, + "theoretical_loss": 3.4445974942702025, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304040404040404, + "loss": 2.7847, + "theoretical_loss": 3.4445875115047655, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303838383838384, + "loss": 2.7662, + "theoretical_loss": 3.4445775291791794, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036363636363633, + "loss": 2.6755, + "theoretical_loss": 3.444567547293409, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043034343434343434, + "loss": 2.8277, + "theoretical_loss": 3.444557565847421, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1076839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4996190071105957, + "objective/train/theoretical_loss": 3.444547584841179, + "objective/train/tokens_used": 262942176, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032323232323235, + "loss": 2.6335, + "theoretical_loss": 3.444547584841179, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303030303030303, + "loss": 2.4921, + "theoretical_loss": 3.4445376042746503, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028282828282827, + "loss": 2.6737, + "theoretical_loss": 3.444527624147799, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302626262626262, + "loss": 2.6463, + "theoretical_loss": 3.444517644460591, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302424242424243, + "loss": 2.5381, + "theoretical_loss": 3.4445076652129925, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043022222222222225, + "loss": 2.7264, + "theoretical_loss": 3.4444976864049686, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302020202020202, + "loss": 2.8457, + "theoretical_loss": 3.444487708036484, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043018181818181816, + "loss": 2.6152, + "theoretical_loss": 3.444477730107505, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043016161616161617, + "loss": 2.4122, + "theoretical_loss": 3.4444677526179976, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301414141414142, + "loss": 2.7439, + "theoretical_loss": 3.444457775567927, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012121212121214, + "loss": 2.8234, + "theoretical_loss": 3.444447798957258, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301010101010101, + "loss": 2.5962, + "theoretical_loss": 3.4444378227859564, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043008080808080805, + "loss": 2.5078, + "theoretical_loss": 3.4444278470539884, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300606060606061, + "loss": 2.7247, + "theoretical_loss": 3.4444178717613188, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300404040404041, + "loss": 2.7075, + "theoretical_loss": 3.4444078969079133, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002020202020203, + "loss": 2.6488, + "theoretical_loss": 3.4443979224937378, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 2.6001, + "theoretical_loss": 3.4443879485187576, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997979797979795, + "loss": 2.627, + "theoretical_loss": 3.4443779749829386, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429959595959596, + "loss": 2.9583, + "theoretical_loss": 3.4443680018862457, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993939393939397, + "loss": 2.8714, + "theoretical_loss": 3.4443580292286446, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299191919191919, + "loss": 2.8132, + "theoretical_loss": 3.4443480570101017, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298989898989899, + "loss": 2.5943, + "theoretical_loss": 3.444338085230582, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298787878787879, + "loss": 2.6111, + "theoretical_loss": 3.44432811389005, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298585858585859, + "loss": 2.7233, + "theoretical_loss": 3.444318142988473, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042983838383838386, + "loss": 2.7228, + "theoretical_loss": 3.444308172525816, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1077487, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.277385950088501, + "objective/train/theoretical_loss": 3.444298202502044, + "objective/train/tokens_used": 264580576, + "theoretical_loss": 3.444298202502044, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298181818181818, + "loss": 2.4898, + "theoretical_loss": 3.444298202502044, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297979797979798, + "loss": 2.6082, + "theoretical_loss": 3.444288232917123, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297777777777778, + "loss": 2.7607, + "theoretical_loss": 3.444278263771019, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297575757575758, + "loss": 2.5416, + "theoretical_loss": 3.444268295063697, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973737373737375, + "loss": 2.892, + "theoretical_loss": 3.444258326795123, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297171717171717, + "loss": 2.8372, + "theoretical_loss": 3.444248358965263, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296969696969697, + "loss": 2.5166, + "theoretical_loss": 3.444238391574081, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296767676767677, + "loss": 2.6708, + "theoretical_loss": 3.4442284246215444, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296565656565657, + "loss": 2.7981, + "theoretical_loss": 3.444218458107618, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042963636363636365, + "loss": 2.6832, + "theoretical_loss": 3.444208492032267, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296161616161616, + "loss": 2.7131, + "theoretical_loss": 3.444198526395458, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295959595959596, + "loss": 2.8732, + "theoretical_loss": 3.444188561197156, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957575757575757, + "loss": 2.6032, + "theoretical_loss": 3.4441785964373266, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295555555555556, + "loss": 2.6017, + "theoretical_loss": 3.444168632115936, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042953535353535354, + "loss": 2.6758, + "theoretical_loss": 3.444158668232949, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042951515151515155, + "loss": 2.6171, + "theoretical_loss": 3.444148704788332, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294949494949495, + "loss": 2.6636, + "theoretical_loss": 3.44413874178205, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947474747474746, + "loss": 2.8905, + "theoretical_loss": 3.4441287792140693, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294545454545455, + "loss": 2.6362, + "theoretical_loss": 3.4441188170843553, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042943434343434343, + "loss": 2.6423, + "theoretical_loss": 3.444108855392874, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941414141414144, + "loss": 2.6154, + "theoretical_loss": 3.44409889413959, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293939393939394, + "loss": 2.7057, + "theoretical_loss": 3.4440889333244695, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937373737373736, + "loss": 2.6953, + "theoretical_loss": 3.444078972947479, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042935353535353537, + "loss": 2.6045, + "theoretical_loss": 3.4440690130085825, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293333333333334, + "loss": 2.6675, + "theoretical_loss": 3.4440590535077478, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1078017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.120889902114868, + "objective/train/theoretical_loss": 3.4440490944449387, + "objective/train/tokens_used": 266218976, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042931313131313134, + "loss": 2.5581, + "theoretical_loss": 3.4440490944449387, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292929292929293, + "loss": 2.5608, + "theoretical_loss": 3.444039135820122, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042927272727272725, + "loss": 2.8178, + "theoretical_loss": 3.444029177633263, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925252525252526, + "loss": 2.5556, + "theoretical_loss": 3.4440192198843276, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042923232323232327, + "loss": 2.6555, + "theoretical_loss": 3.4440092625732808, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921212121212123, + "loss": 2.4992, + "theoretical_loss": 3.443999305700089, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291919191919192, + "loss": 2.6801, + "theoretical_loss": 3.443989349264718, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042917171717171714, + "loss": 2.5762, + "theoretical_loss": 3.4439793932671323, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291515151515152, + "loss": 2.7817, + "theoretical_loss": 3.4439694377073, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913131313131316, + "loss": 2.844, + "theoretical_loss": 3.4439594825851843, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291111111111111, + "loss": 2.8671, + "theoretical_loss": 3.4439495279007524, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290909090909091, + "loss": 2.8253, + "theoretical_loss": 3.443939573653969, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042907070707070703, + "loss": 2.4921, + "theoretical_loss": 3.443929619844801, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290505050505051, + "loss": 2.6439, + "theoretical_loss": 3.4439196664732137, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903030303030306, + "loss": 2.7558, + "theoretical_loss": 3.4439097135391723, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429010101010101, + "loss": 2.7156, + "theoretical_loss": 3.443899761042643, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898989898989897, + "loss": 2.9078, + "theoretical_loss": 3.4438898089835916, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042896969696969693, + "loss": 3.0627, + "theoretical_loss": 3.443879857361984, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428949494949495, + "loss": 2.6492, + "theoretical_loss": 3.4438699061777855, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042892929292929295, + "loss": 2.5642, + "theoretical_loss": 3.4438599554309617, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289090909090909, + "loss": 2.7501, + "theoretical_loss": 3.4438500051214787, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042888888888888886, + "loss": 2.5366, + "theoretical_loss": 3.4438400552493027, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288686868686869, + "loss": 2.5917, + "theoretical_loss": 3.4438301058143987, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288484848484849, + "loss": 2.6946, + "theoretical_loss": 3.443820156816733, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882828282828284, + "loss": 2.8147, + "theoretical_loss": 3.443810208256271, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1079511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9551053047180176, + "objective/train/theoretical_loss": 3.4438002601329787, + "objective/train/tokens_used": 267857376, + "theoretical_loss": 3.4438002601329787, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288080808080808, + "loss": 2.9128, + "theoretical_loss": 3.4438002601329787, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878787878787876, + "loss": 2.8342, + "theoretical_loss": 3.443790312446822, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042876767676767677, + "loss": 2.4505, + "theoretical_loss": 3.4437803651977665, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287474747474748, + "loss": 2.819, + "theoretical_loss": 3.4437704183857782, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042872727272727274, + "loss": 2.6408, + "theoretical_loss": 3.443760472010822, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287070707070707, + "loss": 2.6174, + "theoretical_loss": 3.443750526072865, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286868686868687, + "loss": 2.6985, + "theoretical_loss": 3.4437405805718724, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042866666666666666, + "loss": 2.6345, + "theoretical_loss": 3.4437306355078094, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864646464646467, + "loss": 2.6218, + "theoretical_loss": 3.4437206908806433, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042862626262626263, + "loss": 2.8083, + "theoretical_loss": 3.4437107466903383, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286060606060606, + "loss": 2.8247, + "theoretical_loss": 3.443700802936861, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285858585858586, + "loss": 2.7066, + "theoretical_loss": 3.443690859620178, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856565656565655, + "loss": 2.6625, + "theoretical_loss": 3.443680916740253, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854545454545456, + "loss": 2.6301, + "theoretical_loss": 3.443670974297054, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285252525252525, + "loss": 3.0387, + "theoretical_loss": 3.443661032290546, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850505050505053, + "loss": 2.6901, + "theoretical_loss": 3.4436510907206945, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284848484848485, + "loss": 2.7584, + "theoretical_loss": 3.4436411495874655, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846464646464645, + "loss": 2.6689, + "theoretical_loss": 3.4436312088908254, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042844444444444446, + "loss": 2.6447, + "theoretical_loss": 3.4436212686307393, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284242424242424, + "loss": 2.6134, + "theoretical_loss": 3.4436113288071732, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284040404040404, + "loss": 2.6417, + "theoretical_loss": 3.4436013894200936, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283838383838384, + "loss": 2.8404, + "theoretical_loss": 3.443591450469466, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283636363636364, + "loss": 2.5163, + "theoretical_loss": 3.4435815119552555, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834343434343435, + "loss": 2.8022, + "theoretical_loss": 3.443571573877429, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042832323232323236, + "loss": 2.7363, + "theoretical_loss": 3.443561636235952, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1080108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601081132888794, + "objective/train/theoretical_loss": 3.4435516990307904, + "objective/train/tokens_used": 269495776, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283030303030303, + "loss": 2.7027, + "theoretical_loss": 3.4435516990307904, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282828282828283, + "loss": 2.9469, + "theoretical_loss": 3.44354176226191, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282626262626263, + "loss": 3.005, + "theoretical_loss": 3.4435318259292766, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042824242424242424, + "loss": 2.6537, + "theoretical_loss": 3.443521890032856, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822222222222225, + "loss": 2.6807, + "theoretical_loss": 3.4435119545726147, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282020202020202, + "loss": 2.8273, + "theoretical_loss": 3.443502019548518, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818181818181817, + "loss": 2.5975, + "theoretical_loss": 3.4434920849605324, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281616161616162, + "loss": 2.8314, + "theoretical_loss": 3.443482150808623, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281414141414142, + "loss": 2.4922, + "theoretical_loss": 3.443472217092756, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042812121212121215, + "loss": 2.5921, + "theoretical_loss": 3.443462283812898, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281010101010101, + "loss": 2.7378, + "theoretical_loss": 3.4434523509690136, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808080808080806, + "loss": 2.4204, + "theoretical_loss": 3.44344241856107, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042806060606060607, + "loss": 2.7128, + "theoretical_loss": 3.4434324865890322, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280404040404041, + "loss": 2.7773, + "theoretical_loss": 3.4434225550528668, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802020202020204, + "loss": 2.7091, + "theoretical_loss": 3.443412623952539, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428, + "loss": 2.8743, + "theoretical_loss": 3.4434026932880153, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042797979797979795, + "loss": 2.6566, + "theoretical_loss": 3.4433927630592613, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427959595959596, + "loss": 2.932, + "theoretical_loss": 3.4433828332662437, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427939393939394, + "loss": 2.7492, + "theoretical_loss": 3.4433729039089274, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791919191919193, + "loss": 2.4621, + "theoretical_loss": 3.443362974987279, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278989898989899, + "loss": 2.7523, + "theoretical_loss": 3.4433530465012643, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042787878787878784, + "loss": 2.5482, + "theoretical_loss": 3.443343118450849, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278585858585859, + "loss": 2.8314, + "theoretical_loss": 3.4433331908359994, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042783838383838387, + "loss": 2.9894, + "theoretical_loss": 3.4433232636566817, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278181818181818, + "loss": 2.6614, + "theoretical_loss": 3.443313336912861, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1081319, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.620631456375122, + "objective/train/theoretical_loss": 3.443303410604504, + "objective/train/tokens_used": 271134176, + "theoretical_loss": 3.443303410604504, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277979797979798, + "loss": 2.6672, + "theoretical_loss": 3.443303410604504, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277777777777778, + "loss": 2.7726, + "theoretical_loss": 3.4432934847315764, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277575757575758, + "loss": 2.8353, + "theoretical_loss": 3.443283559294044, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773737373737376, + "loss": 2.7925, + "theoretical_loss": 3.4432736342918737, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277171717171717, + "loss": 2.6392, + "theoretical_loss": 3.44326370972503, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276969696969697, + "loss": 2.7439, + "theoretical_loss": 3.4432537855934804, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276767676767677, + "loss": 2.3873, + "theoretical_loss": 3.44324386189719, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276565656565657, + "loss": 2.8104, + "theoretical_loss": 3.4432339386361246, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042763636363636365, + "loss": 2.777, + "theoretical_loss": 3.4432240158102507, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276161616161616, + "loss": 2.6828, + "theoretical_loss": 3.443214093419534, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275959595959596, + "loss": 2.6473, + "theoretical_loss": 3.4432041714639414, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275757575757576, + "loss": 2.5726, + "theoretical_loss": 3.4431942499434376, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275555555555556, + "loss": 2.5018, + "theoretical_loss": 3.4431843288579893, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753535353535354, + "loss": 2.7409, + "theoretical_loss": 3.443174408207563, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275151515151515, + "loss": 2.7841, + "theoretical_loss": 3.4431644879921235, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274949494949495, + "loss": 2.8187, + "theoretical_loss": 3.4431545682116376, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042747474747474747, + "loss": 2.868, + "theoretical_loss": 3.4431446488660713, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274545454545455, + "loss": 2.8821, + "theoretical_loss": 3.44313472995539, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743434343434344, + "loss": 2.8163, + "theoretical_loss": 3.443124811479561, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042741414141414145, + "loss": 2.6401, + "theoretical_loss": 3.44311489343855, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273939393939394, + "loss": 2.5481, + "theoretical_loss": 3.443104975832322, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042737373737373736, + "loss": 2.5437, + "theoretical_loss": 3.4430950586608438, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273535353535354, + "loss": 2.7811, + "theoretical_loss": 3.4430851419240818, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042733333333333333, + "loss": 2.7768, + "theoretical_loss": 3.4430752256220014, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731313131313134, + "loss": 2.7087, + "theoretical_loss": 3.443065309754569, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1081978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6425259113311768, + "objective/train/theoretical_loss": 3.4430553943217506, + "objective/train/tokens_used": 272772576, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272929292929293, + "loss": 2.7002, + "theoretical_loss": 3.4430553943217506, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727272727272726, + "loss": 2.5526, + "theoretical_loss": 3.443045479323512, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042725252525252527, + "loss": 2.8075, + "theoretical_loss": 3.4430355647598194, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272323232323233, + "loss": 2.574, + "theoretical_loss": 3.4430256506306396, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721212121212123, + "loss": 2.6478, + "theoretical_loss": 3.443015736935938, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271919191919192, + "loss": 2.5519, + "theoretical_loss": 3.4430058236756804, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042717171717171715, + "loss": 2.8293, + "theoretical_loss": 3.4429959108498336, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715151515151516, + "loss": 2.784, + "theoretical_loss": 3.4429859984583633, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713131313131317, + "loss": 2.7762, + "theoretical_loss": 3.4429760865012353, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271111111111111, + "loss": 2.5127, + "theoretical_loss": 3.442966174978416, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270909090909091, + "loss": 2.4419, + "theoretical_loss": 3.4429562638898723, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042707070707070704, + "loss": 2.5691, + "theoretical_loss": 3.442946353235569, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705050505050505, + "loss": 2.7365, + "theoretical_loss": 3.4429364430154727, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042703030303030306, + "loss": 2.7481, + "theoretical_loss": 3.44292653322955, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427010101010101, + "loss": 2.6823, + "theoretical_loss": 3.4429166238777658, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426989898989899, + "loss": 2.7264, + "theoretical_loss": 3.442906714960088, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042696969696969693, + "loss": 2.6378, + "theoretical_loss": 3.4428968064764813, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426949494949495, + "loss": 2.4712, + "theoretical_loss": 3.442886898426912, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042692929292929296, + "loss": 2.5547, + "theoretical_loss": 3.442876990811347, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269090909090909, + "loss": 2.7754, + "theoretical_loss": 3.4428670836297517, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042688888888888887, + "loss": 2.5846, + "theoretical_loss": 3.442857176882093, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268686868686868, + "loss": 2.8138, + "theoretical_loss": 3.442847270568336, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268484848484849, + "loss": 2.8303, + "theoretical_loss": 3.4428373646884474, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682828282828285, + "loss": 2.6512, + "theoretical_loss": 3.4428274592423938, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268080808080808, + "loss": 2.7974, + "theoretical_loss": 3.4428175542301407, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 1083212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6756677627563477, + "objective/train/theoretical_loss": 3.442807649651654, + "objective/train/tokens_used": 274410976, + "theoretical_loss": 3.442807649651654, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678787878787876, + "loss": 2.6918, + "theoretical_loss": 3.442807649651654, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042676767676767677, + "loss": 2.3226, + "theoretical_loss": 3.4427977455069008, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267474747474748, + "loss": 2.6835, + "theoretical_loss": 3.4427878417958464, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672727272727274, + "loss": 2.5338, + "theoretical_loss": 3.442777938518458, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267070707070707, + "loss": 2.8393, + "theoretical_loss": 3.4427680356747006, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042668686868686865, + "loss": 2.8166, + "theoretical_loss": 3.442758133264541, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266666666666667, + "loss": 2.67, + "theoretical_loss": 3.442748231287946, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266464646464647, + "loss": 2.7486, + "theoretical_loss": 3.4427383297448806, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042662626262626263, + "loss": 2.6207, + "theoretical_loss": 3.4427284286353115, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266060606060606, + "loss": 2.8416, + "theoretical_loss": 3.442718527959205, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265858585858586, + "loss": 2.6879, + "theoretical_loss": 3.4427086277165273, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265656565656566, + "loss": 2.7998, + "theoretical_loss": 3.442698727907244, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042654545454545457, + "loss": 2.6392, + "theoretical_loss": 3.442688828531322, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265252525252525, + "loss": 2.6187, + "theoretical_loss": 3.4426789295887277, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265050505050505, + "loss": 2.6045, + "theoretical_loss": 3.4426690310794266, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264848484848485, + "loss": 2.5709, + "theoretical_loss": 3.442659133003385, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264646464646465, + "loss": 2.5805, + "theoretical_loss": 3.4426492353605695, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042644444444444446, + "loss": 2.746, + "theoretical_loss": 3.442639338150946, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264242424242424, + "loss": 2.6454, + "theoretical_loss": 3.4426294413744816, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640404040404043, + "loss": 2.7877, + "theoretical_loss": 3.4426195450311416, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263838383838384, + "loss": 2.5915, + "theoretical_loss": 3.442609649120892, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263636363636364, + "loss": 2.7116, + "theoretical_loss": 3.4425997536437, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042634343434343435, + "loss": 2.9474, + "theoretical_loss": 3.4425898585995314, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263232323232323, + "loss": 2.4712, + "theoretical_loss": 3.4425799639883525, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263030303030303, + "loss": 2.6075, + "theoretical_loss": 3.442570069810129, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1083753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.224860906600952, + "objective/train/theoretical_loss": 3.442560176064828, + "objective/train/tokens_used": 276049376, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262828282828283, + "loss": 2.5313, + "theoretical_loss": 3.442560176064828, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262626262626263, + "loss": 2.8617, + "theoretical_loss": 3.442550282752415, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624242424242425, + "loss": 2.664, + "theoretical_loss": 3.4425403898728573, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622222222222226, + "loss": 2.7076, + "theoretical_loss": 3.44253049742612, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262020202020202, + "loss": 2.7232, + "theoretical_loss": 3.44252060541217, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042618181818181817, + "loss": 2.8491, + "theoretical_loss": 3.4425107138309734, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261616161616162, + "loss": 2.7145, + "theoretical_loss": 3.4425008226824967, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614141414141414, + "loss": 2.5732, + "theoretical_loss": 3.442490931966706, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612121212121215, + "loss": 2.7515, + "theoretical_loss": 3.4424810416835676, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261010101010101, + "loss": 2.9121, + "theoretical_loss": 3.4424711518330477, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042608080808080806, + "loss": 2.795, + "theoretical_loss": 3.442461262415113, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260606060606061, + "loss": 2.7846, + "theoretical_loss": 3.442451373429729, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260404040404041, + "loss": 2.632, + "theoretical_loss": 3.4424414848768627, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042602020202020204, + "loss": 2.4911, + "theoretical_loss": 3.4424315967564807, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426, + "loss": 2.6223, + "theoretical_loss": 3.4424217090685483, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042597979797979796, + "loss": 2.9115, + "theoretical_loss": 3.4424118218130326, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042595959595959597, + "loss": 2.4854, + "theoretical_loss": 3.442401934989899, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425939393939394, + "loss": 2.6288, + "theoretical_loss": 3.442392048599115, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042591919191919194, + "loss": 2.7059, + "theoretical_loss": 3.442382162640646, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258989898989899, + "loss": 2.9154, + "theoretical_loss": 3.442372277114459, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042587878787878785, + "loss": 2.4787, + "theoretical_loss": 3.44236239202052, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258585858585859, + "loss": 2.7692, + "theoretical_loss": 3.4423525073587955, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042583838383838387, + "loss": 2.6983, + "theoretical_loss": 3.4423426231292518, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042581818181818183, + "loss": 2.7215, + "theoretical_loss": 3.442332739331855, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257979797979798, + "loss": 2.3635, + "theoretical_loss": 3.4423228559665713, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1084888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8273839950561523, + "objective/train/theoretical_loss": 3.442312973033368, + "objective/train/tokens_used": 277687776, + "theoretical_loss": 3.442312973033368, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577777777777774, + "loss": 2.6553, + "theoretical_loss": 3.442312973033368, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257575757575758, + "loss": 2.7447, + "theoretical_loss": 3.4423030905322105, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042573737373737377, + "loss": 2.9124, + "theoretical_loss": 3.4422932084630653, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171717171717, + "loss": 2.8553, + "theoretical_loss": 3.442283326825899, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256969696969697, + "loss": 2.6791, + "theoretical_loss": 3.4422734456206783, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042567676767676764, + "loss": 2.8093, + "theoretical_loss": 3.4422635648473685, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256565656565657, + "loss": 2.5431, + "theoretical_loss": 3.442253684505937, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563636363636366, + "loss": 2.5809, + "theoretical_loss": 3.4422438045963495, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256161616161616, + "loss": 2.7907, + "theoretical_loss": 3.4422339251185727, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042559595959595957, + "loss": 2.8678, + "theoretical_loss": 3.4422240460725737, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255757575757576, + "loss": 2.6176, + "theoretical_loss": 3.4422141674583173, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255555555555556, + "loss": 2.5011, + "theoretical_loss": 3.4422042892757716, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042553535353535355, + "loss": 2.6847, + "theoretical_loss": 3.4421944115249015, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255151515151515, + "loss": 2.681, + "theoretical_loss": 3.442184534205674, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549494949494946, + "loss": 2.5315, + "theoretical_loss": 3.4421746573180556, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254747474747475, + "loss": 2.6036, + "theoretical_loss": 3.4421647808620133, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254545454545455, + "loss": 2.8762, + "theoretical_loss": 3.442154904837512, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042543434343434344, + "loss": 2.7843, + "theoretical_loss": 3.4421450292445197, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254141414141414, + "loss": 2.6677, + "theoretical_loss": 3.442135154083002, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253939393939394, + "loss": 2.8041, + "theoretical_loss": 3.4421252793529247, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537373737373737, + "loss": 2.7381, + "theoretical_loss": 3.4421154050542553, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253535353535354, + "loss": 2.5793, + "theoretical_loss": 3.4421055311869604, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533333333333334, + "loss": 2.8338, + "theoretical_loss": 3.442095657751006, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253131313131313, + "loss": 2.6696, + "theoretical_loss": 3.4420857847463573, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252929292929293, + "loss": 2.7061, + "theoretical_loss": 3.442075912172983, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1085471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.061549663543701, + "objective/train/theoretical_loss": 3.4420660400308476, + "objective/train/tokens_used": 279326176, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042527272727272726, + "loss": 2.4536, + "theoretical_loss": 3.4420660400308476, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525252525252527, + "loss": 2.7403, + "theoretical_loss": 3.442056168319919, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042523232323232323, + "loss": 2.8021, + "theoretical_loss": 3.442046297040163, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042521212121212124, + "loss": 2.6394, + "theoretical_loss": 3.442036426191546, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251919191919192, + "loss": 2.9006, + "theoretical_loss": 3.442026555774034, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517171717171715, + "loss": 2.6406, + "theoretical_loss": 3.442016685787595, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042515151515151516, + "loss": 2.7947, + "theoretical_loss": 3.442006816232194, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251313131313131, + "loss": 2.6113, + "theoretical_loss": 3.441996947107798, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511111111111113, + "loss": 2.9404, + "theoretical_loss": 3.441987078414373, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250909090909091, + "loss": 2.5703, + "theoretical_loss": 3.4419772101518866, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042507070707070705, + "loss": 2.6468, + "theoretical_loss": 3.4419673423203037, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042505050505050506, + "loss": 2.8877, + "theoretical_loss": 3.4419574749195925, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503030303030307, + "loss": 2.6515, + "theoretical_loss": 3.441947607949718, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425010101010101, + "loss": 2.6965, + "theoretical_loss": 3.441937741410648, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424989898989899, + "loss": 2.4998, + "theoretical_loss": 3.441927875302348, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424969696969697, + "loss": 2.5248, + "theoretical_loss": 3.441918009624785, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042494949494949495, + "loss": 2.6036, + "theoretical_loss": 3.4419081443779254, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492929292929296, + "loss": 2.6962, + "theoretical_loss": 3.4418982795617357, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249090909090909, + "loss": 2.6824, + "theoretical_loss": 3.441888415176182, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248888888888889, + "loss": 2.7944, + "theoretical_loss": 3.4418785512212313, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248686868686869, + "loss": 2.8519, + "theoretical_loss": 3.44186868769685, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248484848484849, + "loss": 2.6476, + "theoretical_loss": 3.441858824603005, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042482828282828285, + "loss": 2.6608, + "theoretical_loss": 3.4418489619396624, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248080808080808, + "loss": 2.7293, + "theoretical_loss": 3.4418390997067885, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042478787878787877, + "loss": 2.4583, + "theoretical_loss": 3.44182923790435, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1086847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.871436834335327, + "objective/train/theoretical_loss": 3.4418193765323135, + "objective/train/tokens_used": 280964576, + "theoretical_loss": 3.4418193765323135, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247676767676768, + "loss": 2.8128, + "theoretical_loss": 3.4418193765323135, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247474747474748, + "loss": 2.488, + "theoretical_loss": 3.4418095155906463, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042472727272727275, + "loss": 2.7828, + "theoretical_loss": 3.441799655079314, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247070707070707, + "loss": 2.547, + "theoretical_loss": 3.441789794998283, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468686868686866, + "loss": 2.5905, + "theoretical_loss": 3.4417799353475202, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246666666666667, + "loss": 2.5356, + "theoretical_loss": 3.4417700761269927, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246464646464647, + "loss": 2.768, + "theoretical_loss": 3.4417602173366664, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462626262626264, + "loss": 2.6383, + "theoretical_loss": 3.441750358976508, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246060606060606, + "loss": 2.4163, + "theoretical_loss": 3.4417405010464837, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042458585858585855, + "loss": 2.9941, + "theoretical_loss": 3.441730643546561, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245656565656566, + "loss": 2.8368, + "theoretical_loss": 3.441720786476706, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245454545454546, + "loss": 2.7983, + "theoretical_loss": 3.441710929836885, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042452525252525253, + "loss": 2.6048, + "theoretical_loss": 3.4417010736270646, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245050505050505, + "loss": 2.4772, + "theoretical_loss": 3.441691217847212, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244848484848485, + "loss": 2.7283, + "theoretical_loss": 3.4416813624972935, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244646464646465, + "loss": 2.5403, + "theoretical_loss": 3.441671507577275, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042444444444444447, + "loss": 2.5222, + "theoretical_loss": 3.441661653087124, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244242424242424, + "loss": 2.7154, + "theoretical_loss": 3.4416517990268067, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244040404040404, + "loss": 2.5209, + "theoretical_loss": 3.44164194539629, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243838383838384, + "loss": 2.7483, + "theoretical_loss": 3.4416320921955403, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243636363636364, + "loss": 2.7089, + "theoretical_loss": 3.4416222394245235, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042434343434343436, + "loss": 2.7632, + "theoretical_loss": 3.441612387083208, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243232323232323, + "loss": 2.6108, + "theoretical_loss": 3.4416025351715587, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430303030303033, + "loss": 2.96, + "theoretical_loss": 3.441592683689543, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828282828283, + "loss": 2.694, + "theoretical_loss": 3.4415828326371276, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1087440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6119213104248047, + "objective/train/theoretical_loss": 3.4415729820142786, + "objective/train/tokens_used": 282602976, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242626262626263, + "loss": 2.6647, + "theoretical_loss": 3.4415729820142786, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042424242424242425, + "loss": 2.7141, + "theoretical_loss": 3.441563131820963, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242222222222222, + "loss": 2.7173, + "theoretical_loss": 3.4415532820571473, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242020202020202, + "loss": 2.5161, + "theoretical_loss": 3.4415434327227983, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241818181818182, + "loss": 2.7635, + "theoretical_loss": 3.441533583817883, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241616161616162, + "loss": 2.7223, + "theoretical_loss": 3.4415237353423676, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414141414141415, + "loss": 2.8099, + "theoretical_loss": 3.4415138872962183, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042412121212121216, + "loss": 2.5967, + "theoretical_loss": 3.4415040396794025, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241010101010101, + "loss": 2.7751, + "theoretical_loss": 3.441494192491887, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408080808080807, + "loss": 2.7381, + "theoretical_loss": 3.4414843457336373, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240606060606061, + "loss": 2.6461, + "theoretical_loss": 3.441474499404621, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042404040404040404, + "loss": 2.6364, + "theoretical_loss": 3.441464653504805, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042402020202020205, + "loss": 2.8703, + "theoretical_loss": 3.4414548080341554, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424, + "loss": 2.6251, + "theoretical_loss": 3.441444962992639, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397979797979796, + "loss": 2.8216, + "theoretical_loss": 3.441435118380223, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423959595959596, + "loss": 2.6424, + "theoretical_loss": 3.4414252741968734, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042393939393939393, + "loss": 2.7731, + "theoretical_loss": 3.4414154304425573, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042391919191919194, + "loss": 2.6378, + "theoretical_loss": 3.4414055871172406, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238989898989899, + "loss": 2.5661, + "theoretical_loss": 3.441395744220891, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042387878787878786, + "loss": 2.6202, + "theoretical_loss": 3.441385901753475, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042385858585858587, + "loss": 2.6602, + "theoretical_loss": 3.441376059714959, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238383838383839, + "loss": 2.6932, + "theoretical_loss": 3.44136621810531, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381818181818183, + "loss": 2.685, + "theoretical_loss": 3.4413563769244946, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237979797979798, + "loss": 2.8668, + "theoretical_loss": 3.441346536172479, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042377777777777775, + "loss": 2.6902, + "theoretical_loss": 3.441336695849231, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1088918, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.99633526802063, + "objective/train/theoretical_loss": 3.441326855954716, + "objective/train/tokens_used": 284241376, + "theoretical_loss": 3.441326855954716, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042375757575757576, + "loss": 2.8637, + "theoretical_loss": 3.441326855954716, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373737373737377, + "loss": 2.7209, + "theoretical_loss": 3.4413170164889024, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042371717171717173, + "loss": 2.5859, + "theoretical_loss": 3.441307177451755, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236969696969697, + "loss": 2.6887, + "theoretical_loss": 3.4412973388432424, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042367676767676764, + "loss": 2.6184, + "theoretical_loss": 3.44128750066333, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236565656565657, + "loss": 2.8497, + "theoretical_loss": 3.4412776629119852, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042363636363636366, + "loss": 2.6344, + "theoretical_loss": 3.441267825589174, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236161616161616, + "loss": 2.7627, + "theoretical_loss": 3.4412579886948644, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235959595959596, + "loss": 2.5397, + "theoretical_loss": 3.441248152229022, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357575757575753, + "loss": 2.631, + "theoretical_loss": 3.4412383161916145, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235555555555556, + "loss": 2.868, + "theoretical_loss": 3.4412284805826077, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042353535353535356, + "loss": 2.6615, + "theoretical_loss": 3.441218645401969, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235151515151515, + "loss": 2.5752, + "theoretical_loss": 3.441208810649665, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349494949494947, + "loss": 2.8538, + "theoretical_loss": 3.4411989763256625, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234747474747475, + "loss": 2.9235, + "theoretical_loss": 3.441189142429928, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234545454545455, + "loss": 2.6264, + "theoretical_loss": 3.441179308962429, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343434343434345, + "loss": 2.9151, + "theoretical_loss": 3.4411694759231315, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234141414141414, + "loss": 2.8869, + "theoretical_loss": 3.4411596433120026, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339393939393936, + "loss": 2.5293, + "theoretical_loss": 3.4411498111290095, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233737373737374, + "loss": 2.7696, + "theoretical_loss": 3.4411399793741184, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233535353535354, + "loss": 2.6195, + "theoretical_loss": 3.441130148047296, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042333333333333334, + "loss": 2.7265, + "theoretical_loss": 3.44112031714851, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233131313131313, + "loss": 2.6709, + "theoretical_loss": 3.441110486677726, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232929292929293, + "loss": 2.8546, + "theoretical_loss": 3.4411006566349114, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232727272727273, + "loss": 2.6115, + "theoretical_loss": 3.441090827020033, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1089550, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5444271564483643, + "objective/train/theoretical_loss": 3.4410809978330583, + "objective/train/tokens_used": 285879776, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232525252525253, + "loss": 2.3488, + "theoretical_loss": 3.4410809978330583, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323232323232323, + "loss": 2.8, + "theoretical_loss": 3.441071169073953, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232121212121212, + "loss": 2.5789, + "theoretical_loss": 3.441061340742684, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231919191919192, + "loss": 2.7777, + "theoretical_loss": 3.4410515128392194, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231717171717172, + "loss": 2.5736, + "theoretical_loss": 3.4410416853635244, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315151515151517, + "loss": 2.78, + "theoretical_loss": 3.441031858315567, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231313131313131, + "loss": 2.6557, + "theoretical_loss": 3.4410220316953133, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042311111111111114, + "loss": 2.845, + "theoretical_loss": 3.441012205502731, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230909090909091, + "loss": 2.7278, + "theoretical_loss": 3.441002379737786, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230707070707071, + "loss": 2.6452, + "theoretical_loss": 3.4409925544004456, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042305050505050506, + "loss": 2.6479, + "theoretical_loss": 3.4409827294906767, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423030303030303, + "loss": 2.8121, + "theoretical_loss": 3.4409729050084463, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301010101010103, + "loss": 2.6022, + "theoretical_loss": 3.440963080953721, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422989898989899, + "loss": 2.6272, + "theoretical_loss": 3.440953257326467, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422969696969697, + "loss": 2.7298, + "theoretical_loss": 3.4409434341266527, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294949494949496, + "loss": 2.9831, + "theoretical_loss": 3.440933611354244, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292929292929297, + "loss": 2.6753, + "theoretical_loss": 3.4409237890092075, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229090909090909, + "loss": 2.3691, + "theoretical_loss": 3.440913967091511, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228888888888889, + "loss": 2.4764, + "theoretical_loss": 3.4409041456011202, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686868686869, + "loss": 2.7042, + "theoretical_loss": 3.4408943245380033, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284848484848485, + "loss": 2.6011, + "theoretical_loss": 3.4408845039021263, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042282828282828286, + "loss": 2.6329, + "theoretical_loss": 3.4408746836934565, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228080808080808, + "loss": 2.4699, + "theoretical_loss": 3.4408648639119606, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042278787878787877, + "loss": 2.5502, + "theoretical_loss": 3.4408550445576056, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227676767676768, + "loss": 2.9811, + "theoretical_loss": 3.4408452256303583, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1091004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3473057746887207, + "objective/train/theoretical_loss": 3.4408354071301854, + "objective/train/tokens_used": 287518176, + "theoretical_loss": 3.4408354071301854, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227474747474748, + "loss": 2.5292, + "theoretical_loss": 3.4408354071301854, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272727272727275, + "loss": 2.8198, + "theoretical_loss": 3.4408255890570545, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227070707070707, + "loss": 2.8187, + "theoretical_loss": 3.440815771410932, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042268686868686867, + "loss": 2.7224, + "theoretical_loss": 3.440805954191785, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226666666666667, + "loss": 2.8339, + "theoretical_loss": 3.4407961373995803, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226464646464647, + "loss": 2.8221, + "theoretical_loss": 3.4407863210342846, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042262626262626264, + "loss": 2.8472, + "theoretical_loss": 3.440776505095865, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226060606060606, + "loss": 2.6253, + "theoretical_loss": 3.440766689584289, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258585858585856, + "loss": 2.7356, + "theoretical_loss": 3.4407568744995234, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225656565656566, + "loss": 2.6607, + "theoretical_loss": 3.4407470598415344, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225454545454546, + "loss": 2.8589, + "theoretical_loss": 3.440737245610289, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252525252525254, + "loss": 2.7366, + "theoretical_loss": 3.4407274318057555, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225050505050505, + "loss": 2.7264, + "theoretical_loss": 3.440717618427899, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248484848484845, + "loss": 2.543, + "theoretical_loss": 3.4407078054766878, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224646464646465, + "loss": 2.6368, + "theoretical_loss": 3.440697992952088, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224444444444445, + "loss": 2.7579, + "theoretical_loss": 3.4406881808540675, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042242424242424243, + "loss": 2.8526, + "theoretical_loss": 3.440678369182592, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224040404040404, + "loss": 2.7511, + "theoretical_loss": 3.44066855793763, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042238383838383834, + "loss": 2.5729, + "theoretical_loss": 3.440658747119147, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223636363636364, + "loss": 2.8031, + "theoretical_loss": 3.440648936727111, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042234343434343437, + "loss": 2.7105, + "theoretical_loss": 3.4406391267614884, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223232323232323, + "loss": 2.7474, + "theoretical_loss": 3.4406293172222466, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223030303030303, + "loss": 2.709, + "theoretical_loss": 3.4406195081093527, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222828282828283, + "loss": 2.4564, + "theoretical_loss": 3.4406096994227733, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222626262626263, + "loss": 2.6125, + "theoretical_loss": 3.440599891162475, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1091522, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6944491863250732, + "objective/train/theoretical_loss": 3.4405900833284258, + "objective/train/tokens_used": 289156576, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042224242424242426, + "loss": 2.6919, + "theoretical_loss": 3.4405900833284258, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222222222222222, + "loss": 2.6007, + "theoretical_loss": 3.440580275920592, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220202020202017, + "loss": 2.685, + "theoretical_loss": 3.440570468938941, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221818181818182, + "loss": 2.6827, + "theoretical_loss": 3.4405606623834393, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221616161616162, + "loss": 2.9079, + "theoretical_loss": 3.4405508562540543, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042214141414141415, + "loss": 2.9371, + "theoretical_loss": 3.4405410505507534, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221212121212121, + "loss": 2.69, + "theoretical_loss": 3.440531245273503, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221010101010101, + "loss": 2.4639, + "theoretical_loss": 3.4405214404222706, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220808080808081, + "loss": 2.6265, + "theoretical_loss": 3.4405116359970225, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220606060606061, + "loss": 2.571, + "theoretical_loss": 3.4405018319977265, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204040404040404, + "loss": 2.9211, + "theoretical_loss": 3.440492028424349, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422020202020202, + "loss": 2.7833, + "theoretical_loss": 3.440482225276858, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422, + "loss": 2.8678, + "theoretical_loss": 3.440472422555219, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197979797979797, + "loss": 2.7693, + "theoretical_loss": 3.4404626202594004, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421959595959596, + "loss": 2.5943, + "theoretical_loss": 3.4404528183893692, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193939393939394, + "loss": 2.9513, + "theoretical_loss": 3.4404430169450917, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191919191919195, + "loss": 2.7032, + "theoretical_loss": 3.4404332159265354, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218989898989899, + "loss": 2.6333, + "theoretical_loss": 3.440423415333667, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187878787878786, + "loss": 2.5872, + "theoretical_loss": 3.440413615166454, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042185858585858587, + "loss": 2.7633, + "theoretical_loss": 3.440403815424864, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183838383838383, + "loss": 2.7036, + "theoretical_loss": 3.440394016108863, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042181818181818184, + "loss": 2.7831, + "theoretical_loss": 3.4403842172184183, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217979797979798, + "loss": 2.8518, + "theoretical_loss": 3.440374418753497, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042177777777777775, + "loss": 2.7134, + "theoretical_loss": 3.4403646207140666, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042175757575757577, + "loss": 2.8389, + "theoretical_loss": 3.440354823100094, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 1092471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.028099536895752, + "objective/train/theoretical_loss": 3.4403450259115465, + "objective/train/tokens_used": 290794976, + "theoretical_loss": 3.4403450259115465, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217373737373738, + "loss": 2.881, + "theoretical_loss": 3.4403450259115465, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171717171717173, + "loss": 2.7978, + "theoretical_loss": 3.4403352291483906, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216969696969697, + "loss": 2.7984, + "theoretical_loss": 3.440325432810594, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042167676767676765, + "loss": 2.9022, + "theoretical_loss": 3.4403156368981236, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042165656565656566, + "loss": 2.6549, + "theoretical_loss": 3.4403058414109458, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163636363636367, + "loss": 2.6524, + "theoretical_loss": 3.4402960463490286, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216161616161616, + "loss": 2.6661, + "theoretical_loss": 3.4402862517123394, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215959595959596, + "loss": 2.8533, + "theoretical_loss": 3.440276457500844, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157575757575754, + "loss": 2.7227, + "theoretical_loss": 3.440266663714511, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215555555555556, + "loss": 2.7392, + "theoretical_loss": 3.440256870353307, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153535353535356, + "loss": 2.8725, + "theoretical_loss": 3.4402470774171987, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215151515151515, + "loss": 2.6886, + "theoretical_loss": 3.4402372849061535, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214949494949495, + "loss": 2.5489, + "theoretical_loss": 3.4402274928201386, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214747474747475, + "loss": 2.7136, + "theoretical_loss": 3.4402177011591206, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214545454545455, + "loss": 2.5907, + "theoretical_loss": 3.4402079099230676, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042143434343434345, + "loss": 2.7192, + "theoretical_loss": 3.4401981191119466, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214141414141414, + "loss": 2.6823, + "theoretical_loss": 3.440188328725724, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042139393939393937, + "loss": 2.7649, + "theoretical_loss": 3.4401785387643677, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137373737373743, + "loss": 2.9461, + "theoretical_loss": 3.440168749227844, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213535353535354, + "loss": 2.7517, + "theoretical_loss": 3.440158960116121, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133333333333335, + "loss": 2.7532, + "theoretical_loss": 3.4401491714291654, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213131313131313, + "loss": 2.5106, + "theoretical_loss": 3.4401393831669447, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129292929292926, + "loss": 2.8114, + "theoretical_loss": 3.440129595329425, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212727272727273, + "loss": 2.6977, + "theoretical_loss": 3.440119807916575, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212525252525253, + "loss": 2.6586, + "theoretical_loss": 3.440110020928361, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1092955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4396960735321045, + "objective/train/theoretical_loss": 3.440100234364751, + "objective/train/tokens_used": 292433376, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042123232323232324, + "loss": 2.5809, + "theoretical_loss": 3.440100234364751, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212121212121212, + "loss": 2.6703, + "theoretical_loss": 3.4400904482257104, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211919191919192, + "loss": 2.6613, + "theoretical_loss": 3.4400806625112086, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211717171717172, + "loss": 2.6931, + "theoretical_loss": 3.440070877221211, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211515151515152, + "loss": 2.9531, + "theoretical_loss": 3.440061092355686, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042113131313131313, + "loss": 2.6447, + "theoretical_loss": 3.4400513079146, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211111111111111, + "loss": 2.7321, + "theoretical_loss": 3.440041523897921, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210909090909091, + "loss": 2.7478, + "theoretical_loss": 3.4400317403056153, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210707070707071, + "loss": 2.6335, + "theoretical_loss": 3.4400219571376507, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042105050505050507, + "loss": 2.8063, + "theoretical_loss": 3.4400121743939946, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.17, + "learning_rate": 0.000421030303030303, + "loss": 2.8593, + "theoretical_loss": 3.4400023920746134, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042101010101010104, + "loss": 2.7088, + "theoretical_loss": 3.439992610179475, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420989898989899, + "loss": 2.9125, + "theoretical_loss": 3.4399828287085468, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420969696969697, + "loss": 2.6198, + "theoretical_loss": 3.4399730476617956, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042094949494949496, + "loss": 2.6641, + "theoretical_loss": 3.4399632670391886, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209292929292929, + "loss": 2.5624, + "theoretical_loss": 3.439953486840693, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042090909090909093, + "loss": 2.7751, + "theoretical_loss": 3.4399437070662766, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208888888888889, + "loss": 2.6907, + "theoretical_loss": 3.439933927715906, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208686868686869, + "loss": 2.76, + "theoretical_loss": 3.439924148789549, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084848484848485, + "loss": 2.6633, + "theoretical_loss": 3.4399143702871724, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042082828282828286, + "loss": 2.7205, + "theoretical_loss": 3.4399045922087437, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208080808080808, + "loss": 2.8401, + "theoretical_loss": 3.4398948145542296, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207878787878788, + "loss": 2.9093, + "theoretical_loss": 3.4398850373235987, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207676767676768, + "loss": 2.594, + "theoretical_loss": 3.439875260516817, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042074747474747475, + "loss": 2.6663, + "theoretical_loss": 3.439865484133852, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1093966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5984678268432617, + "objective/train/theoretical_loss": 3.4398557081746715, + "objective/train/tokens_used": 294071776, + "theoretical_loss": 3.4398557081746715, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042072727272727276, + "loss": 2.8804, + "theoretical_loss": 3.4398557081746715, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207070707070707, + "loss": 2.908, + "theoretical_loss": 3.439845932639242, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068686868686867, + "loss": 2.725, + "theoretical_loss": 3.4398361575275316, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206666666666667, + "loss": 2.8103, + "theoretical_loss": 3.439826382839507, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042064646464646464, + "loss": 2.7326, + "theoretical_loss": 3.4398166085751356, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042062626262626265, + "loss": 2.9334, + "theoretical_loss": 3.4398068347343855, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206060606060606, + "loss": 2.6792, + "theoretical_loss": 3.439797061317223, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058585858585856, + "loss": 2.8061, + "theoretical_loss": 3.439787288323615, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205656565656566, + "loss": 2.7941, + "theoretical_loss": 3.43977751575353, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205454545454546, + "loss": 2.4972, + "theoretical_loss": 3.439767743606935, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042052525252525254, + "loss": 2.6196, + "theoretical_loss": 3.439757971883797, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205050505050505, + "loss": 2.8913, + "theoretical_loss": 3.4397482005840834, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042048484848484846, + "loss": 2.7028, + "theoretical_loss": 3.439738429707762, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046464646464647, + "loss": 2.7798, + "theoretical_loss": 3.439728659254799, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204444444444445, + "loss": 2.7166, + "theoretical_loss": 3.439718889225163, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042424242424244, + "loss": 2.6637, + "theoretical_loss": 3.4397091196188203, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204040404040404, + "loss": 2.8689, + "theoretical_loss": 3.4396993504357387, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038383838383835, + "loss": 2.9006, + "theoretical_loss": 3.4396895816758857, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203636363636364, + "loss": 2.7685, + "theoretical_loss": 3.439679813339229, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034343434343437, + "loss": 2.8248, + "theoretical_loss": 3.439670045425734, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042032323232323233, + "loss": 2.8191, + "theoretical_loss": 3.4396602779353707, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203030303030303, + "loss": 2.8265, + "theoretical_loss": 3.4396505108681046, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028282828282824, + "loss": 2.8745, + "theoretical_loss": 3.4396407442239036, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202626262626263, + "loss": 2.9189, + "theoretical_loss": 3.4396309780027354, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042024242424242426, + "loss": 2.9862, + "theoretical_loss": 3.4396212122045666, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1094637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2781291007995605, + "objective/train/theoretical_loss": 3.4396114468293657, + "objective/train/tokens_used": 295710176, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202222222222222, + "loss": 2.7767, + "theoretical_loss": 3.4396114468293657, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202020202020202, + "loss": 2.7256, + "theoretical_loss": 3.4396016818770994, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201818181818182, + "loss": 2.7943, + "theoretical_loss": 3.4395919173477347, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201616161616162, + "loss": 2.5563, + "theoretical_loss": 3.43958215324124, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042014141414141416, + "loss": 2.7276, + "theoretical_loss": 3.439572389557581, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201212121212121, + "loss": 2.6772, + "theoretical_loss": 3.439562626296727, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042010101010101007, + "loss": 2.7797, + "theoretical_loss": 3.439552863458644, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200808080808081, + "loss": 2.7879, + "theoretical_loss": 3.4395431010433, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200606060606061, + "loss": 2.9044, + "theoretical_loss": 3.439533339050662, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004040404040405, + "loss": 2.5636, + "theoretical_loss": 3.4395235774806983, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420020202020202, + "loss": 2.7635, + "theoretical_loss": 3.439513816333376, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 2.6369, + "theoretical_loss": 3.439504055608661, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419979797979798, + "loss": 2.71, + "theoretical_loss": 3.439494295306523, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419959595959596, + "loss": 2.7718, + "theoretical_loss": 3.439484535426928, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041993939393939394, + "loss": 2.5152, + "theoretical_loss": 3.4394747759698436, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199191919191919, + "loss": 2.5835, + "theoretical_loss": 3.439465016935238, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198989898989899, + "loss": 2.7509, + "theoretical_loss": 3.4394552583230773, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987878787878787, + "loss": 2.7328, + "theoretical_loss": 3.4394455001333295, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198585858585859, + "loss": 2.5843, + "theoretical_loss": 3.439435742365963, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983838383838383, + "loss": 2.7745, + "theoretical_loss": 3.439425985020944, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041981818181818185, + "loss": 2.7479, + "theoretical_loss": 3.43941622809824, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197979797979798, + "loss": 2.97, + "theoretical_loss": 3.439406471597819, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197777777777778, + "loss": 2.631, + "theoretical_loss": 3.4393967155196483, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975757575757577, + "loss": 2.9375, + "theoretical_loss": 3.439386959863695, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973737373737373, + "loss": 2.7821, + "theoretical_loss": 3.439377204629927, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1095318, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.385798215866089, + "objective/train/theoretical_loss": 3.4393674498183118, + "objective/train/tokens_used": 297348576, + "theoretical_loss": 3.4393674498183118, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971717171717174, + "loss": 2.5487, + "theoretical_loss": 3.4393674498183118, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196969696969697, + "loss": 2.6915, + "theoretical_loss": 3.439357695428816, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196767676767677, + "loss": 2.827, + "theoretical_loss": 3.4393479414614085, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041965656565656566, + "loss": 2.7963, + "theoretical_loss": 3.4393381879160554, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196363636363637, + "loss": 2.7697, + "theoretical_loss": 3.439328434792725, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961616161616163, + "loss": 2.906, + "theoretical_loss": 3.439318682091385, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195959595959596, + "loss": 2.6331, + "theoretical_loss": 3.4393089298120016, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195757575757576, + "loss": 3.0021, + "theoretical_loss": 3.4392991779545437, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041955555555555556, + "loss": 2.781, + "theoretical_loss": 3.439289426518978, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041953535353535357, + "loss": 2.8686, + "theoretical_loss": 3.4392796755052712, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195151515151515, + "loss": 2.8565, + "theoretical_loss": 3.439269924913393, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194949494949495, + "loss": 2.8024, + "theoretical_loss": 3.439260174743309, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194747474747475, + "loss": 2.8567, + "theoretical_loss": 3.439250424994988, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194545454545455, + "loss": 2.6297, + "theoretical_loss": 3.4392406756683958, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943434343434346, + "loss": 2.7035, + "theoretical_loss": 3.4392309267635017, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194141414141414, + "loss": 2.7123, + "theoretical_loss": 3.439221178280272, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193939393939394, + "loss": 2.7465, + "theoretical_loss": 3.439211430218675, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193737373737374, + "loss": 2.7708, + "theoretical_loss": 3.439201682578678, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193535353535354, + "loss": 2.6249, + "theoretical_loss": 3.439191935360249, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041933333333333335, + "loss": 2.6548, + "theoretical_loss": 3.439182188563354, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193131313131313, + "loss": 2.8385, + "theoretical_loss": 3.4391724421879615, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929292929292927, + "loss": 2.8896, + "theoretical_loss": 3.4391626962340394, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927272727272733, + "loss": 2.7301, + "theoretical_loss": 3.439152950701555, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192525252525253, + "loss": 2.7565, + "theoretical_loss": 3.439143205590475, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923232323232325, + "loss": 2.764, + "theoretical_loss": 3.439133460900768, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1096386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9110429286956787, + "objective/train/theoretical_loss": 3.439123716632401, + "objective/train/tokens_used": 298986976, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192121212121212, + "loss": 2.8236, + "theoretical_loss": 3.439123716632401, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919191919191916, + "loss": 2.8326, + "theoretical_loss": 3.439113972785342, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191717171717172, + "loss": 2.6891, + "theoretical_loss": 3.4391042293595584, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191515151515152, + "loss": 2.7399, + "theoretical_loss": 3.4390944863550175, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913131313131314, + "loss": 2.7732, + "theoretical_loss": 3.4390847437716863, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191111111111111, + "loss": 2.6307, + "theoretical_loss": 3.4390750016095337, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041909090909090905, + "loss": 2.9233, + "theoretical_loss": 3.439065259868527, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190707070707071, + "loss": 2.7966, + "theoretical_loss": 3.4390555185486327, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190505050505051, + "loss": 2.6796, + "theoretical_loss": 3.4390457776498193, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041903030303030303, + "loss": 2.5683, + "theoretical_loss": 3.4390360371720545, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419010101010101, + "loss": 2.5764, + "theoretical_loss": 3.439026297115305, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418989898989899, + "loss": 2.779, + "theoretical_loss": 3.4390165574795395, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418969696969697, + "loss": 2.706, + "theoretical_loss": 3.4390068182647244, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041894949494949497, + "loss": 3.1147, + "theoretical_loss": 3.4389970794708287, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189292929292929, + "loss": 2.9586, + "theoretical_loss": 3.4389873410978185, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189090909090909, + "loss": 2.8361, + "theoretical_loss": 3.4389776031456623, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188888888888889, + "loss": 2.6443, + "theoretical_loss": 3.4389678656143277, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188686868686869, + "loss": 2.8644, + "theoretical_loss": 3.4389581285037822, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041884848484848486, + "loss": 2.5699, + "theoretical_loss": 3.4389483918139927, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188282828282828, + "loss": 2.9068, + "theoretical_loss": 3.438938655544928, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880808080808083, + "loss": 3.0656, + "theoretical_loss": 3.438928919696555, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187878787878788, + "loss": 3.0264, + "theoretical_loss": 3.4389191842688414, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187676767676768, + "loss": 2.6277, + "theoretical_loss": 3.438909449261755, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874747474747475, + "loss": 2.82, + "theoretical_loss": 3.4388997146752636, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187272727272727, + "loss": 2.752, + "theoretical_loss": 3.4388899805093343, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1097742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2330455780029297, + "objective/train/theoretical_loss": 3.438880246763935, + "objective/train/tokens_used": 300625376, + "theoretical_loss": 3.438880246763935, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187070707070707, + "loss": 2.7891, + "theoretical_loss": 3.438880246763935, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186868686868687, + "loss": 2.361, + "theoretical_loss": 3.438870513439033, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186666666666667, + "loss": 2.6697, + "theoretical_loss": 3.438860780534597, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864646464646464, + "loss": 2.8041, + "theoretical_loss": 3.438851048050594, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041862626262626266, + "loss": 2.8745, + "theoretical_loss": 3.4388413159869913, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186060606060606, + "loss": 2.6424, + "theoretical_loss": 3.438831584343757, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041858585858585857, + "loss": 2.9307, + "theoretical_loss": 3.4388218531208583, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185656565656566, + "loss": 2.886, + "theoretical_loss": 3.438812122318263, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041854545454545454, + "loss": 2.8017, + "theoretical_loss": 3.4388023919359396, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041852525252525255, + "loss": 2.7151, + "theoretical_loss": 3.4387926619738547, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185050505050505, + "loss": 2.6981, + "theoretical_loss": 3.438782932431976, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848484848484846, + "loss": 2.7955, + "theoretical_loss": 3.438773203310272, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184646464646465, + "loss": 2.8388, + "theoretical_loss": 3.4387634746087103, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184444444444445, + "loss": 2.9127, + "theoretical_loss": 3.4387537463272575, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041842424242424244, + "loss": 2.9037, + "theoretical_loss": 3.438744018465882, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184040404040404, + "loss": 2.6185, + "theoretical_loss": 3.438734291024552, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041838383838383835, + "loss": 2.8023, + "theoretical_loss": 3.438724564003235, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836363636363637, + "loss": 2.3916, + "theoretical_loss": 3.438714837401897, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183434343434344, + "loss": 2.8952, + "theoretical_loss": 3.4387051112205085, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832323232323233, + "loss": 2.5776, + "theoretical_loss": 3.4386953854590354, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183030303030303, + "loss": 2.8785, + "theoretical_loss": 3.4386856601174456, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041828282828282825, + "loss": 2.6819, + "theoretical_loss": 3.4386759351957066, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182626262626263, + "loss": 2.851, + "theoretical_loss": 3.4386662106937873, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824242424242427, + "loss": 3.0631, + "theoretical_loss": 3.438656486611654, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182222222222222, + "loss": 2.7781, + "theoretical_loss": 3.4386467629492756, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1098461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9863665103912354, + "objective/train/theoretical_loss": 3.4386370397066193, + "objective/train/tokens_used": 302263776, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182020202020202, + "loss": 3.0449, + "theoretical_loss": 3.4386370397066193, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818181818181814, + "loss": 2.7751, + "theoretical_loss": 3.4386273168836525, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181616161616162, + "loss": 2.5683, + "theoretical_loss": 3.4386175944803434, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041814141414141416, + "loss": 2.9025, + "theoretical_loss": 3.4386078724966596, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181212121212121, + "loss": 2.8136, + "theoretical_loss": 3.438598150932569, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181010101010101, + "loss": 2.652, + "theoretical_loss": 3.4385884297880387, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041808080808080814, + "loss": 2.6254, + "theoretical_loss": 3.4385787090630373, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180606060606061, + "loss": 2.7287, + "theoretical_loss": 3.438568988757532, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041804040404040406, + "loss": 2.7794, + "theoretical_loss": 3.438559268871491, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418020202020202, + "loss": 2.7849, + "theoretical_loss": 3.4385495494048817, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799999999999997, + "loss": 2.9604, + "theoretical_loss": 3.4385398303576715, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041797979797979803, + "loss": 2.8752, + "theoretical_loss": 3.438530111729829, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417959595959596, + "loss": 3.0803, + "theoretical_loss": 3.4385203935213218, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041793939393939395, + "loss": 2.7713, + "theoretical_loss": 3.4385106757321173, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179191919191919, + "loss": 2.8082, + "theoretical_loss": 3.438500958362183, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178989898989899, + "loss": 2.9712, + "theoretical_loss": 3.438491241411487, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178787878787879, + "loss": 2.4782, + "theoretical_loss": 3.438481524879998, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178585858585859, + "loss": 2.7454, + "theoretical_loss": 3.438471808767683, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783838383838384, + "loss": 2.8455, + "theoretical_loss": 3.4384620930745093, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178181818181818, + "loss": 2.8259, + "theoretical_loss": 3.438452377800445, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177979797979798, + "loss": 2.7958, + "theoretical_loss": 3.4384426629454587, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177777777777778, + "loss": 2.587, + "theoretical_loss": 3.438432948509517, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177575757575758, + "loss": 2.724, + "theoretical_loss": 3.4384232344925882, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041773737373737373, + "loss": 2.6514, + "theoretical_loss": 3.438413520894641, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041771717171717174, + "loss": 2.7255, + "theoretical_loss": 3.438403807715641, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1099474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.02063250541687, + "objective/train/theoretical_loss": 3.4383940949555587, + "objective/train/tokens_used": 303902176, + "theoretical_loss": 3.4383940949555587, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176969696969697, + "loss": 2.8693, + "theoretical_loss": 3.4383940949555587, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176767676767677, + "loss": 2.5454, + "theoretical_loss": 3.43838438261436, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041765656565656567, + "loss": 2.5652, + "theoretical_loss": 3.4383746706920135, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176363636363636, + "loss": 2.6156, + "theoretical_loss": 3.438364959188487, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761616161616164, + "loss": 2.7015, + "theoretical_loss": 3.438355248103748, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175959595959596, + "loss": 2.6069, + "theoretical_loss": 3.4383455374377645, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175757575757576, + "loss": 2.8681, + "theoretical_loss": 3.4383358271905045, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041755555555555556, + "loss": 2.7187, + "theoretical_loss": 3.4383261173619353, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753535353535357, + "loss": 2.8664, + "theoretical_loss": 3.438316407952026, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041751515151515153, + "loss": 2.73, + "theoretical_loss": 3.438306698960743, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174949494949495, + "loss": 2.728, + "theoretical_loss": 3.4382969903880545, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174747474747475, + "loss": 2.9334, + "theoretical_loss": 3.4382872822339294, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745454545454545, + "loss": 2.702, + "theoretical_loss": 3.4382775744983345, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041743434343434347, + "loss": 2.7149, + "theoretical_loss": 3.4382678671812377, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174141414141414, + "loss": 2.5881, + "theoretical_loss": 3.438258160282607, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173939393939394, + "loss": 2.843, + "theoretical_loss": 3.4382484538024105, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173737373737374, + "loss": 2.8196, + "theoretical_loss": 3.438238747740616, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041735353535353535, + "loss": 2.4911, + "theoretical_loss": 3.4382290420971913, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041733333333333336, + "loss": 2.8339, + "theoretical_loss": 3.438219336872104, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173131313131313, + "loss": 2.8608, + "theoretical_loss": 3.4382096320653224, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729292929292927, + "loss": 2.8082, + "theoretical_loss": 3.4381999276768145, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172727272727273, + "loss": 2.7947, + "theoretical_loss": 3.438190223706548, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172525252525253, + "loss": 2.801, + "theoretical_loss": 3.43818052015449, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041723232323232325, + "loss": 2.9744, + "theoretical_loss": 3.43817081702061, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172121212121212, + "loss": 2.5946, + "theoretical_loss": 3.4381611143048745, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1099984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5757904052734375, + "objective/train/theoretical_loss": 3.4381514120072523, + "objective/train/tokens_used": 305540576, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041719191919191916, + "loss": 2.8271, + "theoretical_loss": 3.4381514120072523, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171717171717172, + "loss": 2.8467, + "theoretical_loss": 3.4381417101277107, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171515151515152, + "loss": 2.6924, + "theoretical_loss": 3.438132008666218, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713131313131314, + "loss": 2.6107, + "theoretical_loss": 3.438122307622742, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171111111111111, + "loss": 2.8392, + "theoretical_loss": 3.43811260699725, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041709090909090906, + "loss": 2.7676, + "theoretical_loss": 3.4381029067897115, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170707070707071, + "loss": 2.7096, + "theoretical_loss": 3.438093207000093, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170505050505051, + "loss": 3.0108, + "theoretical_loss": 3.4380835076283622, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041703030303030304, + "loss": 2.7469, + "theoretical_loss": 3.4380738086744884, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417010101010101, + "loss": 2.6818, + "theoretical_loss": 3.4380641101384386, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041698989898989895, + "loss": 2.7944, + "theoretical_loss": 3.4380544120201812, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416969696969697, + "loss": 2.6515, + "theoretical_loss": 3.438044714319684, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041694949494949497, + "loss": 2.6606, + "theoretical_loss": 3.4380350170369143, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041692929292929293, + "loss": 2.6464, + "theoretical_loss": 3.438025320171841, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169090909090909, + "loss": 2.7168, + "theoretical_loss": 3.438015623724432, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168888888888889, + "loss": 2.7547, + "theoretical_loss": 3.4380059276946544, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168686868686869, + "loss": 2.9078, + "theoretical_loss": 3.437996232082477, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041684848484848486, + "loss": 2.8253, + "theoretical_loss": 3.437986536887867, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168282828282828, + "loss": 2.8849, + "theoretical_loss": 3.437976842110793, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168080808080808, + "loss": 2.6493, + "theoretical_loss": 3.4379671477512233, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167878787878788, + "loss": 2.5279, + "theoretical_loss": 3.4379574538091244, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167676767676768, + "loss": 2.9887, + "theoretical_loss": 3.437947760284466, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041674747474747476, + "loss": 2.9071, + "theoretical_loss": 3.437938067177215, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167272727272727, + "loss": 2.6065, + "theoretical_loss": 3.43792837448734, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167070707070707, + "loss": 2.8296, + "theoretical_loss": 3.437918682214808, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 1101410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.112593650817871, + "objective/train/theoretical_loss": 3.4379089903595883, + "objective/train/tokens_used": 307178976, + "theoretical_loss": 3.4379089903595883, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166868686868687, + "loss": 2.7197, + "theoretical_loss": 3.4379089903595883, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166666666666667, + "loss": 2.448, + "theoretical_loss": 3.437899298921648, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041664646464646465, + "loss": 2.5241, + "theoretical_loss": 3.4378896079009555, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166262626262626, + "loss": 2.6552, + "theoretical_loss": 3.437879917297479, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166060606060606, + "loss": 2.8713, + "theoretical_loss": 3.437870227111186, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165858585858586, + "loss": 2.8386, + "theoretical_loss": 3.4378605373420443, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165656565656566, + "loss": 2.9404, + "theoretical_loss": 3.4378508479900227, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654545454545454, + "loss": 2.8457, + "theoretical_loss": 3.437841159055089, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041652525252525255, + "loss": 2.815, + "theoretical_loss": 3.4378314705372106, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165050505050505, + "loss": 2.6695, + "theoretical_loss": 3.437821782436356, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041648484848484847, + "loss": 2.8221, + "theoretical_loss": 3.4378120947524935, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164646464646465, + "loss": 2.7532, + "theoretical_loss": 3.4378024074855906, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041644444444444444, + "loss": 2.7296, + "theoretical_loss": 3.437792720635616, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041642424242424245, + "loss": 2.6503, + "theoretical_loss": 3.437783034202537, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164040404040404, + "loss": 2.8257, + "theoretical_loss": 3.4377733481863215, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638383838383836, + "loss": 2.7318, + "theoretical_loss": 3.4377636625869386, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041636363636363637, + "loss": 2.7181, + "theoretical_loss": 3.4377539774043555, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163434343434344, + "loss": 2.7162, + "theoretical_loss": 3.4377442926385404, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041632323232323234, + "loss": 2.7789, + "theoretical_loss": 3.4377346082894618, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163030303030303, + "loss": 2.9107, + "theoretical_loss": 3.437724924357087, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162828282828283, + "loss": 2.6606, + "theoretical_loss": 3.437715240841385, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041626262626262626, + "loss": 2.8125, + "theoretical_loss": 3.437705557742323, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162424242424243, + "loss": 2.8321, + "theoretical_loss": 3.4376958750598696, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622222222222223, + "loss": 2.7798, + "theoretical_loss": 3.437686192793992, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162020202020202, + "loss": 2.63, + "theoretical_loss": 3.4376765109446596, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1102181, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3956081867218018, + "objective/train/theoretical_loss": 3.4376668295118398, + "objective/train/tokens_used": 308817376, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161818181818182, + "loss": 2.7075, + "theoretical_loss": 3.4376668295118398, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161616161616162, + "loss": 2.5769, + "theoretical_loss": 3.4376571484955005, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614141414141417, + "loss": 2.9438, + "theoretical_loss": 3.4376474678956104, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161212121212121, + "loss": 2.5424, + "theoretical_loss": 3.437637787712137, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161010101010101, + "loss": 2.7124, + "theoretical_loss": 3.4376281079450486, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160808080808081, + "loss": 2.9563, + "theoretical_loss": 3.437618428594313, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160606060606061, + "loss": 2.7298, + "theoretical_loss": 3.4376087496598986, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041604040404040406, + "loss": 2.6504, + "theoretical_loss": 3.437599071141774, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416020202020202, + "loss": 2.652, + "theoretical_loss": 3.4375893930399064, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.18, + "learning_rate": 0.000416, + "loss": 2.491, + "theoretical_loss": 3.4375797153542647, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041597979797979804, + "loss": 2.7367, + "theoretical_loss": 3.437570038084816, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415959595959596, + "loss": 2.786, + "theoretical_loss": 3.4375603612315295, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041593939393939395, + "loss": 2.7747, + "theoretical_loss": 3.437550684794373, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159191919191919, + "loss": 2.6086, + "theoretical_loss": 3.4375410087733145, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589898989898987, + "loss": 2.7516, + "theoretical_loss": 3.437531333168322, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041587878787878793, + "loss": 2.6745, + "theoretical_loss": 3.4375216579793637, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158585858585859, + "loss": 2.6211, + "theoretical_loss": 3.4375119832064076, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041583838383838385, + "loss": 2.7523, + "theoretical_loss": 3.4375023088494228, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158181818181818, + "loss": 2.7124, + "theoretical_loss": 3.4374926349083763, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041579797979797976, + "loss": 2.6458, + "theoretical_loss": 3.4374829613832363, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157777777777778, + "loss": 2.6612, + "theoretical_loss": 3.4374732882739716, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157575757575758, + "loss": 2.4542, + "theoretical_loss": 3.43746361558055, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573737373737374, + "loss": 2.6685, + "theoretical_loss": 3.4374539433029394, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171717171717, + "loss": 2.7992, + "theoretical_loss": 3.437444271441109, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156969696969697, + "loss": 2.7239, + "theoretical_loss": 3.4374345999950253, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1103575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7954049110412598, + "objective/train/theoretical_loss": 3.437424928964658, + "objective/train/tokens_used": 310455776, + "theoretical_loss": 3.437424928964658, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156767676767677, + "loss": 2.7791, + "theoretical_loss": 3.437424928964658, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156565656565657, + "loss": 2.4751, + "theoretical_loss": 3.437415258349975, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041563636363636363, + "loss": 2.8122, + "theoretical_loss": 3.437405588150943, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156161616161616, + "loss": 2.6699, + "theoretical_loss": 3.437395918367532, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155959595959596, + "loss": 2.7476, + "theoretical_loss": 3.43738624899971, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155757575757576, + "loss": 2.7259, + "theoretical_loss": 3.437376580047444, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041555555555555557, + "loss": 2.8147, + "theoretical_loss": 3.437366911510703, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155353535353535, + "loss": 2.8619, + "theoretical_loss": 3.437357243389455, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551515151515154, + "loss": 2.6552, + "theoretical_loss": 3.437347575683668, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154949494949495, + "loss": 2.7474, + "theoretical_loss": 3.437337908393311, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154747474747475, + "loss": 2.6948, + "theoretical_loss": 3.437328241518352, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041545454545454546, + "loss": 2.7993, + "theoretical_loss": 3.4373185750587583, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154343434343434, + "loss": 2.6983, + "theoretical_loss": 3.4373089090144986, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041541414141414143, + "loss": 3.0031, + "theoretical_loss": 3.4372992433855414, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153939393939394, + "loss": 2.7338, + "theoretical_loss": 3.4372895781718547, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153737373737374, + "loss": 2.6607, + "theoretical_loss": 3.437279913373407, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535353535353535, + "loss": 2.8741, + "theoretical_loss": 3.437270248990166, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041533333333333336, + "loss": 2.5479, + "theoretical_loss": 3.4372605850221003, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153131313131313, + "loss": 2.7998, + "theoretical_loss": 3.437250921469178, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152929292929293, + "loss": 2.6287, + "theoretical_loss": 3.437241258331367, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152727272727273, + "loss": 3.0053, + "theoretical_loss": 3.437231595608636, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041525252525252525, + "loss": 2.4429, + "theoretical_loss": 3.4372219333009535, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041523232323232326, + "loss": 2.8695, + "theoretical_loss": 3.437212271408287, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152121212121212, + "loss": 2.8847, + "theoretical_loss": 3.4372026099306057, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519191919191917, + "loss": 2.9006, + "theoretical_loss": 3.4371929488678767, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1104173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6953542232513428, + "objective/train/theoretical_loss": 3.4371832882200692, + "objective/train/tokens_used": 312094176, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151717171717172, + "loss": 2.6723, + "theoretical_loss": 3.4371832882200692, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151515151515152, + "loss": 2.4675, + "theoretical_loss": 3.4371736279871508, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513131313131315, + "loss": 2.8734, + "theoretical_loss": 3.43716396816909, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151111111111111, + "loss": 2.7789, + "theoretical_loss": 3.4371543087658556, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509090909090906, + "loss": 2.6038, + "theoretical_loss": 3.437144649777415, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150707070707071, + "loss": 2.5508, + "theoretical_loss": 3.4371349912037372, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150505050505051, + "loss": 2.823, + "theoretical_loss": 3.43712533304479, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503030303030304, + "loss": 2.5943, + "theoretical_loss": 3.437115675300542, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415010101010101, + "loss": 2.7094, + "theoretical_loss": 3.4371060179709607, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498989898989896, + "loss": 2.8999, + "theoretical_loss": 3.4370963610560157, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414969696969697, + "loss": 2.6944, + "theoretical_loss": 3.437086704555674, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414949494949495, + "loss": 2.7544, + "theoretical_loss": 3.437077048469905, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041492929292929293, + "loss": 2.7003, + "theoretical_loss": 3.437067392798676, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149090909090909, + "loss": 2.9027, + "theoretical_loss": 3.4370577375419566, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041488888888888885, + "loss": 2.6362, + "theoretical_loss": 3.4370480826997136, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148686868686869, + "loss": 2.6895, + "theoretical_loss": 3.437038428271916, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041484848484848487, + "loss": 2.708, + "theoretical_loss": 3.4370287742585326, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041482828282828283, + "loss": 2.7225, + "theoretical_loss": 3.4370191206595306, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148080808080808, + "loss": 2.7043, + "theoretical_loss": 3.437009467474879, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147878787878788, + "loss": 2.8485, + "theoretical_loss": 3.4369998147045466, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147676767676768, + "loss": 2.8801, + "theoretical_loss": 3.4369901623485006, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041474747474747476, + "loss": 2.7384, + "theoretical_loss": 3.43698051040671, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147272727272727, + "loss": 2.8058, + "theoretical_loss": 3.4369708588791434, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147070707070707, + "loss": 2.717, + "theoretical_loss": 3.4369612077657687, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146868686868687, + "loss": 2.6864, + "theoretical_loss": 3.4369515570665543, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1105519, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.572352409362793, + "objective/train/theoretical_loss": 3.4369419067814686, + "objective/train/tokens_used": 313732576, + "theoretical_loss": 3.4369419067814686, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146666666666667, + "loss": 2.6173, + "theoretical_loss": 3.4369419067814686, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464646464646466, + "loss": 2.8068, + "theoretical_loss": 3.4369322569104797, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146262626262626, + "loss": 2.8917, + "theoretical_loss": 3.4369226074535564, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146060606060606, + "loss": 2.6516, + "theoretical_loss": 3.436912958410667, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041458585858585863, + "loss": 2.5644, + "theoretical_loss": 3.436903309781779, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145656565656566, + "loss": 2.8992, + "theoretical_loss": 3.436893661566862, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454545454545455, + "loss": 2.8429, + "theoretical_loss": 3.4368840137658836, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145252525252525, + "loss": 2.7801, + "theoretical_loss": 3.436874366378812, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145050505050505, + "loss": 2.7461, + "theoretical_loss": 3.4368647194056168, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448484848484853, + "loss": 2.6189, + "theoretical_loss": 3.436855072846265, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144646464646465, + "loss": 2.6174, + "theoretical_loss": 3.4368454267007253, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444444444444444, + "loss": 2.8964, + "theoretical_loss": 3.4368357809689667, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041442424242424245, + "loss": 2.6476, + "theoretical_loss": 3.436826135650957, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144040404040404, + "loss": 2.7165, + "theoretical_loss": 3.436816490746665, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143838383838384, + "loss": 2.7637, + "theoretical_loss": 3.436806846256058, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143636363636364, + "loss": 2.7099, + "theoretical_loss": 3.436797202179106, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041434343434343433, + "loss": 2.8947, + "theoretical_loss": 3.4367875585157766, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432323232323235, + "loss": 2.8986, + "theoretical_loss": 3.4367779152660383, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143030303030303, + "loss": 2.5322, + "theoretical_loss": 3.436768272429859, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828282828283, + "loss": 2.676, + "theoretical_loss": 3.4367586300072075, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041426262626262627, + "loss": 2.7839, + "theoretical_loss": 3.4367489879980524, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142424242424243, + "loss": 2.5572, + "theoretical_loss": 3.436739346402362, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422222222222224, + "loss": 3.0926, + "theoretical_loss": 3.436729705220105, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142020202020202, + "loss": 2.7156, + "theoretical_loss": 3.436720064451249, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141818181818182, + "loss": 2.7393, + "theoretical_loss": 3.4367104240957635, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1106102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.592937707901001, + "objective/train/theoretical_loss": 3.436700784153616, + "objective/train/tokens_used": 315370976, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416161616161616, + "loss": 2.7039, + "theoretical_loss": 3.436700784153616, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141414141414142, + "loss": 2.7626, + "theoretical_loss": 3.4366911446247754, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041412121212121213, + "loss": 2.6145, + "theoretical_loss": 3.43668150550921, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141010101010101, + "loss": 2.8466, + "theoretical_loss": 3.4366718668068885, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140808080808081, + "loss": 2.6139, + "theoretical_loss": 3.436662228517779, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406060606060606, + "loss": 2.8577, + "theoretical_loss": 3.4366525906418497, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041404040404040407, + "loss": 2.8423, + "theoretical_loss": 3.43664295317907, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414020202020202, + "loss": 2.3126, + "theoretical_loss": 3.4366333161294076, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414, + "loss": 2.6911, + "theoretical_loss": 3.436623679492831, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413979797979798, + "loss": 2.7116, + "theoretical_loss": 3.436614043269309, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413959595959596, + "loss": 2.669, + "theoretical_loss": 3.43660440745881, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041393939393939396, + "loss": 2.6863, + "theoretical_loss": 3.4365947720613015, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139191919191919, + "loss": 2.8445, + "theoretical_loss": 3.4365851370767535, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041389898989898987, + "loss": 2.8147, + "theoretical_loss": 3.436575502505134, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138787878787879, + "loss": 2.7289, + "theoretical_loss": 3.4365658683464106, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138585858585859, + "loss": 2.696, + "theoretical_loss": 3.4365562346005527, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383838383838385, + "loss": 2.6036, + "theoretical_loss": 3.436546601267529, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138181818181818, + "loss": 2.5584, + "theoretical_loss": 3.4365369683473066, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041379797979797977, + "loss": 2.3833, + "theoretical_loss": 3.4365273358398554, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041377777777777783, + "loss": 2.6292, + "theoretical_loss": 3.4365177037451433, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137575757575758, + "loss": 2.6632, + "theoretical_loss": 3.4365080720631394, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373737373737374, + "loss": 2.9039, + "theoretical_loss": 3.4364984407938115, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137171717171717, + "loss": 2.6055, + "theoretical_loss": 3.436488809937128, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041369696969696966, + "loss": 2.7573, + "theoretical_loss": 3.4364791794930576, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136767676767677, + "loss": 2.6695, + "theoretical_loss": 3.4364695494615694, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1106840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6775410175323486, + "objective/train/theoretical_loss": 3.4364599198426307, + "objective/train/tokens_used": 317009376, + "theoretical_loss": 3.4364599198426307, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136565656565657, + "loss": 2.709, + "theoretical_loss": 3.4364599198426307, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041363636363636364, + "loss": 2.7683, + "theoretical_loss": 3.4364502906362118, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136161616161616, + "loss": 2.5759, + "theoretical_loss": 3.4364406618422794, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135959595959596, + "loss": 2.6133, + "theoretical_loss": 3.436431033460803, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135757575757576, + "loss": 2.8469, + "theoretical_loss": 3.4364214054917515, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135555555555556, + "loss": 2.6666, + "theoretical_loss": 3.436411777935092, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353535353535353, + "loss": 2.6998, + "theoretical_loss": 3.4364021507907943, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135151515151515, + "loss": 2.6709, + "theoretical_loss": 3.4363925240588267, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134949494949495, + "loss": 2.7161, + "theoretical_loss": 3.4363828977391577, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134747474747475, + "loss": 2.8733, + "theoretical_loss": 3.436373271831755, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345454545454547, + "loss": 2.7866, + "theoretical_loss": 3.4363636463365888, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134343434343434, + "loss": 2.7132, + "theoretical_loss": 3.436354021253626, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341414141414143, + "loss": 2.6046, + "theoretical_loss": 3.436344396582837, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133939393939394, + "loss": 2.6659, + "theoretical_loss": 3.436334772324188, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133737373737374, + "loss": 2.7634, + "theoretical_loss": 3.43632514847765, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335353535353536, + "loss": 2.7563, + "theoretical_loss": 3.4363155250431894, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133333333333333, + "loss": 2.8055, + "theoretical_loss": 3.4363059020207762, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133131313131313, + "loss": 2.7074, + "theoretical_loss": 3.436296279410379, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132929292929293, + "loss": 2.4858, + "theoretical_loss": 3.436286657211965, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132727272727273, + "loss": 2.7291, + "theoretical_loss": 3.4362770354255043, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325252525252525, + "loss": 2.9576, + "theoretical_loss": 3.436267414050965, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323232323232326, + "loss": 2.5794, + "theoretical_loss": 3.4362577930883154, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132121212121212, + "loss": 2.8163, + "theoretical_loss": 3.436248172537524, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131919191919192, + "loss": 2.7367, + "theoretical_loss": 3.43623855239856, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131717171717172, + "loss": 2.5185, + "theoretical_loss": 3.4362289326713915, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1108280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.724792718887329, + "objective/train/theoretical_loss": 3.4362193133559877, + "objective/train/tokens_used": 318647776, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315151515151514, + "loss": 2.6603, + "theoretical_loss": 3.4362193133559877, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041313131313131315, + "loss": 2.6073, + "theoretical_loss": 3.4362096944523164, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131111111111111, + "loss": 2.5518, + "theoretical_loss": 3.436200075960347, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041309090909090907, + "loss": 2.8241, + "theoretical_loss": 3.4361904578800466, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130707070707071, + "loss": 2.5862, + "theoretical_loss": 3.436180840211386, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130505050505051, + "loss": 2.7115, + "theoretical_loss": 3.436171222954332, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303030303030305, + "loss": 2.6927, + "theoretical_loss": 3.436161606108855, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413010101010101, + "loss": 2.8522, + "theoretical_loss": 3.436151989674922, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298989898989896, + "loss": 2.4866, + "theoretical_loss": 3.436142373652502, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041296969696969697, + "loss": 2.6956, + "theoretical_loss": 3.4361327580415644, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412949494949495, + "loss": 2.92, + "theoretical_loss": 3.436123142842077, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041292929292929294, + "loss": 2.4502, + "theoretical_loss": 3.4361135280540087, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129090909090909, + "loss": 2.6442, + "theoretical_loss": 3.436103913677328, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128888888888889, + "loss": 2.7126, + "theoretical_loss": 3.4360942997120043, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686868686869, + "loss": 2.3436, + "theoretical_loss": 3.436084686158005, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128484848484849, + "loss": 2.5505, + "theoretical_loss": 3.4360750730153002, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041282828282828283, + "loss": 2.6202, + "theoretical_loss": 3.4360654602838574, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128080808080808, + "loss": 2.5345, + "theoretical_loss": 3.4360558479636456, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127878787878788, + "loss": 2.8297, + "theoretical_loss": 3.4360462360546333, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127676767676768, + "loss": 2.734, + "theoretical_loss": 3.43603662455679, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274747474747477, + "loss": 2.6078, + "theoretical_loss": 3.436027013470083, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127272727272727, + "loss": 2.6709, + "theoretical_loss": 3.436017402794482, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127070707070707, + "loss": 2.8164, + "theoretical_loss": 3.4360077925299555, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268686868686875, + "loss": 2.5109, + "theoretical_loss": 3.435998182676472, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126666666666667, + "loss": 2.5288, + "theoretical_loss": 3.4359885732340008, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1108943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.563303232192993, + "objective/train/theoretical_loss": 3.4359789642025094, + "objective/train/tokens_used": 320286176, + "theoretical_loss": 3.4359789642025094, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041264646464646466, + "loss": 2.7019, + "theoretical_loss": 3.4359789642025094, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126262626262626, + "loss": 2.6963, + "theoretical_loss": 3.4359693555819675, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126060606060606, + "loss": 2.6961, + "theoretical_loss": 3.435959747372343, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258585858585864, + "loss": 2.359, + "theoretical_loss": 3.4359501395736056, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125656565656566, + "loss": 2.4773, + "theoretical_loss": 3.4359405321857235, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254545454545455, + "loss": 2.4403, + "theoretical_loss": 3.4359309252086647, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125252525252525, + "loss": 2.5431, + "theoretical_loss": 3.435921318642399, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250505050505047, + "loss": 2.6144, + "theoretical_loss": 3.4359117124868948, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041248484848484853, + "loss": 2.9529, + "theoretical_loss": 3.4359021067421205, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124646464646465, + "loss": 2.8148, + "theoretical_loss": 3.435892501408045, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041244444444444445, + "loss": 2.5448, + "theoretical_loss": 3.435882896484637, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124242424242424, + "loss": 2.5726, + "theoretical_loss": 3.4358732919718653, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124040404040404, + "loss": 2.684, + "theoretical_loss": 3.435863687869699, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123838383838384, + "loss": 2.4436, + "theoretical_loss": 3.4358540841781062, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123636363636364, + "loss": 2.4198, + "theoretical_loss": 3.435844480897056, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234343434343434, + "loss": 2.6718, + "theoretical_loss": 3.435834878026516, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123232323232323, + "loss": 2.5046, + "theoretical_loss": 3.435825275566457, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123030303030303, + "loss": 2.7974, + "theoretical_loss": 3.4358156735168466, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122828282828283, + "loss": 2.5536, + "theoretical_loss": 3.4358060718776535, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122626262626263, + "loss": 2.5841, + "theoretical_loss": 3.435796470648846, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041224242424242423, + "loss": 2.6335, + "theoretical_loss": 3.4357868698303946, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041222222222222224, + "loss": 2.7844, + "theoretical_loss": 3.435777269422266, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122020202020202, + "loss": 2.6106, + "theoretical_loss": 3.4357676694244303, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121818181818182, + "loss": 2.6654, + "theoretical_loss": 3.4357580698368557, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041216161616161617, + "loss": 2.5359, + "theoretical_loss": 3.435748470659511, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1110285, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.693556308746338, + "objective/train/theoretical_loss": 3.4357388718923656, + "objective/train/tokens_used": 321924576, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121414141414141, + "loss": 2.8293, + "theoretical_loss": 3.4357388718923656, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212121212121214, + "loss": 2.6772, + "theoretical_loss": 3.4357292735353875, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121010101010101, + "loss": 2.7768, + "theoretical_loss": 3.4357196755885457, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120808080808081, + "loss": 2.6617, + "theoretical_loss": 3.435710078051809, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041206060606060606, + "loss": 2.6102, + "theoretical_loss": 3.435700480925146, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204040404040407, + "loss": 2.6763, + "theoretical_loss": 3.435690884208526, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041202020202020203, + "loss": 2.6653, + "theoretical_loss": 3.4356812879019176, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412, + "loss": 2.8014, + "theoretical_loss": 3.4356716920052897, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411979797979798, + "loss": 2.5855, + "theoretical_loss": 3.4356620965186107, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041195959595959595, + "loss": 2.7301, + "theoretical_loss": 3.435652501441849, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193939393939396, + "loss": 2.5788, + "theoretical_loss": 3.4356429067749747, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119191919191919, + "loss": 2.6587, + "theoretical_loss": 3.4356333125179557, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118989898989899, + "loss": 2.5489, + "theoretical_loss": 3.435623718670761, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118787878787879, + "loss": 2.7359, + "theoretical_loss": 3.43561412523336, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118585858585859, + "loss": 2.5192, + "theoretical_loss": 3.4356045322057205, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041183838383838386, + "loss": 2.4568, + "theoretical_loss": 3.435594939587812, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118181818181818, + "loss": 2.2525, + "theoretical_loss": 3.435585347379603, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179797979797977, + "loss": 2.5127, + "theoretical_loss": 3.4355757555810627, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117777777777778, + "loss": 2.632, + "theoretical_loss": 3.435566164192159, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117575757575758, + "loss": 2.6536, + "theoretical_loss": 3.435556573212862, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041173737373737375, + "loss": 2.7557, + "theoretical_loss": 3.4355469826431406, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117171717171717, + "loss": 2.5706, + "theoretical_loss": 3.435537392482962, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041169696969696966, + "loss": 2.6634, + "theoretical_loss": 3.4355278027322966, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041167676767676773, + "loss": 2.5651, + "theoretical_loss": 3.435518213391113, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116565656565657, + "loss": 2.8892, + "theoretical_loss": 3.4355086244593793, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 1110737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.67488956451416, + "objective/train/theoretical_loss": 3.435499035937065, + "objective/train/tokens_used": 323562976, + "theoretical_loss": 3.435499035937065, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163636363636364, + "loss": 2.5113, + "theoretical_loss": 3.435499035937065, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116161616161616, + "loss": 2.6499, + "theoretical_loss": 3.435489447824139, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041159595959595956, + "loss": 2.6372, + "theoretical_loss": 3.43547986012057, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115757575757576, + "loss": 2.6351, + "theoretical_loss": 3.4354702728263264, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115555555555556, + "loss": 2.7052, + "theoretical_loss": 3.435460685941378, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041153535353535354, + "loss": 2.7472, + "theoretical_loss": 3.435451099465693, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115151515151515, + "loss": 2.5689, + "theoretical_loss": 3.4354415133992404, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114949494949495, + "loss": 2.6688, + "theoretical_loss": 3.4354319277419894, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114747474747475, + "loss": 2.6491, + "theoretical_loss": 3.4354223424939088, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041145454545454547, + "loss": 2.7581, + "theoretical_loss": 3.435412757654967, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143434343434343, + "loss": 2.5054, + "theoretical_loss": 3.4354031732251333, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114141414141414, + "loss": 2.4835, + "theoretical_loss": 3.4353935892043768, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113939393939394, + "loss": 2.9046, + "theoretical_loss": 3.435384005592666, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113737373737374, + "loss": 2.6465, + "theoretical_loss": 3.4353744223899696, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135353535353536, + "loss": 2.6245, + "theoretical_loss": 3.4353648395962573, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113333333333333, + "loss": 2.7779, + "theoretical_loss": 3.435355257211498, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131313131313133, + "loss": 2.7055, + "theoretical_loss": 3.4353456752356593, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112929292929293, + "loss": 2.5649, + "theoretical_loss": 3.4353360936687114, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112727272727273, + "loss": 2.6001, + "theoretical_loss": 3.4353265125106223, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125252525252526, + "loss": 2.6114, + "theoretical_loss": 3.4353169317613617, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112323232323232, + "loss": 2.6417, + "theoretical_loss": 3.4353073514208985, + "tokens_seen": 1966014464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112121212121212, + "loss": 2.5888, + "theoretical_loss": 3.4352977714892017, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041119191919191924, + "loss": 2.7342, + "theoretical_loss": 3.4352881919662392, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111717171717172, + "loss": 2.6205, + "theoretical_loss": 3.435278612851981, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041115151515151515, + "loss": 2.9213, + "theoretical_loss": 3.435269034146396, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1112038, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7978675365448, + "objective/train/theoretical_loss": 3.4352594558494527, + "objective/train/tokens_used": 325201376, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041113131313131316, + "loss": 2.5659, + "theoretical_loss": 3.4352594558494527, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111111111111111, + "loss": 2.9982, + "theoretical_loss": 3.4352498779611196, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109090909090913, + "loss": 2.5429, + "theoretical_loss": 3.435240300481367, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110707070707071, + "loss": 2.6717, + "theoretical_loss": 3.4352307234101627, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041105050505050504, + "loss": 2.6222, + "theoretical_loss": 3.4352211467474767, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041103030303030305, + "loss": 2.7506, + "theoretical_loss": 3.4352115704932764, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411010101010101, + "loss": 2.6287, + "theoretical_loss": 3.4352019946475325, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410989898989899, + "loss": 2.5156, + "theoretical_loss": 3.435192419210213, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410969696969697, + "loss": 2.8124, + "theoretical_loss": 3.435182844181287, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410949494949495, + "loss": 2.526, + "theoretical_loss": 3.435173269560723, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041092929292929295, + "loss": 2.5468, + "theoretical_loss": 3.435163695348491, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109090909090909, + "loss": 2.6379, + "theoretical_loss": 3.435154121544559, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108888888888889, + "loss": 2.502, + "theoretical_loss": 3.4351445481488967, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086868686868687, + "loss": 2.6441, + "theoretical_loss": 3.435134975161473, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108484848484849, + "loss": 2.7624, + "theoretical_loss": 3.4351254025822566, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082828282828284, + "loss": 2.6438, + "theoretical_loss": 3.435115830411217, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108080808080808, + "loss": 2.577, + "theoretical_loss": 3.4351062586483225, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107878787878788, + "loss": 2.9406, + "theoretical_loss": 3.4350966872935422, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076767676767676, + "loss": 2.7491, + "theoretical_loss": 3.4350871163468457, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107474747474748, + "loss": 2.5723, + "theoretical_loss": 3.435077545808202, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041072727272727273, + "loss": 2.481, + "theoretical_loss": 3.435067975677579, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107070707070707, + "loss": 2.764, + "theoretical_loss": 3.4350584059549467, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106868686868687, + "loss": 2.8283, + "theoretical_loss": 3.435048836640274, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106666666666667, + "loss": 2.6113, + "theoretical_loss": 3.4350392677335297, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041064646464646467, + "loss": 2.5772, + "theoretical_loss": 3.435029699234683, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1112624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5079517364501953, + "objective/train/theoretical_loss": 3.435020131143703, + "objective/train/tokens_used": 326839776, + "theoretical_loss": 3.435020131143703, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106262626262626, + "loss": 2.5559, + "theoretical_loss": 3.435020131143703, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106060606060606, + "loss": 2.782, + "theoretical_loss": 3.4350105634605583, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105858585858586, + "loss": 2.5907, + "theoretical_loss": 3.435000996185219, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105656565656566, + "loss": 2.4109, + "theoretical_loss": 3.4349914293176527, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041054545454545456, + "loss": 2.792, + "theoretical_loss": 3.4349818628578292, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105252525252525, + "loss": 2.7001, + "theoretical_loss": 3.4349722968057175, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105050505050505, + "loss": 2.7647, + "theoretical_loss": 3.4349627311612867, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048484848484854, + "loss": 2.7581, + "theoretical_loss": 3.434953165924506, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104646464646465, + "loss": 2.6706, + "theoretical_loss": 3.434943601095344, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044444444444445, + "loss": 2.5606, + "theoretical_loss": 3.43493403667377, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104242424242424, + "loss": 2.7117, + "theoretical_loss": 3.4349244726597528, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041040404040404037, + "loss": 2.6172, + "theoretical_loss": 3.434914909053262, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041038383838383843, + "loss": 2.6484, + "theoretical_loss": 3.434905345854266, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103636363636364, + "loss": 2.7408, + "theoretical_loss": 3.4348957830627347, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041034343434343435, + "loss": 2.7419, + "theoretical_loss": 3.434886220678637, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103232323232323, + "loss": 2.7201, + "theoretical_loss": 3.434876658701941, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103030303030303, + "loss": 2.6781, + "theoretical_loss": 3.434867097132617, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102828282828283, + "loss": 2.8421, + "theoretical_loss": 3.4348575359706333, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102626262626263, + "loss": 2.7205, + "theoretical_loss": 3.43484797521596, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041024242424242424, + "loss": 2.538, + "theoretical_loss": 3.4348384148685644, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102222222222222, + "loss": 2.4128, + "theoretical_loss": 3.4348288549284174, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102020202020202, + "loss": 2.5698, + "theoretical_loss": 3.434819295395487, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101818181818182, + "loss": 2.4338, + "theoretical_loss": 3.434809736269743, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101616161616162, + "loss": 2.7444, + "theoretical_loss": 3.4348001775511543, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014141414141413, + "loss": 2.6185, + "theoretical_loss": 3.4347906192396893, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1113989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4740281105041504, + "objective/train/theoretical_loss": 3.4347810613353182, + "objective/train/tokens_used": 328478176, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012121212121214, + "loss": 2.591, + "theoretical_loss": 3.4347810613353182, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101010101010101, + "loss": 2.6515, + "theoretical_loss": 3.434771503838009, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100808080808081, + "loss": 2.5272, + "theoretical_loss": 3.434761946747732, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006060606060607, + "loss": 2.5469, + "theoretical_loss": 3.434752390064456, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410040404040404, + "loss": 2.5917, + "theoretical_loss": 3.4347428337881496, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002020202020203, + "loss": 2.484, + "theoretical_loss": 3.434733277918782, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 2.4248, + "theoretical_loss": 3.4347237224563223, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409979797979798, + "loss": 2.5742, + "theoretical_loss": 3.4347141674007404, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040995959595959596, + "loss": 2.4701, + "theoretical_loss": 3.4347046127520047, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993939393939397, + "loss": 2.5059, + "theoretical_loss": 3.4346950585100853, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099191919191919, + "loss": 2.5935, + "theoretical_loss": 3.43468550467495, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098989898989899, + "loss": 2.6359, + "theoretical_loss": 3.434675951246568, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098787878787879, + "loss": 2.5536, + "theoretical_loss": 3.43466639822491, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985858585858585, + "loss": 2.5583, + "theoretical_loss": 3.434656845609944, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983838383838386, + "loss": 2.7447, + "theoretical_loss": 3.434647293401639, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098181818181818, + "loss": 2.4007, + "theoretical_loss": 3.434637741599965, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097979797979798, + "loss": 2.7236, + "theoretical_loss": 3.4346281902048905, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097777777777778, + "loss": 2.721, + "theoretical_loss": 3.4346186392163847, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097575757575758, + "loss": 2.6786, + "theoretical_loss": 3.434609088634417, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040973737373737376, + "loss": 2.866, + "theoretical_loss": 3.4345995384589565, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097171717171717, + "loss": 2.7624, + "theoretical_loss": 3.4345899886899725, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969696969696967, + "loss": 2.7783, + "theoretical_loss": 3.434580439327434, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096767676767677, + "loss": 2.6089, + "theoretical_loss": 3.4345708903713104, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096565656565657, + "loss": 2.6478, + "theoretical_loss": 3.4345613418215706, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040963636363636365, + "loss": 2.5821, + "theoretical_loss": 3.4345517936781835, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1114737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8529152870178223, + "objective/train/theoretical_loss": 3.4345422459411195, + "objective/train/tokens_used": 330116576, + "theoretical_loss": 3.4345422459411195, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096161616161616, + "loss": 2.7152, + "theoretical_loss": 3.4345422459411195, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040959595959595956, + "loss": 2.5974, + "theoretical_loss": 3.434532698610347, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040957575757575763, + "loss": 2.5801, + "theoretical_loss": 3.434523151685835, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095555555555556, + "loss": 2.4955, + "theoretical_loss": 3.4345136051675524, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953535353535354, + "loss": 2.5695, + "theoretical_loss": 3.43450405905547, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095151515151515, + "loss": 2.5718, + "theoretical_loss": 3.4344945133495552, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949494949494945, + "loss": 2.6138, + "theoretical_loss": 3.4344849680497784, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094747474747475, + "loss": 2.7049, + "theoretical_loss": 3.4344754231561083, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094545454545455, + "loss": 2.7474, + "theoretical_loss": 3.4344658786685143, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040943434343434343, + "loss": 2.5975, + "theoretical_loss": 3.4344563345869656, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094141414141414, + "loss": 2.5324, + "theoretical_loss": 3.4344467909114313, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040939393939393946, + "loss": 2.495, + "theoretical_loss": 3.4344372476418807, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093737373737374, + "loss": 2.6627, + "theoretical_loss": 3.434427704778283, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040935353535353537, + "loss": 2.6484, + "theoretical_loss": 3.434418162320608, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093333333333333, + "loss": 2.5429, + "theoretical_loss": 3.434408620268824, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093131313131313, + "loss": 2.8034, + "theoretical_loss": 3.434399078622901, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929292929292935, + "loss": 2.9239, + "theoretical_loss": 3.434389537382808, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092727272727273, + "loss": 2.7245, + "theoretical_loss": 3.434379996548514, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040925252525252526, + "loss": 2.7481, + "theoretical_loss": 3.4343704561199884, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092323232323232, + "loss": 2.7297, + "theoretical_loss": 3.4343609160972006, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092121212121212, + "loss": 2.8361, + "theoretical_loss": 3.4343513764801203, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040919191919191924, + "loss": 2.7049, + "theoretical_loss": 3.4343418372687156, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091717171717172, + "loss": 2.5745, + "theoretical_loss": 3.434332298462957, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915151515151515, + "loss": 2.4716, + "theoretical_loss": 3.434322760062813, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091313131313131, + "loss": 2.6181, + "theoretical_loss": 3.434313222068253, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1116152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.517631769180298, + "objective/train/theoretical_loss": 3.4343036844792465, + "objective/train/tokens_used": 331754976, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091111111111111, + "loss": 2.6784, + "theoretical_loss": 3.4343036844792465, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040909090909090913, + "loss": 2.7579, + "theoretical_loss": 3.4342941472957627, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090707070707071, + "loss": 2.5476, + "theoretical_loss": 3.4342846105177705, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040905050505050505, + "loss": 2.5895, + "theoretical_loss": 3.4342750741452397, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409030303030303, + "loss": 2.5805, + "theoretical_loss": 3.43426553817814, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409010101010101, + "loss": 2.5214, + "theoretical_loss": 3.4342560026164395, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408989898989899, + "loss": 2.6161, + "theoretical_loss": 3.4342464674601083, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408969696969697, + "loss": 2.5312, + "theoretical_loss": 3.4342369327091156, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040894949494949494, + "loss": 2.7674, + "theoretical_loss": 3.434227398363431, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040892929292929295, + "loss": 2.8763, + "theoretical_loss": 3.4342178644230232, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089090909090909, + "loss": 2.5495, + "theoretical_loss": 3.4342083308878615, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088888888888889, + "loss": 2.8818, + "theoretical_loss": 3.4341987977579156, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088686868686869, + "loss": 2.6674, + "theoretical_loss": 3.434189265033155, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040884848484848483, + "loss": 2.8293, + "theoretical_loss": 3.4341797327135484, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040882828282828284, + "loss": 2.4307, + "theoretical_loss": 3.4341702007990658, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088080808080808, + "loss": 2.4858, + "theoretical_loss": 3.434160669289676, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087878787878788, + "loss": 2.2492, + "theoretical_loss": 3.4341511381853485, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040876767676767677, + "loss": 2.6865, + "theoretical_loss": 3.434141607486053, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087474747474748, + "loss": 2.7414, + "theoretical_loss": 3.4341320771917583, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872727272727274, + "loss": 2.5901, + "theoretical_loss": 3.434122547302434, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087070707070707, + "loss": 2.7447, + "theoretical_loss": 3.4341130178180492, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086868686868687, + "loss": 2.5586, + "theoretical_loss": 3.4341034887385735, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866666666666666, + "loss": 2.7715, + "theoretical_loss": 3.4340939600639766, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864646464646467, + "loss": 2.7034, + "theoretical_loss": 3.4340844317942274, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040862626262626263, + "loss": 2.6621, + "theoretical_loss": 3.4340749039292957, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1116806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2888708114624023, + "objective/train/theoretical_loss": 3.4340653764691496, + "objective/train/tokens_used": 333393376, + "theoretical_loss": 3.4340653764691496, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086060606060606, + "loss": 2.8104, + "theoretical_loss": 3.4340653764691496, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085858585858586, + "loss": 2.6988, + "theoretical_loss": 3.43405584941376, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656565656566, + "loss": 2.5712, + "theoretical_loss": 3.4340463227630957, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854545454545457, + "loss": 2.5299, + "theoretical_loss": 3.434036796517126, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085252525252525, + "loss": 2.4223, + "theoretical_loss": 3.43402727067582, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085050505050505, + "loss": 2.6589, + "theoretical_loss": 3.4340177452391476, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084848484848485, + "loss": 2.5612, + "theoretical_loss": 3.4340082202070774, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084646464646465, + "loss": 2.469, + "theoretical_loss": 3.43399869557958, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040844444444444446, + "loss": 2.7408, + "theoretical_loss": 3.4339891713566244, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084242424242424, + "loss": 2.6872, + "theoretical_loss": 3.433979647538179, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040840404040404037, + "loss": 2.6825, + "theoretical_loss": 3.4339701241242144, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040838383838383844, + "loss": 2.5724, + "theoretical_loss": 3.4339606011146993, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083636363636364, + "loss": 2.6956, + "theoretical_loss": 3.4339510785096037, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834343434343435, + "loss": 2.584, + "theoretical_loss": 3.4339415563088966, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083232323232323, + "loss": 2.5819, + "theoretical_loss": 3.433932034512547, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040830303030303026, + "loss": 2.5414, + "theoretical_loss": 3.4339225131205255, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040828282828282833, + "loss": 2.6592, + "theoretical_loss": 3.4339129921328, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082626262626263, + "loss": 2.8118, + "theoretical_loss": 3.4339034715493417, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040824242424242424, + "loss": 2.4732, + "theoretical_loss": 3.433893951370118, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082222222222222, + "loss": 2.5846, + "theoretical_loss": 3.4338844315951, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082020202020202, + "loss": 2.6277, + "theoretical_loss": 3.4338749122242564, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081818181818182, + "loss": 2.6535, + "theoretical_loss": 3.433865393257557, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081616161616162, + "loss": 2.8319, + "theoretical_loss": 3.433855874694971, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040814141414141414, + "loss": 2.701, + "theoretical_loss": 3.433846356536468, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081212121212121, + "loss": 2.6384, + "theoretical_loss": 3.4338368387820166, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1117380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.495161771774292, + "objective/train/theoretical_loss": 3.433827321431587, + "objective/train/tokens_used": 335031776, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081010101010101, + "loss": 2.4768, + "theoretical_loss": 3.433827321431587, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080808080808081, + "loss": 2.4908, + "theoretical_loss": 3.433817804485149, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040806060606060607, + "loss": 2.6165, + "theoretical_loss": 3.4338082879426715, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804040404040403, + "loss": 2.7464, + "theoretical_loss": 3.433798771804124, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040802020202020204, + "loss": 2.7567, + "theoretical_loss": 3.433789256069476, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408, + "loss": 2.7176, + "theoretical_loss": 3.433779740738697, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407979797979798, + "loss": 2.9107, + "theoretical_loss": 3.433770225811757, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040795959595959596, + "loss": 2.9295, + "theoretical_loss": 3.433760711288624, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079393939393939, + "loss": 2.5802, + "theoretical_loss": 3.4337511971692694, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040791919191919193, + "loss": 2.5384, + "theoretical_loss": 3.433741683453661, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078989898989899, + "loss": 2.6373, + "theoretical_loss": 3.4337321701417696, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078787878787879, + "loss": 2.9006, + "theoretical_loss": 3.433722657233564, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785858585858586, + "loss": 2.6932, + "theoretical_loss": 3.4337131447290132, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040783838383838387, + "loss": 2.5684, + "theoretical_loss": 3.4337036326280876, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078181818181818, + "loss": 2.523, + "theoretical_loss": 3.433694120930756, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077979797979798, + "loss": 2.8221, + "theoretical_loss": 3.4336846096369893, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077777777777778, + "loss": 2.341, + "theoretical_loss": 3.433675098746755, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040775757575757575, + "loss": 2.5631, + "theoretical_loss": 3.4336655882600238, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040773737373737376, + "loss": 2.6766, + "theoretical_loss": 3.433656078176765, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077171717171717, + "loss": 2.6299, + "theoretical_loss": 3.4336465684969477, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769696969696973, + "loss": 2.6926, + "theoretical_loss": 3.4336370592205423, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076767676767677, + "loss": 2.5324, + "theoretical_loss": 3.4336275503475178, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076565656565657, + "loss": 2.6586, + "theoretical_loss": 3.4336180418778435, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763636363636365, + "loss": 2.6364, + "theoretical_loss": 3.4336085338114897, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076161616161616, + "loss": 2.6143, + "theoretical_loss": 3.433599026148425, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1118619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.304875612258911, + "objective/train/theoretical_loss": 3.4335895188886187, + "objective/train/tokens_used": 336670176, + "theoretical_loss": 3.4335895188886187, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075959595959596, + "loss": 2.8207, + "theoretical_loss": 3.4335895188886187, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075757575757576, + "loss": 2.578, + "theoretical_loss": 3.433580012032042, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075555555555556, + "loss": 2.5663, + "theoretical_loss": 3.433570505578663, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040753535353535355, + "loss": 2.8436, + "theoretical_loss": 3.433560999528451, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075151515151515, + "loss": 2.4786, + "theoretical_loss": 3.433551493881377, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074949494949495, + "loss": 2.6323, + "theoretical_loss": 3.4335419886374092, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747474747474747, + "loss": 2.722, + "theoretical_loss": 3.433532483796518, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074545454545455, + "loss": 2.5927, + "theoretical_loss": 3.433522979358673, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743434343434344, + "loss": 2.7983, + "theoretical_loss": 3.433513475323843, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074141414141414, + "loss": 2.5855, + "theoretical_loss": 3.433503971691998, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073939393939394, + "loss": 2.7786, + "theoretical_loss": 3.433494468463107, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073737373737374, + "loss": 2.67, + "theoretical_loss": 3.4334849656371405, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073535353535354, + "loss": 2.5466, + "theoretical_loss": 3.4334754632140676, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040733333333333333, + "loss": 2.4499, + "theoretical_loss": 3.433465961193858, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073131313131313, + "loss": 2.646, + "theoretical_loss": 3.4334564595764814, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072929292929293, + "loss": 2.9071, + "theoretical_loss": 3.4334469583619067, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072727272727273, + "loss": 2.5047, + "theoretical_loss": 3.433437457550104, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040725252525252527, + "loss": 2.3961, + "theoretical_loss": 3.4334279571410438, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072323232323232, + "loss": 2.6198, + "theoretical_loss": 3.4334184571346933, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072121212121212, + "loss": 2.63, + "theoretical_loss": 3.4334089575310247, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040719191919191925, + "loss": 2.6339, + "theoretical_loss": 3.433399458330006, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071717171717172, + "loss": 2.4784, + "theoretical_loss": 3.433389959531607, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715151515151516, + "loss": 2.7807, + "theoretical_loss": 3.433380461135798, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071313131313131, + "loss": 2.7214, + "theoretical_loss": 3.433370963142548, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071111111111111, + "loss": 2.8174, + "theoretical_loss": 3.433361465551827, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1119354, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841550350189209, + "objective/train/theoretical_loss": 3.4333519683636036, + "objective/train/tokens_used": 338308576, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709090909090914, + "loss": 2.6936, + "theoretical_loss": 3.4333519683636036, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070707070707071, + "loss": 2.5794, + "theoretical_loss": 3.433342471577849, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705050505050505, + "loss": 2.7771, + "theoretical_loss": 3.4333329751945314, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407030303030303, + "loss": 2.7444, + "theoretical_loss": 3.433323479213622, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407010101010101, + "loss": 2.5691, + "theoretical_loss": 3.4333139836350886, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698989898989903, + "loss": 2.6519, + "theoretical_loss": 3.433304488458902, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406969696969697, + "loss": 2.7357, + "theoretical_loss": 3.433294993685031, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694949494949495, + "loss": 2.6758, + "theoretical_loss": 3.4332854993134463, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069292929292929, + "loss": 2.5607, + "theoretical_loss": 3.4332760053441174, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069090909090909, + "loss": 2.576, + "theoretical_loss": 3.4332665117770125, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068888888888889, + "loss": 2.7002, + "theoretical_loss": 3.433257018612103, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068686868686869, + "loss": 2.747, + "theoretical_loss": 3.433247525849358, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040684848484848484, + "loss": 2.8522, + "theoretical_loss": 3.433238033488747, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682828282828285, + "loss": 2.6813, + "theoretical_loss": 3.433228541530239, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068080808080808, + "loss": 2.6897, + "theoretical_loss": 3.433219049973805, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067878787878788, + "loss": 2.5501, + "theoretical_loss": 3.4332095588194136, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067676767676768, + "loss": 2.4729, + "theoretical_loss": 3.433200068067035, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674747474747473, + "loss": 2.6656, + "theoretical_loss": 3.4331905777166387, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672727272727274, + "loss": 2.5903, + "theoretical_loss": 3.4331810877681948, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067070707070707, + "loss": 2.7306, + "theoretical_loss": 3.433171598221672, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066868686868687, + "loss": 2.4783, + "theoretical_loss": 3.43316210907704, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040666666666666667, + "loss": 2.7976, + "theoretical_loss": 3.43315262033427, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066464646464647, + "loss": 2.6885, + "theoretical_loss": 3.4331431319933303, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662626262626264, + "loss": 2.499, + "theoretical_loss": 3.4331336440541915, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066060606060606, + "loss": 2.6433, + "theoretical_loss": 3.433124156516823, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 1120511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7251203060150146, + "objective/train/theoretical_loss": 3.4331146693811934, + "objective/train/tokens_used": 339946976, + "theoretical_loss": 3.4331146693811934, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065858585858586, + "loss": 2.5408, + "theoretical_loss": 3.4331146693811934, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040656565656565656, + "loss": 2.8884, + "theoretical_loss": 3.4331051826472736, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040654545454545457, + "loss": 2.3817, + "theoretical_loss": 3.433095696315033, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652525252525253, + "loss": 2.7238, + "theoretical_loss": 3.4330862103844417, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065050505050505, + "loss": 2.7844, + "theoretical_loss": 3.4330767248554688, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064848484848485, + "loss": 2.7799, + "theoretical_loss": 3.433067239728084, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064646464646465, + "loss": 2.5905, + "theoretical_loss": 3.4330577550022574, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644444444444446, + "loss": 2.6872, + "theoretical_loss": 3.4330482706779586, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064242424242424, + "loss": 2.5083, + "theoretical_loss": 3.4330387867551573, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064040404040404, + "loss": 2.6541, + "theoretical_loss": 3.4330293032338233, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063838383838384, + "loss": 2.4347, + "theoretical_loss": 3.433019820113926, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063636363636364, + "loss": 2.5209, + "theoretical_loss": 3.4330103373954355, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040634343434343436, + "loss": 2.5635, + "theoretical_loss": 3.4330008550783213, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063232323232323, + "loss": 2.9054, + "theoretical_loss": 3.4329913731625536, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630303030303027, + "loss": 2.49, + "theoretical_loss": 3.4329818916481014, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040628282828282834, + "loss": 3.1041, + "theoretical_loss": 3.4329724105349353, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062626262626263, + "loss": 2.6499, + "theoretical_loss": 3.432962929823024, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624242424242425, + "loss": 2.7937, + "theoretical_loss": 3.432953449512338, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062222222222222, + "loss": 2.6403, + "theoretical_loss": 3.432943969602847, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040620202020202016, + "loss": 2.8389, + "theoretical_loss": 3.432934490094521, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618181818181823, + "loss": 2.7806, + "theoretical_loss": 3.4329250109873293, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061616161616162, + "loss": 2.8095, + "theoretical_loss": 3.4329155322812417, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614141414141414, + "loss": 2.5243, + "theoretical_loss": 3.432906053976228, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061212121212121, + "loss": 2.7121, + "theoretical_loss": 3.432896576072258, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610101010101006, + "loss": 2.6449, + "theoretical_loss": 3.432887098569301, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1121670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2210757732391357, + "objective/train/theoretical_loss": 3.4328776214673278, + "objective/train/tokens_used": 341585376, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060808080808081, + "loss": 2.8907, + "theoretical_loss": 3.4328776214673278, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060606060606061, + "loss": 2.5579, + "theoretical_loss": 3.432868144766308, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040604040404040403, + "loss": 2.7188, + "theoretical_loss": 3.432858668466211, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406020202020202, + "loss": 2.6193, + "theoretical_loss": 3.432849192567006, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040600000000000006, + "loss": 2.8841, + "theoretical_loss": 3.4328397170686635, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405979797979798, + "loss": 2.7199, + "theoretical_loss": 3.4328302419711534, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595959595959597, + "loss": 2.6318, + "theoretical_loss": 3.4328207672744453, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040593939393939393, + "loss": 2.6874, + "theoretical_loss": 3.432811292978509, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059191919191919, + "loss": 2.753, + "theoretical_loss": 3.432801819083315, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040589898989898995, + "loss": 2.4945, + "theoretical_loss": 3.4327923455888314, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058787878787879, + "loss": 2.7391, + "theoretical_loss": 3.4327828724950296, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040585858585858586, + "loss": 2.509, + "theoretical_loss": 3.432773399801879, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058383838383838, + "loss": 2.5471, + "theoretical_loss": 3.432763927509349, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581818181818183, + "loss": 2.8927, + "theoretical_loss": 3.4327544556174097, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040579797979797984, + "loss": 2.6224, + "theoretical_loss": 3.432744984126031, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057777777777778, + "loss": 2.806, + "theoretical_loss": 3.4327355130351824, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575757575757576, + "loss": 2.6272, + "theoretical_loss": 3.4327260423448345, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057373737373737, + "loss": 2.3699, + "theoretical_loss": 3.4327165720549564, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171717171717, + "loss": 2.708, + "theoretical_loss": 3.4327071021655176, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569696969696973, + "loss": 2.4559, + "theoretical_loss": 3.432697632676489, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056767676767677, + "loss": 2.6198, + "theoretical_loss": 3.43268816358784, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565656565656565, + "loss": 2.5452, + "theoretical_loss": 3.4326786948995407, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040563636363636366, + "loss": 2.843, + "theoretical_loss": 3.43266922661156, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056161616161616, + "loss": 2.7317, + "theoretical_loss": 3.4326597587238687, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040559595959595963, + "loss": 2.6288, + "theoretical_loss": 3.432650291236436, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1122413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0547091960906982, + "objective/train/theoretical_loss": 3.4326408241492326, + "objective/train/tokens_used": 343223776, + "theoretical_loss": 3.4326408241492326, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055757575757576, + "loss": 2.6299, + "theoretical_loss": 3.4326408241492326, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040555555555555554, + "loss": 2.747, + "theoretical_loss": 3.4326313574622276, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553535353535355, + "loss": 2.764, + "theoretical_loss": 3.432621891175391, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055151515151515, + "loss": 2.7035, + "theoretical_loss": 3.432612425288693, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054949494949495, + "loss": 2.6031, + "theoretical_loss": 3.4326029598021033, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054747474747475, + "loss": 2.843, + "theoretical_loss": 3.432593494715592, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054545454545455, + "loss": 2.7244, + "theoretical_loss": 3.432584030029128, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543434343434344, + "loss": 2.7839, + "theoretical_loss": 3.4325745657426827, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054141414141414, + "loss": 2.6955, + "theoretical_loss": 3.432565101856224, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053939393939394, + "loss": 2.7448, + "theoretical_loss": 3.4325556383697244, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537373737373737, + "loss": 2.7343, + "theoretical_loss": 3.432546175283152, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053535353535354, + "loss": 2.5757, + "theoretical_loss": 3.4325367125964767, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533333333333334, + "loss": 2.4938, + "theoretical_loss": 3.4325272503096693, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053131313131313, + "loss": 2.8022, + "theoretical_loss": 3.4325177884226985, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052929292929293, + "loss": 2.7456, + "theoretical_loss": 3.4325083269355354, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052727272727273, + "loss": 2.5784, + "theoretical_loss": 3.4324988658481494, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052525252525253, + "loss": 2.848, + "theoretical_loss": 3.43248940516051, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040523232323232323, + "loss": 2.7586, + "theoretical_loss": 3.432479944872588, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052121212121212, + "loss": 2.7821, + "theoretical_loss": 3.4324704849843526, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051919191919192, + "loss": 2.6329, + "theoretical_loss": 3.4324610254957744, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051717171717172, + "loss": 2.7577, + "theoretical_loss": 3.4324515664068223, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040515151515151517, + "loss": 2.799, + "theoretical_loss": 3.432442107717467, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051313131313131, + "loss": 2.8611, + "theoretical_loss": 3.4324326494276782, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051111111111111, + "loss": 2.5285, + "theoretical_loss": 3.432423191537426, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040509090909090915, + "loss": 2.6692, + "theoretical_loss": 3.4324137340466807, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1123893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3100128173828125, + "objective/train/theoretical_loss": 3.432404276955411, + "objective/train/tokens_used": 344862176, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050707070707071, + "loss": 2.791, + "theoretical_loss": 3.432404276955411, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040505050505050506, + "loss": 2.6569, + "theoretical_loss": 3.432394820263588, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405030303030303, + "loss": 2.5506, + "theoretical_loss": 3.432385363971181, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040501010101010097, + "loss": 2.568, + "theoretical_loss": 3.4323759080781606, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040498989898989904, + "loss": 2.7298, + "theoretical_loss": 3.432366452584496, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404969696969697, + "loss": 2.7822, + "theoretical_loss": 3.4323569974901575, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494949494949495, + "loss": 2.6326, + "theoretical_loss": 3.4323475427951156, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049292929292929, + "loss": 2.8223, + "theoretical_loss": 3.4323380884993395, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049090909090909, + "loss": 2.602, + "theoretical_loss": 3.4323286346027997, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488888888888893, + "loss": 2.7207, + "theoretical_loss": 3.4323191811054654, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048686868686869, + "loss": 2.5488, + "theoretical_loss": 3.432309728007307, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484848484848484, + "loss": 2.4879, + "theoretical_loss": 3.432300275308295, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048282828282828, + "loss": 2.6236, + "theoretical_loss": 3.4322908230083984, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048080808080808, + "loss": 2.6004, + "theoretical_loss": 3.432281371107588, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047878787878788, + "loss": 2.97, + "theoretical_loss": 3.4322719196058338, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047676767676768, + "loss": 2.5354, + "theoretical_loss": 3.4322624685031053, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040474747474747474, + "loss": 2.7347, + "theoretical_loss": 3.4322530177993724, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040472727272727275, + "loss": 2.5054, + "theoretical_loss": 3.4322435674946057, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047070707070707, + "loss": 2.761, + "theoretical_loss": 3.4322341175887745, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046868686868687, + "loss": 2.5689, + "theoretical_loss": 3.4322246680818496, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040466666666666667, + "loss": 2.4813, + "theoretical_loss": 3.4322152189738, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040464646464646463, + "loss": 2.7617, + "theoretical_loss": 3.4322057702645967, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462626262626264, + "loss": 2.8087, + "theoretical_loss": 3.4321963219542098, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046060606060606, + "loss": 2.6481, + "theoretical_loss": 3.4321868740426082, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045858585858586, + "loss": 2.9988, + "theoretical_loss": 3.432177426529763, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1124511, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.707653045654297, + "objective/train/theoretical_loss": 3.4321679794156434, + "objective/train/tokens_used": 346500576, + "theoretical_loss": 3.4321679794156434, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040456565656565657, + "loss": 2.8713, + "theoretical_loss": 3.4321679794156434, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045454545454546, + "loss": 2.567, + "theoretical_loss": 3.4321585327002198, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040452525252525253, + "loss": 2.5797, + "theoretical_loss": 3.432149086383462, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045050505050505, + "loss": 2.7903, + "theoretical_loss": 3.4321396404653406, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044848484848485, + "loss": 2.6355, + "theoretical_loss": 3.4321301949458256, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446464646464646, + "loss": 2.7075, + "theoretical_loss": 3.432120749824886, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040444444444444447, + "loss": 2.8916, + "theoretical_loss": 3.4321113051024934, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044242424242424, + "loss": 2.8469, + "theoretical_loss": 3.4321018607786167, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044040404040404, + "loss": 2.7592, + "theoretical_loss": 3.4320924168532256, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043838383838384, + "loss": 2.4816, + "theoretical_loss": 3.432082973326292, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040436363636363635, + "loss": 2.6919, + "theoretical_loss": 3.432073530197784, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040434343434343436, + "loss": 2.5821, + "theoretical_loss": 3.4320640874676727, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043232323232323, + "loss": 2.459, + "theoretical_loss": 3.4320546451359277, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430303030303033, + "loss": 2.6799, + "theoretical_loss": 3.4320452032025193, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828282828283, + "loss": 2.7062, + "theoretical_loss": 3.4320357616674175, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042626262626263, + "loss": 2.5152, + "theoretical_loss": 3.4320263205305928, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424242424242425, + "loss": 2.5826, + "theoretical_loss": 3.4320168797920143, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042222222222222, + "loss": 2.57, + "theoretical_loss": 3.432007439451653, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042020202020202, + "loss": 2.8676, + "theoretical_loss": 3.4319979995094787, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041818181818182, + "loss": 2.7375, + "theoretical_loss": 3.431988559965461, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041616161616162, + "loss": 2.8017, + "theoretical_loss": 3.431979120819571, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040414141414141415, + "loss": 2.7551, + "theoretical_loss": 3.4319696820717778, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041212121212121, + "loss": 2.5862, + "theoretical_loss": 3.431960243722052, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041010101010101, + "loss": 2.6215, + "theoretical_loss": 3.4319508057703634, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040808080808081, + "loss": 2.5913, + "theoretical_loss": 3.4319413682166826, + "tokens_seen": 1989214208 + }, + { + "debugging/Self-BLEU-5": 0.4683363022964717, + "debugging/distinct-1-grams": 0.8034092501484725, + "debugging/distinct-2-grams": 0.9641002767459668, + "debugging/entropy-1-grams": 6.198537273223117, + "debugging/entropy-2-grams": 7.019141508423205, + "debugging/length": 468.3529411764706, + "debugging/num_segments": 17, + "epoch": 0.2, + "objective/train/docs_used": 1125297, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6886789798736572, + "objective/train/theoretical_loss": 3.431931931060979, + "objective/train/tokens_used": 348138976, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040606060606061, + "loss": 2.6278, + "theoretical_loss": 3.431931931060979, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404040404040404, + "loss": 2.8335, + "theoretical_loss": 3.4319224943032234, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404020202020202, + "loss": 2.758, + "theoretical_loss": 3.431913057943386, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404, + "loss": 2.746, + "theoretical_loss": 3.4319036219814363, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403979797979798, + "loss": 2.7227, + "theoretical_loss": 3.4318941864173445, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403959595959596, + "loss": 2.4946, + "theoretical_loss": 3.4318847512510806, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393939393939393, + "loss": 2.7424, + "theoretical_loss": 3.431875316482615, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039191919191919, + "loss": 2.669, + "theoretical_loss": 3.4318658821119183, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389898989898995, + "loss": 2.8623, + "theoretical_loss": 3.43185644813896, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038787878787879, + "loss": 2.4418, + "theoretical_loss": 3.43184701456371, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040385858585858587, + "loss": 2.6542, + "theoretical_loss": 3.431837581386139, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038383838383838, + "loss": 2.701, + "theoretical_loss": 3.4318281486062174, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038181818181818, + "loss": 2.6742, + "theoretical_loss": 3.4318187162239147, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040379797979797985, + "loss": 2.5197, + "theoretical_loss": 3.4318092842392005, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037777777777778, + "loss": 2.507, + "theoretical_loss": 3.4317998526520466, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375757575757576, + "loss": 2.8868, + "theoretical_loss": 3.4317904214624217, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037373737373737, + "loss": 2.6804, + "theoretical_loss": 3.4317809906702967, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040371717171717173, + "loss": 2.6729, + "theoretical_loss": 3.4317715602756413, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040369696969696974, + "loss": 2.4765, + "theoretical_loss": 3.4317621302784262, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036767676767677, + "loss": 2.6587, + "theoretical_loss": 3.4317527006786217, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040365656565656565, + "loss": 2.5383, + "theoretical_loss": 3.431743271476197, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036363636363636, + "loss": 2.7055, + "theoretical_loss": 3.4317338426711226, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036161616161616, + "loss": 2.4412, + "theoretical_loss": 3.4317244142633694, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359595959595963, + "loss": 2.6303, + "theoretical_loss": 3.4317149862529073, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035757575757576, + "loss": 2.7678, + "theoretical_loss": 3.4317055586397056, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1126404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3860960006713867, + "objective/train/theoretical_loss": 3.4316961314237355, + "objective/train/tokens_used": 349777376, + "theoretical_loss": 3.4316961314237355, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355555555555555, + "loss": 2.8394, + "theoretical_loss": 3.4316961314237355, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040353535353535356, + "loss": 2.7256, + "theoretical_loss": 3.431686704604967, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035151515151515, + "loss": 2.5348, + "theoretical_loss": 3.4316772781833693, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034949494949495, + "loss": 2.6026, + "theoretical_loss": 3.431667852158914, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034747474747475, + "loss": 2.5914, + "theoretical_loss": 3.4316584265315706, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040345454545454544, + "loss": 2.7001, + "theoretical_loss": 3.43164900130131, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343434343434345, + "loss": 2.5079, + "theoretical_loss": 3.4316395764681014, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034141414141414, + "loss": 2.499, + "theoretical_loss": 3.4316301520319152, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033939393939394, + "loss": 2.5509, + "theoretical_loss": 3.4316207279927218, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033737373737374, + "loss": 2.6126, + "theoretical_loss": 3.4316113043504917, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033535353535354, + "loss": 2.4595, + "theoretical_loss": 3.4316018811051947, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040333333333333334, + "loss": 2.6473, + "theoretical_loss": 3.4315924582568016, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033131313131313, + "loss": 2.5763, + "theoretical_loss": 3.4315830358052817, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032929292929293, + "loss": 2.5224, + "theoretical_loss": 3.431573613750606, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040327272727272727, + "loss": 2.6076, + "theoretical_loss": 3.431564192092744, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032525252525253, + "loss": 2.6282, + "theoretical_loss": 3.431554770831667, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040323232323232324, + "loss": 2.5781, + "theoretical_loss": 3.4315453499673443, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032121212121212, + "loss": 2.7832, + "theoretical_loss": 3.4315359294997467, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031919191919192, + "loss": 2.6047, + "theoretical_loss": 3.431526509428844, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031717171717172, + "loss": 2.6477, + "theoretical_loss": 3.431517089754607, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040315151515151517, + "loss": 2.6119, + "theoretical_loss": 3.4315076704770053, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040313131313131313, + "loss": 2.6704, + "theoretical_loss": 3.4314982515960093, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031111111111111, + "loss": 2.9986, + "theoretical_loss": 3.4314888331115894, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030909090909091, + "loss": 2.4957, + "theoretical_loss": 3.4314794150237162, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030707070707071, + "loss": 2.6992, + "theoretical_loss": 3.4314699973323597, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1126980, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7978928089141846, + "objective/train/theoretical_loss": 3.4314605800374895, + "objective/train/tokens_used": 351415776, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305050505050506, + "loss": 2.685, + "theoretical_loss": 3.4314605800374895, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403030303030303, + "loss": 2.5625, + "theoretical_loss": 3.4314511631390774, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403010101010101, + "loss": 2.5429, + "theoretical_loss": 3.431441746637092, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298989898989904, + "loss": 2.6394, + "theoretical_loss": 3.4314323305315044, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402969696969697, + "loss": 2.7991, + "theoretical_loss": 3.431422914822285, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294949494949496, + "loss": 2.6242, + "theoretical_loss": 3.431413499509404, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029292929292929, + "loss": 2.9248, + "theoretical_loss": 3.431404084592831, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290909090909087, + "loss": 2.3903, + "theoretical_loss": 3.4313946700725375, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040288888888888894, + "loss": 2.3707, + "theoretical_loss": 3.4313852559484928, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686868686869, + "loss": 2.525, + "theoretical_loss": 3.4313758422206675, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284848484848485, + "loss": 2.6904, + "theoretical_loss": 3.4313664288890324, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028282828282828, + "loss": 2.7228, + "theoretical_loss": 3.431357015953557, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040280808080808076, + "loss": 2.6244, + "theoretical_loss": 3.4313476034142116, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040278787878787883, + "loss": 2.8185, + "theoretical_loss": 3.431338191270967, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027676767676768, + "loss": 2.9305, + "theoretical_loss": 3.4313287795237932, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040274747474747474, + "loss": 2.6676, + "theoretical_loss": 3.4313193681726606, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027272727272727, + "loss": 2.6057, + "theoretical_loss": 3.4313099572175405, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027070707070707, + "loss": 2.7653, + "theoretical_loss": 3.4313005466584015, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026868686868687, + "loss": 2.6795, + "theoretical_loss": 3.431291136495215, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026666666666667, + "loss": 2.472, + "theoretical_loss": 3.4312817267279505, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040264646464646464, + "loss": 2.785, + "theoretical_loss": 3.431272317356579, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026262626262626, + "loss": 2.7195, + "theoretical_loss": 3.431262908381071, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026060606060606, + "loss": 2.7538, + "theoretical_loss": 3.4312534998013966, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025858585858586, + "loss": 2.7386, + "theoretical_loss": 3.431244091617526, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256565656565657, + "loss": 2.6347, + "theoretical_loss": 3.4312346838294294, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1128284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.593355178833008, + "objective/train/theoretical_loss": 3.431225276437078, + "objective/train/tokens_used": 353054176, + "theoretical_loss": 3.431225276437078, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040254545454545453, + "loss": 2.5819, + "theoretical_loss": 3.431225276437078, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252525252525254, + "loss": 2.6981, + "theoretical_loss": 3.431215869440441, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040250505050505055, + "loss": 2.4646, + "theoretical_loss": 3.431206462839489, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024848484848485, + "loss": 2.7572, + "theoretical_loss": 3.4311970566341934, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040246464646464646, + "loss": 2.5427, + "theoretical_loss": 3.4311876508245227, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024444444444444, + "loss": 2.4181, + "theoretical_loss": 3.4311782454104494, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040242424242424243, + "loss": 2.6887, + "theoretical_loss": 3.4311688403919423, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040240404040404044, + "loss": 2.7391, + "theoretical_loss": 3.431159435768972, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023838383838384, + "loss": 2.4654, + "theoretical_loss": 3.4311500315415095, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236363636363636, + "loss": 2.9443, + "theoretical_loss": 3.431140627709525, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040234343434343437, + "loss": 2.5955, + "theoretical_loss": 3.4311312242729883, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023232323232323, + "loss": 3.0277, + "theoretical_loss": 3.43112182123187, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230303030303034, + "loss": 2.5234, + "theoretical_loss": 3.431112418586141, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022828282828283, + "loss": 2.6175, + "theoretical_loss": 3.431103016335771, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040226262626262625, + "loss": 2.4859, + "theoretical_loss": 3.4310936144807314, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040224242424242426, + "loss": 2.4116, + "theoretical_loss": 3.4310842130209913, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022222222222222, + "loss": 2.7629, + "theoretical_loss": 3.431074811956522, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220202020202023, + "loss": 2.6529, + "theoretical_loss": 3.4310654112872934, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021818181818182, + "loss": 2.7032, + "theoretical_loss": 3.4310560110132764, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021616161616162, + "loss": 2.6024, + "theoretical_loss": 3.431046611134441, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214141414141415, + "loss": 2.7227, + "theoretical_loss": 3.4310372116507573, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021212121212121, + "loss": 2.5678, + "theoretical_loss": 3.4310278125621965, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021010101010101, + "loss": 2.6036, + "theoretical_loss": 3.431018413868729, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020808080808081, + "loss": 2.6384, + "theoretical_loss": 3.4310090155703246, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020606060606061, + "loss": 2.6316, + "theoretical_loss": 3.430999617666954, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1128921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.690654993057251, + "objective/train/theoretical_loss": 3.430990220158588, + "objective/train/tokens_used": 354692576, + "theoretical_loss": 3.430990220158588, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040204040404040405, + "loss": 2.7516, + "theoretical_loss": 3.430990220158588, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402020202020202, + "loss": 2.7522, + "theoretical_loss": 3.4309808230451955, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402, + "loss": 2.5698, + "theoretical_loss": 3.430971426326749, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401979797979798, + "loss": 2.6108, + "theoretical_loss": 3.4309620300032178, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401959595959596, + "loss": 2.3355, + "theoretical_loss": 3.430952634074573, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040193939393939394, + "loss": 2.4989, + "theoretical_loss": 3.4309432385407836, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019191919191919, + "loss": 2.5484, + "theoretical_loss": 3.4309338434018217, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018989898989899, + "loss": 2.5303, + "theoretical_loss": 3.430924448657657, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018787878787879, + "loss": 2.5248, + "theoretical_loss": 3.43091505430826, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018585858585859, + "loss": 2.6286, + "theoretical_loss": 3.4309056603536012, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040183838383838383, + "loss": 2.5381, + "theoretical_loss": 3.430896266793651, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018181818181818, + "loss": 2.3822, + "theoretical_loss": 3.43088687362838, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179797979797985, + "loss": 2.4635, + "theoretical_loss": 3.4308774808577587, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017777777777778, + "loss": 2.6752, + "theoretical_loss": 3.4308680884817573, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040175757575757577, + "loss": 2.7409, + "theoretical_loss": 3.430858696500346, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017373737373737, + "loss": 2.627, + "theoretical_loss": 3.430849304913496, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017171717171717, + "loss": 2.561, + "theoretical_loss": 3.4308399137211776, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040169696969696975, + "loss": 2.6353, + "theoretical_loss": 3.430830522923361, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016767676767677, + "loss": 2.3372, + "theoretical_loss": 3.4308211325200166, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165656565656566, + "loss": 2.4907, + "theoretical_loss": 3.430811742511115, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016363636363636, + "loss": 2.7739, + "theoretical_loss": 3.4308023528966274, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161616161616163, + "loss": 2.4589, + "theoretical_loss": 3.4307929636765233, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040159595959595964, + "loss": 2.6014, + "theoretical_loss": 3.4307835748507736, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015757575757576, + "loss": 2.5064, + "theoretical_loss": 3.430774186419349, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040155555555555555, + "loss": 2.5066, + "theoretical_loss": 3.4307647983822194, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 1129917, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.966796875, + "objective/train/theoretical_loss": 3.430755410739356, + "objective/train/tokens_used": 356330976, + "theoretical_loss": 3.430755410739356, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015353535353535, + "loss": 2.6414, + "theoretical_loss": 3.430755410739356, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015151515151515, + "loss": 2.6696, + "theoretical_loss": 3.4307460234907285, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149494949494953, + "loss": 2.4997, + "theoretical_loss": 3.430736636636308, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014747474747475, + "loss": 2.4858, + "theoretical_loss": 3.430727250176065, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040145454545454545, + "loss": 2.4787, + "theoretical_loss": 3.4307178641099703, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040143434343434346, + "loss": 2.5118, + "theoretical_loss": 3.4307084784379933, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014141414141414, + "loss": 2.661, + "theoretical_loss": 3.430699093160106, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013939393939394, + "loss": 2.8092, + "theoretical_loss": 3.4306897082762777, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013737373737374, + "loss": 2.3799, + "theoretical_loss": 3.4306803237864796, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040135353535353534, + "loss": 2.5231, + "theoretical_loss": 3.4306709396906823, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133333333333335, + "loss": 2.6939, + "theoretical_loss": 3.4306615559888556, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013131313131313, + "loss": 2.489, + "theoretical_loss": 3.4306521726809707, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012929292929293, + "loss": 2.6027, + "theoretical_loss": 3.430642789766998, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012727272727273, + "loss": 2.7012, + "theoretical_loss": 3.4306334072469085, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012525252525253, + "loss": 2.6855, + "theoretical_loss": 3.4306240251206717, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123232323232324, + "loss": 2.714, + "theoretical_loss": 3.4306146433882585, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012121212121212, + "loss": 2.8416, + "theoretical_loss": 3.43060526204964, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011919191919192, + "loss": 2.6983, + "theoretical_loss": 3.4305958811047863, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117171717171717, + "loss": 2.9184, + "theoretical_loss": 3.4305865005536686, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011515151515152, + "loss": 2.4376, + "theoretical_loss": 3.430577120396256, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040113131313131313, + "loss": 2.727, + "theoretical_loss": 3.4305677406325206, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011111111111111, + "loss": 2.5081, + "theoretical_loss": 3.430558361262433, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010909090909091, + "loss": 2.6777, + "theoretical_loss": 3.430548982285962, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107070707070706, + "loss": 2.731, + "theoretical_loss": 3.43053960370308, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040105050505050507, + "loss": 2.6043, + "theoretical_loss": 3.4305302255137566, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1130465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.899060010910034, + "objective/train/theoretical_loss": 3.430520847717963, + "objective/train/tokens_used": 357969376, + "theoretical_loss": 3.430520847717963, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401030303030303, + "loss": 2.5984, + "theoretical_loss": 3.430520847717963, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401010101010101, + "loss": 2.8209, + "theoretical_loss": 3.430511470315669, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400989898989899, + "loss": 2.603, + "theoretical_loss": 3.4305020933068464, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400969696969697, + "loss": 2.5725, + "theoretical_loss": 3.4304927166914645, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040094949494949496, + "loss": 2.7079, + "theoretical_loss": 3.430483340469495, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009292929292929, + "loss": 2.7121, + "theoretical_loss": 3.4304739646409077, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009090909090909, + "loss": 2.5109, + "theoretical_loss": 3.4304645892056733, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008888888888889, + "loss": 2.7247, + "theoretical_loss": 3.4304552141637625, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008686868686869, + "loss": 2.5887, + "theoretical_loss": 3.4304458395151465, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084848484848486, + "loss": 2.5257, + "theoretical_loss": 3.430436465259795, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008282828282828, + "loss": 2.6084, + "theoretical_loss": 3.4304270913976795, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008080808080808, + "loss": 2.5686, + "theoretical_loss": 3.430417717928769, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078787878787883, + "loss": 2.6101, + "theoretical_loss": 3.4304083448530367, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007676767676768, + "loss": 2.5047, + "theoretical_loss": 3.4303989721704506, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074747474747475, + "loss": 2.368, + "theoretical_loss": 3.430389599880983, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007272727272727, + "loss": 2.5847, + "theoretical_loss": 3.430380227984604, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007070707070707, + "loss": 2.4146, + "theoretical_loss": 3.4303708564812845, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068686868686873, + "loss": 2.7344, + "theoretical_loss": 3.4303614853709945, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006666666666667, + "loss": 2.5034, + "theoretical_loss": 3.4303521146537053, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040064646464646464, + "loss": 2.6179, + "theoretical_loss": 3.430342744329387, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006262626262626, + "loss": 2.7422, + "theoretical_loss": 3.430333374398011, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040060606060606066, + "loss": 2.6721, + "theoretical_loss": 3.430324004859547, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005858585858586, + "loss": 2.3831, + "theoretical_loss": 3.430314635713967, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005656565656566, + "loss": 2.6054, + "theoretical_loss": 3.4303052669612395, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054545454545453, + "loss": 2.475, + "theoretical_loss": 3.4302958986013374, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1131148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.794546127319336, + "objective/train/theoretical_loss": 3.43028653063423, + "objective/train/tokens_used": 359607776, + "theoretical_loss": 3.43028653063423, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005252525252525, + "loss": 2.7943, + "theoretical_loss": 3.43028653063423, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050505050505056, + "loss": 2.5118, + "theoretical_loss": 3.4302771630598885, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004848484848485, + "loss": 2.6438, + "theoretical_loss": 3.4302677958782835, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046464646464647, + "loss": 2.6051, + "theoretical_loss": 3.4302584290893856, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004444444444444, + "loss": 2.5233, + "theoretical_loss": 3.430249062693165, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040042424242424244, + "loss": 2.7153, + "theoretical_loss": 3.430239696689594, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040040404040404045, + "loss": 2.8019, + "theoretical_loss": 3.4302303310786417, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003838383838384, + "loss": 2.5611, + "theoretical_loss": 3.4302209658602787, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036363636363636, + "loss": 2.5793, + "theoretical_loss": 3.4302116010344763, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003434343434343, + "loss": 2.6567, + "theoretical_loss": 3.4302022366012057, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040032323232323233, + "loss": 2.662, + "theoretical_loss": 3.4301928725604367, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030303030303034, + "loss": 2.8829, + "theoretical_loss": 3.4301835089121404, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002828282828283, + "loss": 2.7492, + "theoretical_loss": 3.430174145656287, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040026262626262625, + "loss": 2.5698, + "theoretical_loss": 3.4301647827928483, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024242424242427, + "loss": 2.7348, + "theoretical_loss": 3.430155420321794, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002222222222222, + "loss": 2.4581, + "theoretical_loss": 3.430146058243095, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020202020202023, + "loss": 2.7218, + "theoretical_loss": 3.430136696556722, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001818181818182, + "loss": 2.4765, + "theoretical_loss": 3.4301273352626462, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016161616161615, + "loss": 2.6248, + "theoretical_loss": 3.430117974360838, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014141414141416, + "loss": 2.745, + "theoretical_loss": 3.4301086138512678, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001212121212121, + "loss": 2.7984, + "theoretical_loss": 3.430099253733907, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001010101010101, + "loss": 2.6117, + "theoretical_loss": 3.4300898940087254, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000808080808081, + "loss": 2.6312, + "theoretical_loss": 3.430080534675695, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000606060606061, + "loss": 2.5885, + "theoretical_loss": 3.4300711757347853, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040004040404040405, + "loss": 2.4592, + "theoretical_loss": 3.4300618171859676, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1132271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6209566593170166, + "objective/train/theoretical_loss": 3.4300524590292127, + "objective/train/tokens_used": 361246176, + "theoretical_loss": 3.4300524590292127, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400020202020202, + "loss": 2.6239, + "theoretical_loss": 3.4300524590292127, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 2.4764, + "theoretical_loss": 3.4300431012644914, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399979797979798, + "loss": 2.9292, + "theoretical_loss": 3.430033743891774, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399959595959596, + "loss": 2.6946, + "theoretical_loss": 3.4300243869110316, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993939393939394, + "loss": 2.6526, + "theoretical_loss": 3.430015030322235, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999191919191919, + "loss": 2.6167, + "theoretical_loss": 3.4300056741253546, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998989898989899, + "loss": 2.6005, + "theoretical_loss": 3.4299963183203617, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998787878787879, + "loss": 2.8798, + "theoretical_loss": 3.4299869629072264, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998585858585859, + "loss": 2.7811, + "theoretical_loss": 3.4299776078859203, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039983838383838384, + "loss": 2.2802, + "theoretical_loss": 3.4299682532564137, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998181818181818, + "loss": 2.4756, + "theoretical_loss": 3.4299588990186773, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997979797979798, + "loss": 2.6694, + "theoretical_loss": 3.429949545172682, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997777777777778, + "loss": 2.6341, + "theoretical_loss": 3.429940191718398, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975757575757577, + "loss": 2.8142, + "theoretical_loss": 3.429930838655797, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039973737373737373, + "loss": 2.5922, + "theoretical_loss": 3.4299214859848495, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997171717171717, + "loss": 2.4585, + "theoretical_loss": 3.429912133705526, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969696969696975, + "loss": 2.8098, + "theoretical_loss": 3.4299027818177974, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996767676767677, + "loss": 2.8006, + "theoretical_loss": 3.429893430321635, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039965656565656567, + "loss": 2.3448, + "theoretical_loss": 3.4298840792170084, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996363636363636, + "loss": 2.5341, + "theoretical_loss": 3.42987472850389, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996161616161616, + "loss": 2.5892, + "theoretical_loss": 3.4298653781822495, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959595959595964, + "loss": 2.5917, + "theoretical_loss": 3.429856028252058, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995757575757576, + "loss": 2.6787, + "theoretical_loss": 3.429846678713286, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955555555555556, + "loss": 2.6223, + "theoretical_loss": 3.4298373295659053, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995353535353535, + "loss": 2.7406, + "theoretical_loss": 3.4298279808098853, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1132794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.300257921218872, + "objective/train/theoretical_loss": 3.4298186324451976, + "objective/train/tokens_used": 362884576, + "theoretical_loss": 3.4298186324451976, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951515151515147, + "loss": 2.4009, + "theoretical_loss": 3.4298186324451976, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039949494949494954, + "loss": 2.7926, + "theoretical_loss": 3.4298092844718133, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994747474747475, + "loss": 2.6793, + "theoretical_loss": 3.4297999368897027, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039945454545454545, + "loss": 2.5923, + "theoretical_loss": 3.429790589698837, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994343434343434, + "loss": 2.5821, + "theoretical_loss": 3.4297812428991867, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994141414141414, + "loss": 2.8107, + "theoretical_loss": 3.429771896490723, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939393939393943, + "loss": 2.9293, + "theoretical_loss": 3.4297625504734164, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993737373737374, + "loss": 2.4747, + "theoretical_loss": 3.4297532048472377, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039935353535353534, + "loss": 2.5923, + "theoretical_loss": 3.4297438596121577, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993333333333333, + "loss": 2.4446, + "theoretical_loss": 3.429734514768148, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993131313131313, + "loss": 2.6292, + "theoretical_loss": 3.4297251703151783, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992929292929293, + "loss": 2.5493, + "theoretical_loss": 3.4297158262532204, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992727272727273, + "loss": 2.84, + "theoretical_loss": 3.4297064825822448, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925252525252524, + "loss": 2.4661, + "theoretical_loss": 3.4296971393022226, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923232323232325, + "loss": 2.7428, + "theoretical_loss": 3.4296877964131243, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992121212121212, + "loss": 2.6717, + "theoretical_loss": 3.4296784539149208, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991919191919192, + "loss": 2.7477, + "theoretical_loss": 3.4296691118075833, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917171717171717, + "loss": 2.5737, + "theoretical_loss": 3.4296597700910825, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039915151515151513, + "loss": 2.6183, + "theoretical_loss": 3.429650428765389, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913131313131314, + "loss": 2.5853, + "theoretical_loss": 3.4296410878304737, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039911111111111115, + "loss": 2.6433, + "theoretical_loss": 3.429631747286308, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990909090909091, + "loss": 2.4728, + "theoretical_loss": 3.4296224071328623, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039907070707070706, + "loss": 2.8063, + "theoretical_loss": 3.4296130673701075, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990505050505051, + "loss": 2.6914, + "theoretical_loss": 3.429603727998015, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039903030303030303, + "loss": 2.839, + "theoretical_loss": 3.429594389016555, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1134045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2589073181152344, + "objective/train/theoretical_loss": 3.429585050425699, + "objective/train/tokens_used": 364522976, + "theoretical_loss": 3.429585050425699, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901010101010104, + "loss": 2.5398, + "theoretical_loss": 3.429585050425699, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398989898989899, + "loss": 2.713, + "theoretical_loss": 3.429575712225418, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039896969696969696, + "loss": 2.6179, + "theoretical_loss": 3.429566374415682, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039894949494949497, + "loss": 2.7612, + "theoretical_loss": 3.4295570369964623, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989292929292929, + "loss": 2.5426, + "theoretical_loss": 3.4295476999677303, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890909090909094, + "loss": 2.6305, + "theoretical_loss": 3.4295383633294563, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988888888888889, + "loss": 2.7262, + "theoretical_loss": 3.429529027081612, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988686868686869, + "loss": 2.6242, + "theoretical_loss": 3.4295196912241677, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039884848484848486, + "loss": 2.7757, + "theoretical_loss": 3.429510355757094, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988282828282828, + "loss": 2.7324, + "theoretical_loss": 3.4295010206803624, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880808080808083, + "loss": 2.8253, + "theoretical_loss": 3.429491685993944, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987878787878788, + "loss": 2.7678, + "theoretical_loss": 3.4294823516978092, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987676767676768, + "loss": 2.6828, + "theoretical_loss": 3.4294730177919295, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874747474747475, + "loss": 2.8366, + "theoretical_loss": 3.429463684276275, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987272727272727, + "loss": 2.939, + "theoretical_loss": 3.429454351150817, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987070707070707, + "loss": 2.6962, + "theoretical_loss": 3.429445018415527, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868686868686873, + "loss": 2.9651, + "theoretical_loss": 3.4294356860703754, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986666666666667, + "loss": 2.6486, + "theoretical_loss": 3.429426354115334, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864646464646465, + "loss": 2.5321, + "theoretical_loss": 3.4294170225503717, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986262626262626, + "loss": 2.6465, + "theoretical_loss": 3.429407691375461, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986060606060606, + "loss": 2.6514, + "theoretical_loss": 3.4293983605905733, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985858585858586, + "loss": 2.8038, + "theoretical_loss": 3.429389030195679, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985656565656566, + "loss": 2.6493, + "theoretical_loss": 3.4293797001907484, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039854545454545454, + "loss": 2.867, + "theoretical_loss": 3.4293703705757537, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985252525252525, + "loss": 2.5143, + "theoretical_loss": 3.4293610413506643, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1134700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6063029766082764, + "objective/train/theoretical_loss": 3.4293517125154525, + "objective/train/tokens_used": 366161376, + "theoretical_loss": 3.4293517125154525, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850505050505056, + "loss": 2.607, + "theoretical_loss": 3.4293517125154525, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984848484848485, + "loss": 2.2265, + "theoretical_loss": 3.429342384070089, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984646464646465, + "loss": 2.6767, + "theoretical_loss": 3.4293330560145447, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844444444444443, + "loss": 2.7189, + "theoretical_loss": 3.4293237283487903, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984242424242424, + "loss": 2.5232, + "theoretical_loss": 3.4293144010727974, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840404040404045, + "loss": 2.496, + "theoretical_loss": 3.4293050741865363, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983838383838384, + "loss": 2.6239, + "theoretical_loss": 3.429295747689978, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836363636363637, + "loss": 2.8334, + "theoretical_loss": 3.4292864215830945, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983434343434343, + "loss": 2.651, + "theoretical_loss": 3.4292770958658556, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039832323232323234, + "loss": 2.8451, + "theoretical_loss": 3.4292677705382335, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039830303030303035, + "loss": 2.691, + "theoretical_loss": 3.4292584456001975, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982828282828283, + "loss": 2.6479, + "theoretical_loss": 3.42924912105172, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826262626262626, + "loss": 2.4829, + "theoretical_loss": 3.429239796892772, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982424242424242, + "loss": 2.8045, + "theoretical_loss": 3.429230473123324, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039822222222222223, + "loss": 2.673, + "theoretical_loss": 3.429221149743347, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039820202020202024, + "loss": 2.7428, + "theoretical_loss": 3.4292118267528124, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981818181818182, + "loss": 2.6881, + "theoretical_loss": 3.429202504151691, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039816161616161615, + "loss": 2.5761, + "theoretical_loss": 3.4291931819399535, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814141414141416, + "loss": 2.8545, + "theoretical_loss": 3.429183860117572, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981212121212121, + "loss": 2.6683, + "theoretical_loss": 3.4291745386845163, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810101010101013, + "loss": 2.6189, + "theoretical_loss": 3.429165217640758, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980808080808081, + "loss": 2.5799, + "theoretical_loss": 3.429155896986268, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806060606060605, + "loss": 2.8319, + "theoretical_loss": 3.4291465767210174, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039804040404040406, + "loss": 2.7951, + "theoretical_loss": 3.429137256844977, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398020202020202, + "loss": 2.8178, + "theoretical_loss": 3.429127937358119, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1135755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.328629732131958, + "objective/train/theoretical_loss": 3.429118618260413, + "objective/train/tokens_used": 367799776, + "theoretical_loss": 3.429118618260413, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398, + "loss": 2.5462, + "theoretical_loss": 3.429118618260413, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397979797979798, + "loss": 2.6842, + "theoretical_loss": 3.429109299551831, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397959595959596, + "loss": 2.2959, + "theoretical_loss": 3.4290999812323433, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793939393939395, + "loss": 2.7176, + "theoretical_loss": 3.4290906633019214, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979191919191919, + "loss": 2.7385, + "theoretical_loss": 3.4290813457605362, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978989898989899, + "loss": 2.6347, + "theoretical_loss": 3.429072028608159, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978787878787879, + "loss": 2.7478, + "theoretical_loss": 3.429062711844761, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978585858585859, + "loss": 2.5556, + "theoretical_loss": 3.429053395470313, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039783838383838384, + "loss": 2.6114, + "theoretical_loss": 3.429044079484786, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978181818181818, + "loss": 2.6957, + "theoretical_loss": 3.4290347638881515, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977979797979798, + "loss": 2.7311, + "theoretical_loss": 3.42902544868038, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777777777777777, + "loss": 2.6173, + "theoretical_loss": 3.4290161338614427, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977575757575758, + "loss": 2.6441, + "theoretical_loss": 3.4290068194313115, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039773737373737373, + "loss": 2.7549, + "theoretical_loss": 3.428997505389956, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977171717171717, + "loss": 3.0107, + "theoretical_loss": 3.4289881917373486, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976969696969697, + "loss": 2.7845, + "theoretical_loss": 3.42897887847346, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976767676767677, + "loss": 2.9564, + "theoretical_loss": 3.428969565598261, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765656565656567, + "loss": 2.9336, + "theoretical_loss": 3.428960253111723, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039763636363636363, + "loss": 2.5708, + "theoretical_loss": 3.4289509410138175, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976161616161616, + "loss": 2.5309, + "theoretical_loss": 3.4289416293045147, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975959595959596, + "loss": 2.5989, + "theoretical_loss": 3.4289323179837865, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975757575757576, + "loss": 2.6427, + "theoretical_loss": 3.4289230070516035, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039755555555555556, + "loss": 2.6585, + "theoretical_loss": 3.428913696507937, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975353535353535, + "loss": 2.7107, + "theoretical_loss": 3.428904386352758, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975151515151515, + "loss": 2.5721, + "theoretical_loss": 3.428895076586038, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1136767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.669450521469116, + "objective/train/theoretical_loss": 3.428885767207748, + "objective/train/tokens_used": 369438176, + "theoretical_loss": 3.428885767207748, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749494949494954, + "loss": 2.6505, + "theoretical_loss": 3.428885767207748, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974747474747475, + "loss": 2.8398, + "theoretical_loss": 3.428876458217859, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745454545454546, + "loss": 2.984, + "theoretical_loss": 3.4288671496163423, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974343434343434, + "loss": 2.8403, + "theoretical_loss": 3.4288578414031687, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039741414141414137, + "loss": 2.8432, + "theoretical_loss": 3.42884853357831, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739393939393944, + "loss": 2.4447, + "theoretical_loss": 3.4288392261417364, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973737373737374, + "loss": 2.7073, + "theoretical_loss": 3.4288299190934195, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039735353535353535, + "loss": 2.6596, + "theoretical_loss": 3.4288206124333307, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973333333333333, + "loss": 2.9337, + "theoretical_loss": 3.4288113061614407, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731313131313137, + "loss": 2.571, + "theoretical_loss": 3.4288020002777215, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039729292929292933, + "loss": 2.8459, + "theoretical_loss": 3.4287926947821434, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972727272727273, + "loss": 2.7237, + "theoretical_loss": 3.4287833896746776, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039725252525252524, + "loss": 2.8203, + "theoretical_loss": 3.428774084955296, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972323232323232, + "loss": 2.8712, + "theoretical_loss": 3.428764780623969, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039721212121212126, + "loss": 2.8997, + "theoretical_loss": 3.428755476680668, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971919191919192, + "loss": 2.8816, + "theoretical_loss": 3.428746173125364, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971717171717172, + "loss": 2.7935, + "theoretical_loss": 3.428736869958029, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715151515151513, + "loss": 2.5695, + "theoretical_loss": 3.4287275671786333, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713131313131315, + "loss": 2.642, + "theoretical_loss": 3.4287182647871486, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039711111111111116, + "loss": 2.5356, + "theoretical_loss": 3.4287089627835456, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970909090909091, + "loss": 2.9735, + "theoretical_loss": 3.428699661167796, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039707070707070707, + "loss": 2.86, + "theoretical_loss": 3.4286903599398704, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397050505050505, + "loss": 2.8286, + "theoretical_loss": 3.4286810590997403, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703030303030304, + "loss": 2.7634, + "theoretical_loss": 3.4286717586473774, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701010101010105, + "loss": 2.5681, + "theoretical_loss": 3.4286624585827523, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1137326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8470661640167236, + "objective/train/theoretical_loss": 3.428653158905836, + "objective/train/tokens_used": 371076576, + "theoretical_loss": 3.428653158905836, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.21, + "learning_rate": 0.000396989898989899, + "loss": 2.6908, + "theoretical_loss": 3.428653158905836, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696969696969696, + "loss": 2.7909, + "theoretical_loss": 3.4286438596166007, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.21, + "learning_rate": 0.000396949494949495, + "loss": 2.7957, + "theoretical_loss": 3.428634560715017, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039692929292929293, + "loss": 2.8062, + "theoretical_loss": 3.428625262201056, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690909090909094, + "loss": 2.7851, + "theoretical_loss": 3.4286159640746883, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968888888888889, + "loss": 2.5913, + "theoretical_loss": 3.4286066663358863, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039686868686868686, + "loss": 2.6842, + "theoretical_loss": 3.428597368984621, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039684848484848487, + "loss": 2.58, + "theoretical_loss": 3.4285880720208635, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968282828282828, + "loss": 2.7374, + "theoretical_loss": 3.428578775444585, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039680808080808083, + "loss": 2.7025, + "theoretical_loss": 3.428569479255756, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967878787878788, + "loss": 2.682, + "theoretical_loss": 3.4285601834543487, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967676767676768, + "loss": 2.7852, + "theoretical_loss": 3.428550888040334, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674747474747476, + "loss": 2.7784, + "theoretical_loss": 3.4285415930136836, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967272727272727, + "loss": 2.6145, + "theoretical_loss": 3.4285322983743676, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039670707070707073, + "loss": 2.8174, + "theoretical_loss": 3.4285230041223587, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966868686868687, + "loss": 2.7571, + "theoretical_loss": 3.4285137102576275, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966666666666667, + "loss": 2.6498, + "theoretical_loss": 3.428504416780145, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039664646464646465, + "loss": 3.0269, + "theoretical_loss": 3.428495123689882, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966262626262626, + "loss": 2.5478, + "theoretical_loss": 3.428485830986811, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966060606060606, + "loss": 2.6758, + "theoretical_loss": 3.4284765386709033, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658585858585863, + "loss": 2.7102, + "theoretical_loss": 3.4284672467421284, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965656565656566, + "loss": 2.8289, + "theoretical_loss": 3.4284579552004595, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654545454545454, + "loss": 2.7493, + "theoretical_loss": 3.4284486640458667, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965252525252525, + "loss": 2.8344, + "theoretical_loss": 3.428439373278322, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965050505050505, + "loss": 2.6865, + "theoretical_loss": 3.4284300828977967, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 1138753, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.317888021469116, + "objective/train/theoretical_loss": 3.428420792904261, + "objective/train/tokens_used": 372714976, + "theoretical_loss": 3.428420792904261, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003964848484848485, + "loss": 2.6689, + "theoretical_loss": 3.428420792904261, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964646464646465, + "loss": 2.698, + "theoretical_loss": 3.428411503297687, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039644444444444444, + "loss": 2.7431, + "theoretical_loss": 3.428402214078046, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964242424242424, + "loss": 2.9417, + "theoretical_loss": 3.4283929252453094, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640404040404046, + "loss": 2.6947, + "theoretical_loss": 3.4283836367994485, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963838383838384, + "loss": 2.7881, + "theoretical_loss": 3.428374348740434, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963636363636364, + "loss": 2.4982, + "theoretical_loss": 3.428365061068238, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039634343434343433, + "loss": 2.6902, + "theoretical_loss": 3.4283557737828314, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963232323232323, + "loss": 2.9503, + "theoretical_loss": 3.4283464868841853, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039630303030303035, + "loss": 2.8396, + "theoretical_loss": 3.428337200372271, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962828282828283, + "loss": 2.861, + "theoretical_loss": 3.4283279142470606, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039626262626262627, + "loss": 2.9209, + "theoretical_loss": 3.4283186285085243, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962424242424242, + "loss": 2.6304, + "theoretical_loss": 3.4283093431566343, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962222222222222, + "loss": 2.5052, + "theoretical_loss": 3.428300058191362, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039620202020202024, + "loss": 2.9644, + "theoretical_loss": 3.428290773612678, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961818181818182, + "loss": 2.5193, + "theoretical_loss": 3.428281489420554, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616161616161616, + "loss": 2.7233, + "theoretical_loss": 3.4282722056149613, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961414141414141, + "loss": 2.959, + "theoretical_loss": 3.428262922195871, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961212121212121, + "loss": 2.7078, + "theoretical_loss": 3.428253639163255, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039610101010101014, + "loss": 2.9444, + "theoretical_loss": 3.428244356517084, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960808080808081, + "loss": 2.7413, + "theoretical_loss": 3.42823507425733, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039606060606060605, + "loss": 2.7726, + "theoretical_loss": 3.428225792383964, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396040404040404, + "loss": 2.6754, + "theoretical_loss": 3.4282165108969576, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396020202020202, + "loss": 2.6725, + "theoretical_loss": 3.428207229796281, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600000000000003, + "loss": 2.6293, + "theoretical_loss": 3.4281979490819072, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1139429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7300171852111816, + "objective/train/theoretical_loss": 3.4281886687538066, + "objective/train/tokens_used": 374353376, + "theoretical_loss": 3.4281886687538066, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395979797979798, + "loss": 2.6279, + "theoretical_loss": 3.4281886687538066, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039595959595959594, + "loss": 2.7789, + "theoretical_loss": 3.428179388811951, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039593939393939396, + "loss": 2.6587, + "theoretical_loss": 3.4281701092563117, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959191919191919, + "loss": 2.8057, + "theoretical_loss": 3.42816083008686, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958989898989899, + "loss": 2.74, + "theoretical_loss": 3.428151551303567, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958787878787879, + "loss": 2.8309, + "theoretical_loss": 3.428142272906404, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039585858585858584, + "loss": 2.5979, + "theoretical_loss": 3.4281329948953427, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583838383838385, + "loss": 2.7137, + "theoretical_loss": 3.428123717270355, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958181818181818, + "loss": 2.6471, + "theoretical_loss": 3.428114440031411, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957979797979798, + "loss": 2.7019, + "theoretical_loss": 3.4281051631784836, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039577777777777777, + "loss": 2.645, + "theoretical_loss": 3.428095886711543, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957575757575758, + "loss": 2.7763, + "theoretical_loss": 3.4280866106305607, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039573737373737374, + "loss": 2.6072, + "theoretical_loss": 3.4280773349355087, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171717171717, + "loss": 2.6959, + "theoretical_loss": 3.4280680596263586, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956969696969697, + "loss": 2.6911, + "theoretical_loss": 3.4280587847030812, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567676767676767, + "loss": 2.6785, + "theoretical_loss": 3.428049510165647, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956565656565657, + "loss": 2.7695, + "theoretical_loss": 3.4280402360140294, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563636363636363, + "loss": 2.8333, + "theoretical_loss": 3.4280309622481986, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039561616161616164, + "loss": 2.5477, + "theoretical_loss": 3.4280216888681263, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955959595959596, + "loss": 2.8905, + "theoretical_loss": 3.4280124158737837, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955757575757576, + "loss": 2.9211, + "theoretical_loss": 3.4280031432651423, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039555555555555557, + "loss": 2.7333, + "theoretical_loss": 3.4279938710421742, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955353535353535, + "loss": 2.8409, + "theoretical_loss": 3.4279845992048497, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551515151515154, + "loss": 2.8857, + "theoretical_loss": 3.427975327753141, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954949494949495, + "loss": 2.6998, + "theoretical_loss": 3.427966056687019, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1140637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.585150718688965, + "objective/train/theoretical_loss": 3.427956786006456, + "objective/train/tokens_used": 375991776, + "theoretical_loss": 3.427956786006456, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954747474747475, + "loss": 2.733, + "theoretical_loss": 3.427956786006456, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039545454545454546, + "loss": 2.5381, + "theoretical_loss": 3.4279475157114225, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954343434343434, + "loss": 2.6898, + "theoretical_loss": 3.42793824580189, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039541414141414143, + "loss": 2.8961, + "theoretical_loss": 3.427928976277831, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039539393939393944, + "loss": 2.9721, + "theoretical_loss": 3.427919707139216, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953737373737374, + "loss": 2.6188, + "theoretical_loss": 3.4279104383860166, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535353535353535, + "loss": 2.9361, + "theoretical_loss": 3.4279011700182043, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953333333333333, + "loss": 2.796, + "theoretical_loss": 3.427891902035751, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953131313131313, + "loss": 2.5749, + "theoretical_loss": 3.427882634438627, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529292929292933, + "loss": 2.8439, + "theoretical_loss": 3.4278733672268045, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952727272727273, + "loss": 2.701, + "theoretical_loss": 3.4278641004002557, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039525252525252525, + "loss": 2.7596, + "theoretical_loss": 3.4278548339589507, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952323232323232, + "loss": 2.827, + "theoretical_loss": 3.427845567902862, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521212121212127, + "loss": 2.7817, + "theoretical_loss": 3.427836302231961, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951919191919192, + "loss": 2.6371, + "theoretical_loss": 3.4278270369462183, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951717171717172, + "loss": 2.6538, + "theoretical_loss": 3.4278177720456062, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039515151515151514, + "loss": 2.4253, + "theoretical_loss": 3.427808507530096, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951313131313131, + "loss": 2.6444, + "theoretical_loss": 3.427799243399659, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039511111111111116, + "loss": 2.8143, + "theoretical_loss": 3.427789979654267, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950909090909091, + "loss": 2.6727, + "theoretical_loss": 3.427780716293891, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950707070707071, + "loss": 2.9506, + "theoretical_loss": 3.427771453318503, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505050505050503, + "loss": 2.7958, + "theoretical_loss": 3.4277621907280746, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039503030303030304, + "loss": 2.6645, + "theoretical_loss": 3.4277529285225765, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039501010101010105, + "loss": 2.7401, + "theoretical_loss": 3.427743666701981, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394989898989899, + "loss": 2.6733, + "theoretical_loss": 3.4277344052662597, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1141300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8804666996002197, + "objective/train/theoretical_loss": 3.4277251442153833, + "objective/train/tokens_used": 377630176, + "theoretical_loss": 3.4277251442153833, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039496969696969697, + "loss": 2.7991, + "theoretical_loss": 3.4277251442153833, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949494949494949, + "loss": 3.0216, + "theoretical_loss": 3.4277158835493236, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039492929292929294, + "loss": 2.8571, + "theoretical_loss": 3.427706623268053, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039490909090909095, + "loss": 2.5159, + "theoretical_loss": 3.4276973633715415, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948888888888889, + "loss": 2.8921, + "theoretical_loss": 3.4276881038597615, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486868686868686, + "loss": 2.8854, + "theoretical_loss": 3.427678844732685, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039484848484848487, + "loss": 2.6002, + "theoretical_loss": 3.427669585990283, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039482828282828283, + "loss": 2.5878, + "theoretical_loss": 3.427660327632527, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480808080808084, + "loss": 2.8296, + "theoretical_loss": 3.4276510696593876, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947878787878788, + "loss": 2.7003, + "theoretical_loss": 3.4276418120708385, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039476767676767675, + "loss": 2.7496, + "theoretical_loss": 3.4276325548668494, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039474747474747476, + "loss": 2.5367, + "theoretical_loss": 3.4276232980473926, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947272727272727, + "loss": 2.6793, + "theoretical_loss": 3.42761404161244, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039470707070707073, + "loss": 2.6917, + "theoretical_loss": 3.4276047855619622, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946868686868687, + "loss": 2.7398, + "theoretical_loss": 3.4275955298959313, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946666666666667, + "loss": 2.5197, + "theoretical_loss": 3.4275862746143186, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464646464646466, + "loss": 2.739, + "theoretical_loss": 3.4275770197170963, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946262626262626, + "loss": 2.6928, + "theoretical_loss": 3.427567765204236, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946060606060606, + "loss": 2.9677, + "theoretical_loss": 3.4275585110757074, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945858585858586, + "loss": 2.4829, + "theoretical_loss": 3.4275492573314845, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945656565656566, + "loss": 2.9356, + "theoretical_loss": 3.4275400039715374, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039454545454545455, + "loss": 2.7642, + "theoretical_loss": 3.427530750995839, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945252525252525, + "loss": 2.8859, + "theoretical_loss": 3.427521498404359, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945050505050505, + "loss": 2.7276, + "theoretical_loss": 3.42751224619707, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944848484848485, + "loss": 2.4853, + "theoretical_loss": 3.427502994373944, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1142665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.620265245437622, + "objective/train/theoretical_loss": 3.427493742934952, + "objective/train/tokens_used": 379268576, + "theoretical_loss": 3.427493742934952, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944646464646465, + "loss": 2.6001, + "theoretical_loss": 3.427493742934952, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039444444444444444, + "loss": 2.6758, + "theoretical_loss": 3.4274844918800658, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944242424242424, + "loss": 2.8037, + "theoretical_loss": 3.4274752412092573, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944040404040404, + "loss": 2.787, + "theoretical_loss": 3.427465990922497, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943838383838384, + "loss": 2.6208, + "theoretical_loss": 3.427456741019758, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943636363636364, + "loss": 2.5446, + "theoretical_loss": 3.4274474915010105, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434343434343434, + "loss": 2.6955, + "theoretical_loss": 3.427438242366227, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943232323232323, + "loss": 2.8912, + "theoretical_loss": 3.427428993615379, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943030303030303, + "loss": 2.8633, + "theoretical_loss": 3.4274197452484376, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828282828283, + "loss": 2.6855, + "theoretical_loss": 3.4274104972653747, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426262626262627, + "loss": 2.815, + "theoretical_loss": 3.4274012496661626, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424242424242423, + "loss": 2.8005, + "theoretical_loss": 3.4273920024507722, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942222222222222, + "loss": 2.8866, + "theoretical_loss": 3.427382755619175, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039420202020202025, + "loss": 2.6479, + "theoretical_loss": 3.427373509171343, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941818181818182, + "loss": 2.5791, + "theoretical_loss": 3.427364263107248, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039416161616161616, + "loss": 2.6526, + "theoretical_loss": 3.4273550174268603, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941414141414141, + "loss": 2.7447, + "theoretical_loss": 3.4273457721301535, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941212121212121, + "loss": 2.8829, + "theoretical_loss": 3.4273365272170984, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039410101010101014, + "loss": 2.5755, + "theoretical_loss": 3.427327282687666, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940808080808081, + "loss": 2.6299, + "theoretical_loss": 3.427318038541829, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039406060606060606, + "loss": 2.7202, + "theoretical_loss": 3.4273087947795586, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394040404040404, + "loss": 2.7497, + "theoretical_loss": 3.427299551400826, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394020202020202, + "loss": 2.7424, + "theoretical_loss": 3.4272903084056034, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039400000000000004, + "loss": 2.8449, + "theoretical_loss": 3.4272810657938626, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393979797979798, + "loss": 2.7384, + "theoretical_loss": 3.427271823565574, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1143239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.88350772857666, + "objective/train/theoretical_loss": 3.4272625817207114, + "objective/train/tokens_used": 380906976, + "theoretical_loss": 3.4272625817207114, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039395959595959595, + "loss": 2.7637, + "theoretical_loss": 3.4272625817207114, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939393939393939, + "loss": 2.8714, + "theoretical_loss": 3.427253340259245, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391919191919197, + "loss": 2.5022, + "theoretical_loss": 3.427244099181147, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039389898989898993, + "loss": 2.8287, + "theoretical_loss": 3.427234858486388, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938787878787879, + "loss": 2.8633, + "theoretical_loss": 3.4272256181749414, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385858585858584, + "loss": 2.5607, + "theoretical_loss": 3.4272163782467775, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383838383838385, + "loss": 2.7901, + "theoretical_loss": 3.4272071387018688, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381818181818186, + "loss": 2.7659, + "theoretical_loss": 3.427197899540187, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937979797979798, + "loss": 2.8358, + "theoretical_loss": 3.4271886607617024, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937777777777778, + "loss": 2.7872, + "theoretical_loss": 3.4271794223663887, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375757575757574, + "loss": 2.9364, + "theoretical_loss": 3.427170184354216, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373737373737375, + "loss": 2.6487, + "theoretical_loss": 3.427160946725157, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039371717171717176, + "loss": 2.7498, + "theoretical_loss": 3.4271517094791832, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936969696969697, + "loss": 2.9106, + "theoretical_loss": 3.427142472616266, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039367676767676767, + "loss": 2.62, + "theoretical_loss": 3.4271332361363775, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936565656565657, + "loss": 2.9054, + "theoretical_loss": 3.4271240000394885, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039363636363636364, + "loss": 2.8393, + "theoretical_loss": 3.427114764325572, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361616161616165, + "loss": 2.65, + "theoretical_loss": 3.427105528994599, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935959595959596, + "loss": 2.7686, + "theoretical_loss": 3.427096294046541, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357575757575756, + "loss": 2.9455, + "theoretical_loss": 3.4270870594813703, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935555555555556, + "loss": 2.7864, + "theoretical_loss": 3.427077825299058, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353535353535353, + "loss": 2.7365, + "theoretical_loss": 3.4270685914995767, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039351515151515154, + "loss": 2.916, + "theoretical_loss": 3.4270593580828974, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934949494949495, + "loss": 2.5035, + "theoretical_loss": 3.427050125048992, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934747474747475, + "loss": 2.8013, + "theoretical_loss": 3.4270408923978324, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1144661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3514699935913086, + "objective/train/theoretical_loss": 3.42703166012939, + "objective/train/tokens_used": 382545376, + "theoretical_loss": 3.42703166012939, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345454545454547, + "loss": 2.5329, + "theoretical_loss": 3.42703166012939, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934343434343434, + "loss": 2.6504, + "theoretical_loss": 3.427022428243637, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039341414141414144, + "loss": 2.7296, + "theoretical_loss": 3.427013196740545, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933939393939394, + "loss": 2.8023, + "theoretical_loss": 3.4270039656200852, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933737373737374, + "loss": 2.6277, + "theoretical_loss": 3.4269947348822303, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039335353535353536, + "loss": 2.8942, + "theoretical_loss": 3.426985504526951, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933333333333333, + "loss": 2.6153, + "theoretical_loss": 3.42697627455422, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331313131313133, + "loss": 2.7145, + "theoretical_loss": 3.426967044964009, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039329292929292934, + "loss": 2.7279, + "theoretical_loss": 3.426957815756289, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932727272727273, + "loss": 2.6095, + "theoretical_loss": 3.426948586931032, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325252525252525, + "loss": 2.8379, + "theoretical_loss": 3.4269393584882106, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932323232323232, + "loss": 2.805, + "theoretical_loss": 3.4269301304277953, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932121212121212, + "loss": 2.5146, + "theoretical_loss": 3.4269209027497594, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319191919191923, + "loss": 2.8196, + "theoretical_loss": 3.4269116754540727, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931717171717172, + "loss": 3.094, + "theoretical_loss": 3.4269024485407087, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315151515151515, + "loss": 2.8323, + "theoretical_loss": 3.426893222009639, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931313131313131, + "loss": 3.1155, + "theoretical_loss": 3.426883995860834, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311111111111117, + "loss": 2.715, + "theoretical_loss": 3.4268747700942668, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930909090909091, + "loss": 2.7909, + "theoretical_loss": 3.426865544709909, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930707070707071, + "loss": 2.699, + "theoretical_loss": 3.426856319707732, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039305050505050504, + "loss": 2.8026, + "theoretical_loss": 3.426847095087708, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393030303030303, + "loss": 2.7414, + "theoretical_loss": 3.4268378708498086, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039301010101010106, + "loss": 2.7211, + "theoretical_loss": 3.426828646994006, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392989898989899, + "loss": 2.5665, + "theoretical_loss": 3.4268194235202714, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392969696969697, + "loss": 2.5545, + "theoretical_loss": 3.4268102004285765, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1145363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.52899169921875, + "objective/train/theoretical_loss": 3.4268009777188935, + "objective/train/tokens_used": 384183776, + "theoretical_loss": 3.4268009777188935, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039294949494949493, + "loss": 2.7841, + "theoretical_loss": 3.4268009777188935, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929292929292929, + "loss": 2.9114, + "theoretical_loss": 3.4267917553911946, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039290909090909095, + "loss": 2.5893, + "theoretical_loss": 3.426782533445451, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928888888888889, + "loss": 2.8011, + "theoretical_loss": 3.426773311881635, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286868686868687, + "loss": 2.9611, + "theoretical_loss": 3.426764090699718, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928484848484848, + "loss": 3.1595, + "theoretical_loss": 3.4267548698996717, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282828282828283, + "loss": 2.8841, + "theoretical_loss": 3.4267456494814685, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039280808080808085, + "loss": 2.7156, + "theoretical_loss": 3.4267364294450795, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927878787878788, + "loss": 2.9088, + "theoretical_loss": 3.426727209790478, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276767676767676, + "loss": 2.519, + "theoretical_loss": 3.426717990517634, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927474747474747, + "loss": 2.7526, + "theoretical_loss": 3.4267087716265205, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039272727272727273, + "loss": 2.8416, + "theoretical_loss": 3.4266995531171087, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270707070707074, + "loss": 2.6673, + "theoretical_loss": 3.426690334989371, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926868686868687, + "loss": 2.699, + "theoretical_loss": 3.426681117243279, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266666666666665, + "loss": 2.9085, + "theoretical_loss": 3.426671899878804, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039264646464646466, + "loss": 2.6455, + "theoretical_loss": 3.4266626828959197, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926262626262626, + "loss": 2.7381, + "theoretical_loss": 3.4266534662945958, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039260606060606063, + "loss": 2.7602, + "theoretical_loss": 3.4266442500748053, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925858585858586, + "loss": 2.7376, + "theoretical_loss": 3.42663503423652, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039256565656565654, + "loss": 2.647, + "theoretical_loss": 3.426625818779711, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254545454545456, + "loss": 2.8701, + "theoretical_loss": 3.4266166037043515, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925252525252525, + "loss": 2.8853, + "theoretical_loss": 3.4266073890104125, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925050505050505, + "loss": 2.7532, + "theoretical_loss": 3.426598174697866, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924848484848485, + "loss": 2.9966, + "theoretical_loss": 3.4265889607666837, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924646464646465, + "loss": 2.7066, + "theoretical_loss": 3.426579747216838, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1146594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5370357036590576, + "objective/train/theoretical_loss": 3.4265705340483, + "objective/train/tokens_used": 385822176, + "theoretical_loss": 3.4265705340483, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244444444444445, + "loss": 2.7568, + "theoretical_loss": 3.4265705340483, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924242424242424, + "loss": 2.5738, + "theoretical_loss": 3.426561321261042, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924040404040404, + "loss": 2.9419, + "theoretical_loss": 3.426552108855037, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923838383838384, + "loss": 2.769, + "theoretical_loss": 3.426542896830255, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923636363636364, + "loss": 2.6568, + "theoretical_loss": 3.4265336851866692, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039234343434343434, + "loss": 2.8058, + "theoretical_loss": 3.4265244739242506, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923232323232323, + "loss": 2.473, + "theoretical_loss": 3.4265152630429725, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923030303030303, + "loss": 2.7108, + "theoretical_loss": 3.426506052542805, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922828282828283, + "loss": 2.7028, + "theoretical_loss": 3.426496842423721, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922626262626263, + "loss": 2.7755, + "theoretical_loss": 3.4264876326856926, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224242424242423, + "loss": 2.6293, + "theoretical_loss": 3.4264784233286916, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039222222222222225, + "loss": 2.6529, + "theoretical_loss": 3.4264692143526894, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922020202020202, + "loss": 2.8884, + "theoretical_loss": 3.4264600057576584, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921818181818182, + "loss": 2.7211, + "theoretical_loss": 3.4264507975435707, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216161616161617, + "loss": 2.8821, + "theoretical_loss": 3.4264415897103975, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921414141414141, + "loss": 2.9182, + "theoretical_loss": 3.4264323822581115, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212121212121214, + "loss": 2.7265, + "theoretical_loss": 3.426423175186684, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039210101010101015, + "loss": 2.8122, + "theoretical_loss": 3.4264139684960875, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920808080808081, + "loss": 2.8537, + "theoretical_loss": 3.4264047621862934, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039206060606060606, + "loss": 2.6438, + "theoretical_loss": 3.4263955562572743, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392040404040404, + "loss": 2.7142, + "theoretical_loss": 3.4263863507090018, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039202020202020203, + "loss": 2.774, + "theoretical_loss": 3.426377145541448, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200000000000004, + "loss": 2.6198, + "theoretical_loss": 3.426367940754584, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391979797979798, + "loss": 2.6312, + "theoretical_loss": 3.4263587363483827, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195959595959596, + "loss": 2.7818, + "theoretical_loss": 3.426349532322816, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1147311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.05806565284729, + "objective/train/theoretical_loss": 3.4263403286778558, + "objective/train/tokens_used": 387460576, + "theoretical_loss": 3.4263403286778558, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919393939393939, + "loss": 2.6409, + "theoretical_loss": 3.4263403286778558, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391919191919192, + "loss": 2.9482, + "theoretical_loss": 3.4263311254134736, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189898989898993, + "loss": 2.8187, + "theoretical_loss": 3.4263219225296417, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918787878787879, + "loss": 2.8861, + "theoretical_loss": 3.4263127200263326, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039185858585858585, + "loss": 2.8687, + "theoretical_loss": 3.4263035179035173, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918383838383838, + "loss": 2.7108, + "theoretical_loss": 3.4262943161611688, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181818181818187, + "loss": 2.6669, + "theoretical_loss": 3.4262851147992577, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917979797979798, + "loss": 2.7638, + "theoretical_loss": 3.426275913817757, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917777777777778, + "loss": 2.5622, + "theoretical_loss": 3.4262667132166387, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175757575757574, + "loss": 2.7403, + "theoretical_loss": 3.4262575129958748, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039173737373737375, + "loss": 2.5902, + "theoretical_loss": 3.426248313155437, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171717171717176, + "loss": 2.734, + "theoretical_loss": 3.426239113695297, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916969696969697, + "loss": 3.0066, + "theoretical_loss": 3.426229914615427, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916767676767677, + "loss": 2.8934, + "theoretical_loss": 3.4262207159158, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039165656565656563, + "loss": 2.6966, + "theoretical_loss": 3.4262115175963865, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163636363636364, + "loss": 2.5545, + "theoretical_loss": 3.4262023196571594, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039161616161616166, + "loss": 2.909, + "theoretical_loss": 3.4261931220980903, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915959595959596, + "loss": 2.7278, + "theoretical_loss": 3.426183924919152, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039157575757575757, + "loss": 2.8949, + "theoretical_loss": 3.4261747281203148, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915555555555556, + "loss": 2.6968, + "theoretical_loss": 3.4261655317015527, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039153535353535354, + "loss": 2.5438, + "theoretical_loss": 3.426156335662837, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151515151515155, + "loss": 2.967, + "theoretical_loss": 3.426147140004139, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914949494949495, + "loss": 2.7221, + "theoretical_loss": 3.4261379447254314, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147474747474746, + "loss": 2.74, + "theoretical_loss": 3.4261287498266864, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914545454545455, + "loss": 2.7197, + "theoretical_loss": 3.4261195553078756, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 1148399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.758509397506714, + "objective/train/theoretical_loss": 3.4261103611689716, + "objective/train/tokens_used": 389098976, + "theoretical_loss": 3.4261103611689716, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039143434343434343, + "loss": 2.4683, + "theoretical_loss": 3.4261103611689716, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141414141414144, + "loss": 2.8756, + "theoretical_loss": 3.4261011674099455, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913939393939394, + "loss": 2.5093, + "theoretical_loss": 3.4260919740307703, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913737373737374, + "loss": 2.6343, + "theoretical_loss": 3.4260827810314174, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039135353535353537, + "loss": 2.7764, + "theoretical_loss": 3.426073588411859, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913333333333333, + "loss": 2.8057, + "theoretical_loss": 3.426064396172068, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039131313131313133, + "loss": 2.7264, + "theoretical_loss": 3.4260552043120147, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912929292929293, + "loss": 2.8256, + "theoretical_loss": 3.4260460128316725, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912727272727273, + "loss": 2.889, + "theoretical_loss": 3.4260368217310133, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125252525252526, + "loss": 2.9148, + "theoretical_loss": 3.426027631010009, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912323232323232, + "loss": 2.7156, + "theoretical_loss": 3.4260184406686314, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912121212121212, + "loss": 2.8954, + "theoretical_loss": 3.426009250706853, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911919191919192, + "loss": 2.6392, + "theoretical_loss": 3.4260000611246455, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911717171717172, + "loss": 2.7947, + "theoretical_loss": 3.4259908719219814, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039115151515151515, + "loss": 2.6529, + "theoretical_loss": 3.4259816830988328, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911313131313131, + "loss": 2.7646, + "theoretical_loss": 3.425972494655171, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911111111111111, + "loss": 2.8057, + "theoretical_loss": 3.425963306590969, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039109090909090913, + "loss": 2.8471, + "theoretical_loss": 3.4259541189061977, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910707070707071, + "loss": 2.7463, + "theoretical_loss": 3.4259449316008306, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039105050505050504, + "loss": 2.5196, + "theoretical_loss": 3.4259357446748395, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391030303030303, + "loss": 2.6957, + "theoretical_loss": 3.4259265581281957, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391010101010101, + "loss": 2.694, + "theoretical_loss": 3.425917371960872, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390989898989899, + "loss": 3.0065, + "theoretical_loss": 3.4259081861728404, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390969696969697, + "loss": 3.0625, + "theoretical_loss": 3.4258990007640726, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094949494949494, + "loss": 2.6792, + "theoretical_loss": 3.425889815734541, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1149081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8213319778442383, + "objective/train/theoretical_loss": 3.425880631084218, + "objective/train/tokens_used": 390737376, + "theoretical_loss": 3.425880631084218, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909292929292929, + "loss": 2.7853, + "theoretical_loss": 3.425880631084218, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039090909090909096, + "loss": 2.8592, + "theoretical_loss": 3.4258714468130753, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908888888888889, + "loss": 2.8101, + "theoretical_loss": 3.425862262921085, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039086868686868687, + "loss": 2.9265, + "theoretical_loss": 3.425853079408219, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084848484848483, + "loss": 2.8938, + "theoretical_loss": 3.4258438962744506, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908282828282828, + "loss": 2.7551, + "theoretical_loss": 3.4258347135197504, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039080808080808085, + "loss": 2.6192, + "theoretical_loss": 3.425825531144092, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907878787878788, + "loss": 2.7658, + "theoretical_loss": 3.4258163491474463, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076767676767677, + "loss": 2.623, + "theoretical_loss": 3.4258071675297854, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907474747474747, + "loss": 2.6921, + "theoretical_loss": 3.425797986291083, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072727272727273, + "loss": 2.793, + "theoretical_loss": 3.425788805431309, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039070707070707074, + "loss": 2.7013, + "theoretical_loss": 3.4257796249504375, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906868686868687, + "loss": 2.9027, + "theoretical_loss": 3.42577044484844, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039066666666666666, + "loss": 2.8875, + "theoretical_loss": 3.425761265125288, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906464646464646, + "loss": 2.8452, + "theoretical_loss": 3.4257520857809545, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906262626262626, + "loss": 2.5933, + "theoretical_loss": 3.4257429068154113, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060606060606064, + "loss": 2.7734, + "theoretical_loss": 3.4257337282286304, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905858585858586, + "loss": 2.6448, + "theoretical_loss": 3.4257245500205844, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056565656565655, + "loss": 2.8289, + "theoretical_loss": 3.4257153721912452, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054545454545456, + "loss": 2.6792, + "theoretical_loss": 3.4257061947405845, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905252525252525, + "loss": 2.8101, + "theoretical_loss": 3.4256970176685755, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039050505050505053, + "loss": 2.676, + "theoretical_loss": 3.42568784097519, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904848484848485, + "loss": 2.5595, + "theoretical_loss": 3.425678664660399, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046464646464644, + "loss": 2.7342, + "theoretical_loss": 3.4256694887241768, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044444444444445, + "loss": 2.7978, + "theoretical_loss": 3.4256603131664933, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1149676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.451201915740967, + "objective/train/theoretical_loss": 3.4256511379873227, + "objective/train/tokens_used": 392375776, + "theoretical_loss": 3.4256511379873227, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039042424242424247, + "loss": 2.6123, + "theoretical_loss": 3.4256511379873227, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904040404040404, + "loss": 2.5707, + "theoretical_loss": 3.425641963186636, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903838383838384, + "loss": 2.8316, + "theoretical_loss": 3.425632788764406, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903636363636364, + "loss": 2.7074, + "theoretical_loss": 3.425623614720604, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034343434343435, + "loss": 2.7149, + "theoretical_loss": 3.4256144410552034, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032323232323236, + "loss": 2.6731, + "theoretical_loss": 3.4256052677681756, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903030303030303, + "loss": 2.8172, + "theoretical_loss": 3.4255960948594932, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028282828282827, + "loss": 2.7448, + "theoretical_loss": 3.425586922329128, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902626262626263, + "loss": 2.864, + "theoretical_loss": 3.4255777501770526, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024242424242424, + "loss": 2.7011, + "theoretical_loss": 3.4255685784032393, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039022222222222225, + "loss": 2.5577, + "theoretical_loss": 3.4255594070076594, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902020202020202, + "loss": 2.8505, + "theoretical_loss": 3.4255502359902863, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901818181818182, + "loss": 2.8654, + "theoretical_loss": 3.4255410653510916, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901616161616162, + "loss": 2.5409, + "theoretical_loss": 3.425531895090047, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039014141414141413, + "loss": 2.6516, + "theoretical_loss": 3.425522725207126, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012121212121214, + "loss": 2.831, + "theoretical_loss": 3.4255135557022998, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901010101010101, + "loss": 2.614, + "theoretical_loss": 3.4255043865755415, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900808080808081, + "loss": 2.748, + "theoretical_loss": 3.425495217826822, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039006060606060607, + "loss": 2.7024, + "theoretical_loss": 3.425486049456115, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390040404040404, + "loss": 2.6919, + "theoretical_loss": 3.425476881463392, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002020202020204, + "loss": 2.707, + "theoretical_loss": 3.425467713848625, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 2.5421, + "theoretical_loss": 3.425458546611787, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389979797979798, + "loss": 2.97, + "theoretical_loss": 3.42544937975285, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995959595959596, + "loss": 2.7948, + "theoretical_loss": 3.4254402132717856, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899393939393939, + "loss": 2.9629, + "theoretical_loss": 3.4254310471685665, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1150523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841942548751831, + "objective/train/theoretical_loss": 3.4254218814431656, + "objective/train/tokens_used": 394014176, + "theoretical_loss": 3.4254218814431656, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038991919191919193, + "loss": 2.7497, + "theoretical_loss": 3.4254218814431656, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038989898989898994, + "loss": 3.0841, + "theoretical_loss": 3.425412716095554, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898787878787879, + "loss": 2.7899, + "theoretical_loss": 3.4254035511257053, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985858585858585, + "loss": 2.7182, + "theoretical_loss": 3.4253943865335903, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898383838383838, + "loss": 2.7934, + "theoretical_loss": 3.425385222319182, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898181818181819, + "loss": 2.6814, + "theoretical_loss": 3.425376058482453, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038979797979797983, + "loss": 2.4324, + "theoretical_loss": 3.425366895023375, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897777777777778, + "loss": 2.6715, + "theoretical_loss": 3.425357731941921, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038975757575757575, + "loss": 2.6183, + "theoretical_loss": 3.425348569238062, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897373737373737, + "loss": 2.5573, + "theoretical_loss": 3.4253394069117715, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038971717171717177, + "loss": 2.7267, + "theoretical_loss": 3.425330244963021, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896969696969697, + "loss": 2.6806, + "theoretical_loss": 3.4253210833917835, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896767676767677, + "loss": 2.767, + "theoretical_loss": 3.425311922198031, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965656565656564, + "loss": 2.7242, + "theoretical_loss": 3.4253027613817357, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896363636363636, + "loss": 2.7352, + "theoretical_loss": 3.4252936009428696, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961616161616166, + "loss": 2.6166, + "theoretical_loss": 3.4252844408814056, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895959595959596, + "loss": 2.6686, + "theoretical_loss": 3.4252752811973157, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895757575757576, + "loss": 2.8643, + "theoretical_loss": 3.4252661218905724, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955555555555553, + "loss": 2.6064, + "theoretical_loss": 3.4252569629611473, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953535353535354, + "loss": 2.8099, + "theoretical_loss": 3.4252478044090138, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038951515151515155, + "loss": 2.3634, + "theoretical_loss": 3.4252386462341438, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894949494949495, + "loss": 2.6905, + "theoretical_loss": 3.4252294884365093, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947474747474747, + "loss": 2.902, + "theoretical_loss": 3.425220331016083, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894545454545454, + "loss": 2.6194, + "theoretical_loss": 3.4252111739728366, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038943434343434344, + "loss": 2.6187, + "theoretical_loss": 3.4252020173067432, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1151732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6099672317504883, + "objective/train/theoretical_loss": 3.4251928610177744, + "objective/train/tokens_used": 395652576, + "theoretical_loss": 3.4251928610177744, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038941414141414145, + "loss": 2.6192, + "theoretical_loss": 3.4251928610177744, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893939393939394, + "loss": 2.6262, + "theoretical_loss": 3.4251837051059035, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038937373737373736, + "loss": 2.4004, + "theoretical_loss": 3.425174549571102, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038935353535353537, + "loss": 2.6191, + "theoretical_loss": 3.4251653944133427, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038933333333333333, + "loss": 2.6853, + "theoretical_loss": 3.4251562396325976, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931313131313134, + "loss": 2.8017, + "theoretical_loss": 3.4251470852288395, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892929292929293, + "loss": 2.7768, + "theoretical_loss": 3.4251379312020402, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038927272727272725, + "loss": 2.6792, + "theoretical_loss": 3.4251287775521724, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038925252525252526, + "loss": 2.4803, + "theoretical_loss": 3.4251196242792084, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892323232323232, + "loss": 2.5962, + "theoretical_loss": 3.42511047138312, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038921212121212123, + "loss": 2.8596, + "theoretical_loss": 3.425101318863881, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891919191919192, + "loss": 2.7569, + "theoretical_loss": 3.4250921667214627, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891717171717172, + "loss": 2.5429, + "theoretical_loss": 3.4250830149558373, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915151515151516, + "loss": 2.6461, + "theoretical_loss": 3.4250738635669773, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891313131313131, + "loss": 2.5791, + "theoretical_loss": 3.425064712554856, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891111111111111, + "loss": 2.0854, + "theoretical_loss": 3.425055561919444, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890909090909091, + "loss": 2.583, + "theoretical_loss": 3.4250464116607153, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890707070707071, + "loss": 2.6636, + "theoretical_loss": 3.4250372617786415, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038905050505050505, + "loss": 2.6844, + "theoretical_loss": 3.4250281122731954, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389030303030303, + "loss": 2.6474, + "theoretical_loss": 3.4250189631443493, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389010101010101, + "loss": 2.7992, + "theoretical_loss": 3.425009814392075, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898989898989903, + "loss": 2.649, + "theoretical_loss": 3.425000666016346, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388969696969697, + "loss": 2.7291, + "theoretical_loss": 3.4249915180171335, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038894949494949494, + "loss": 2.498, + "theoretical_loss": 3.4249823703944102, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889292929292929, + "loss": 2.8527, + "theoretical_loss": 3.4249732231481493, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1152471, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.215996265411377, + "objective/train/theoretical_loss": 3.4249640762783224, + "objective/train/tokens_used": 397290976, + "theoretical_loss": 3.4249640762783224, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889090909090909, + "loss": 2.5325, + "theoretical_loss": 3.4249640762783224, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888888888888889, + "loss": 2.7907, + "theoretical_loss": 3.424954929784902, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888686868686869, + "loss": 2.5647, + "theoretical_loss": 3.424945783667861, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038884848484848483, + "loss": 2.7356, + "theoretical_loss": 3.4249366379271713, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888282828282828, + "loss": 2.7914, + "theoretical_loss": 3.4249274925628055, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038880808080808086, + "loss": 2.8081, + "theoretical_loss": 3.424918347574736, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887878787878788, + "loss": 2.7871, + "theoretical_loss": 3.4249092029629353, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038876767676767677, + "loss": 2.5257, + "theoretical_loss": 3.424900058727376, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874747474747473, + "loss": 2.9211, + "theoretical_loss": 3.4248909148680298, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038872727272727274, + "loss": 2.5878, + "theoretical_loss": 3.42488177138487, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870707070707075, + "loss": 2.5088, + "theoretical_loss": 3.424872628277868, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886868686868687, + "loss": 2.837, + "theoretical_loss": 3.4248634855469975, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866666666666666, + "loss": 2.5987, + "theoretical_loss": 3.42485434319223, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886464646464646, + "loss": 2.686, + "theoretical_loss": 3.4248452012135386, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886262626262627, + "loss": 2.843, + "theoretical_loss": 3.424836059610895, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038860606060606064, + "loss": 2.7346, + "theoretical_loss": 3.424826918384272, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885858585858586, + "loss": 2.8302, + "theoretical_loss": 3.424817777533643, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038856565656565656, + "loss": 2.6004, + "theoretical_loss": 3.4248086370589785, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885454545454545, + "loss": 2.7186, + "theoretical_loss": 3.4247994969602527, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885252525252526, + "loss": 2.6197, + "theoretical_loss": 3.4247903572374367, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850505050505054, + "loss": 2.7593, + "theoretical_loss": 3.424781217890504, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884848484848485, + "loss": 2.4425, + "theoretical_loss": 3.4247720789194274, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038846464646464645, + "loss": 2.5657, + "theoretical_loss": 3.424762940324178, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844444444444446, + "loss": 2.6833, + "theoretical_loss": 3.424753802104729, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842424242424247, + "loss": 2.8267, + "theoretical_loss": 3.4247446642610524, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1153876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2251062393188477, + "objective/train/theoretical_loss": 3.4247355267931217, + "objective/train/tokens_used": 398929376, + "theoretical_loss": 3.4247355267931217, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038840404040404043, + "loss": 2.5339, + "theoretical_loss": 3.4247355267931217, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883838383838384, + "loss": 2.5085, + "theoretical_loss": 3.4247263897009086, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836363636363634, + "loss": 2.7236, + "theoretical_loss": 3.4247172529843857, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834343434343435, + "loss": 2.7446, + "theoretical_loss": 3.4247081166435254, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038832323232323236, + "loss": 2.8569, + "theoretical_loss": 3.4246989806783006, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883030303030303, + "loss": 2.6384, + "theoretical_loss": 3.4246898450886833, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882828282828283, + "loss": 2.4971, + "theoretical_loss": 3.4246807098746466, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882626262626263, + "loss": 2.8035, + "theoretical_loss": 3.424671575036162, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824242424242425, + "loss": 2.7544, + "theoretical_loss": 3.424662440573203, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038822222222222226, + "loss": 2.5316, + "theoretical_loss": 3.4246533064857414, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882020202020202, + "loss": 2.783, + "theoretical_loss": 3.4246441727737507, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038818181818181817, + "loss": 2.7242, + "theoretical_loss": 3.424635039437202, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881616161616162, + "loss": 2.7523, + "theoretical_loss": 3.4246259064760687, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038814141414141414, + "loss": 2.5878, + "theoretical_loss": 3.4246167738903233, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038812121212121215, + "loss": 2.4679, + "theoretical_loss": 3.424607641679938, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881010101010101, + "loss": 2.6443, + "theoretical_loss": 3.424598509844886, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880808080808081, + "loss": 2.6904, + "theoretical_loss": 3.4245893783851384, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880606060606061, + "loss": 2.7208, + "theoretical_loss": 3.4245802473006695, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038804040404040403, + "loss": 2.7229, + "theoretical_loss": 3.4245711165914505, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802020202020204, + "loss": 2.6238, + "theoretical_loss": 3.4245619862574546, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388, + "loss": 2.6548, + "theoretical_loss": 3.424552856298654, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387979797979798, + "loss": 2.7181, + "theoretical_loss": 3.4245437267150214, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795959595959597, + "loss": 2.7901, + "theoretical_loss": 3.4245345975065296, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879393939393939, + "loss": 2.8573, + "theoretical_loss": 3.4245254686731506, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038791919191919193, + "loss": 2.5879, + "theoretical_loss": 3.424516340214857, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1154501, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466639280319214, + "objective/train/theoretical_loss": 3.424507212131622, + "objective/train/tokens_used": 400567776, + "theoretical_loss": 3.424507212131622, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878989898989899, + "loss": 2.8224, + "theoretical_loss": 3.424507212131622, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878787878787879, + "loss": 2.4791, + "theoretical_loss": 3.4244980844234174, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785858585858586, + "loss": 2.5703, + "theoretical_loss": 3.424488957090216, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878383838383838, + "loss": 2.6981, + "theoretical_loss": 3.4244798301319905, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878181818181818, + "loss": 2.7055, + "theoretical_loss": 3.424470703548714, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779797979797984, + "loss": 2.7377, + "theoretical_loss": 3.4244615773403577, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877777777777778, + "loss": 2.4511, + "theoretical_loss": 3.4244524515068946, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775757575757575, + "loss": 2.5792, + "theoretical_loss": 3.4244433260482983, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877373737373737, + "loss": 2.8483, + "theoretical_loss": 3.4244342009645403, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877171717171717, + "loss": 2.5931, + "theoretical_loss": 3.4244250762555937, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038769696969696973, + "loss": 2.6337, + "theoretical_loss": 3.424415951921431, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876767676767677, + "loss": 2.6645, + "theoretical_loss": 3.4244068279620246, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038765656565656564, + "loss": 2.6791, + "theoretical_loss": 3.4243977043773466, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876363636363636, + "loss": 2.8363, + "theoretical_loss": 3.4243885811673707, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038761616161616167, + "loss": 2.699, + "theoretical_loss": 3.424379458332069, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875959595959596, + "loss": 2.8426, + "theoretical_loss": 3.424370335871414, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875757575757576, + "loss": 2.884, + "theoretical_loss": 3.424361213785378, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755555555555554, + "loss": 2.9275, + "theoretical_loss": 3.424352092073934, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875353535353535, + "loss": 2.6075, + "theoretical_loss": 3.424342970737055, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038751515151515156, + "loss": 2.7817, + "theoretical_loss": 3.4243338497747127, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874949494949495, + "loss": 2.772, + "theoretical_loss": 3.42432472918688, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874747474747475, + "loss": 2.7887, + "theoretical_loss": 3.4243156089735303, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038745454545454543, + "loss": 2.8698, + "theoretical_loss": 3.424306489134635, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038743434343434344, + "loss": 2.7307, + "theoretical_loss": 3.4242973696701675, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741414141414145, + "loss": 2.575, + "theoretical_loss": 3.4242882505801, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1155127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2002182006835938, + "objective/train/theoretical_loss": 3.424279131864406, + "objective/train/tokens_used": 402206176, + "theoretical_loss": 3.424279131864406, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873939393939394, + "loss": 2.8677, + "theoretical_loss": 3.424279131864406, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038737373737373737, + "loss": 2.8405, + "theoretical_loss": 3.4242700135230564, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873535353535353, + "loss": 2.7887, + "theoretical_loss": 3.424260895556025, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038733333333333333, + "loss": 2.6928, + "theoretical_loss": 3.424251777963285, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038731313131313134, + "loss": 2.7426, + "theoretical_loss": 3.4242426607448078, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872929292929293, + "loss": 2.848, + "theoretical_loss": 3.424233543900567, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727272727272726, + "loss": 2.8563, + "theoretical_loss": 3.4242244274305347, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038725252525252527, + "loss": 2.6377, + "theoretical_loss": 3.4242153113346836, + "tokens_seen": 2043805696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872323232323232, + "loss": 2.6204, + "theoretical_loss": 3.424206195612986, + "tokens_seen": 2043871232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038721212121212124, + "loss": 2.5869, + "theoretical_loss": 3.424197080265415, + "tokens_seen": 2043936768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871919191919192, + "loss": 2.6809, + "theoretical_loss": 3.424187965291944, + "tokens_seen": 2044002304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038717171717171715, + "loss": 2.6924, + "theoretical_loss": 3.4241788506925444, + "tokens_seen": 2044067840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038715151515151516, + "loss": 2.7621, + "theoretical_loss": 3.4241697364671886, + "tokens_seen": 2044133376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313131313131, + "loss": 2.8142, + "theoretical_loss": 3.424160622615851, + "tokens_seen": 2044198912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711111111111113, + "loss": 2.7169, + "theoretical_loss": 3.424151509138502, + "tokens_seen": 2044264448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870909090909091, + "loss": 2.9172, + "theoretical_loss": 3.4241423960351165, + "tokens_seen": 2044329984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870707070707071, + "loss": 2.7656, + "theoretical_loss": 3.4241332833056664, + "tokens_seen": 2044395520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038705050505050505, + "loss": 2.3999, + "theoretical_loss": 3.4241241709501233, + "tokens_seen": 2044461056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703030303030307, + "loss": 2.8039, + "theoretical_loss": 3.424115058968461, + "tokens_seen": 2044526592 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387010101010101, + "loss": 2.4627, + "theoretical_loss": 3.424105947360652, + "tokens_seen": 2044592128 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386989898989899, + "loss": 2.6204, + "theoretical_loss": 3.424096836126669, + "tokens_seen": 2044657664 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386969696969697, + "loss": 2.5526, + "theoretical_loss": 3.4240877252664834, + "tokens_seen": 2044723200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038694949494949495, + "loss": 2.8785, + "theoretical_loss": 3.42407861478007, + "tokens_seen": 2044788736 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692929292929296, + "loss": 2.7852, + "theoretical_loss": 3.4240695046674006, + "tokens_seen": 2044854272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869090909090909, + "loss": 2.7448, + "theoretical_loss": 3.424060394928447, + "tokens_seen": 2044919808 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1156711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3755438327789307, + "objective/train/theoretical_loss": 3.4240512855631833, + "objective/train/tokens_used": 403844576, + "theoretical_loss": 3.4240512855631833, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868888888888889, + "loss": 2.6238, + "theoretical_loss": 3.4240512855631833, + "tokens_seen": 2044985344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868686868686869, + "loss": 2.6175, + "theoretical_loss": 3.424042176571582, + "tokens_seen": 2045050880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684848484848484, + "loss": 2.6898, + "theoretical_loss": 3.424033067953615, + "tokens_seen": 2045116416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682828282828285, + "loss": 2.6001, + "theoretical_loss": 3.4240239597092548, + "tokens_seen": 2045181952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868080808080808, + "loss": 2.5658, + "theoretical_loss": 3.424014851838475, + "tokens_seen": 2045247488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867878787878788, + "loss": 2.7502, + "theoretical_loss": 3.4240057443412484, + "tokens_seen": 2045313024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867676767676768, + "loss": 2.5943, + "theoretical_loss": 3.423996637217547, + "tokens_seen": 2045378560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038674747474747473, + "loss": 2.6764, + "theoretical_loss": 3.4239875304673437, + "tokens_seen": 2045444096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672727272727274, + "loss": 2.7065, + "theoretical_loss": 3.4239784240906115, + "tokens_seen": 2045509632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038670707070707076, + "loss": 2.5865, + "theoretical_loss": 3.423969318087323, + "tokens_seen": 2045575168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866868686868687, + "loss": 2.692, + "theoretical_loss": 3.423960212457451, + "tokens_seen": 2045640704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038666666666666667, + "loss": 2.894, + "theoretical_loss": 3.423951107200968, + "tokens_seen": 2045706240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866464646464646, + "loss": 2.4953, + "theoretical_loss": 3.423942002317846, + "tokens_seen": 2045771776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038662626262626264, + "loss": 2.7027, + "theoretical_loss": 3.4239328978080596, + "tokens_seen": 2045837312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038660606060606065, + "loss": 2.8213, + "theoretical_loss": 3.4239237936715803, + "tokens_seen": 2045902848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865858585858586, + "loss": 2.5695, + "theoretical_loss": 3.4239146899083805, + "tokens_seen": 2045968384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656565656565656, + "loss": 2.6575, + "theoretical_loss": 3.4239055865184342, + "tokens_seen": 2046033920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865454545454545, + "loss": 2.2676, + "theoretical_loss": 3.423896483501713, + "tokens_seen": 2046099456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865252525252526, + "loss": 2.5913, + "theoretical_loss": 3.42388738085819, + "tokens_seen": 2046164992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038650505050505054, + "loss": 2.624, + "theoretical_loss": 3.423878278587838, + "tokens_seen": 2046230528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864848484848485, + "loss": 2.8434, + "theoretical_loss": 3.42386917669063, + "tokens_seen": 2046296064 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038646464646464645, + "loss": 2.7237, + "theoretical_loss": 3.4238600751665382, + "tokens_seen": 2046361600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864444444444444, + "loss": 2.6661, + "theoretical_loss": 3.4238509740155356, + "tokens_seen": 2046427136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864242424242425, + "loss": 2.779, + "theoretical_loss": 3.4238418732375955, + "tokens_seen": 2046492672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640404040404043, + "loss": 2.8329, + "theoretical_loss": 3.42383277283269, + "tokens_seen": 2046558208 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 1157531, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3749358654022217, + "objective/train/theoretical_loss": 3.4238236728007916, + "objective/train/tokens_used": 405482976, + "theoretical_loss": 3.4238236728007916, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003863838383838384, + "loss": 2.5916, + "theoretical_loss": 3.4238236728007916, + "tokens_seen": 2046623744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038636363636363635, + "loss": 2.7349, + "theoretical_loss": 3.4238145731418737, + "tokens_seen": 2046689280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863434343434343, + "loss": 2.6268, + "theoretical_loss": 3.4238054738559094, + "tokens_seen": 2046754816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632323232323237, + "loss": 2.7738, + "theoretical_loss": 3.4237963749428704, + "tokens_seen": 2046820352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863030303030303, + "loss": 2.8664, + "theoretical_loss": 3.4237872764027304, + "tokens_seen": 2046885888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862828282828283, + "loss": 2.7549, + "theoretical_loss": 3.4237781782354615, + "tokens_seen": 2046951424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626262626262624, + "loss": 2.6667, + "theoretical_loss": 3.423769080441037, + "tokens_seen": 2047016960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038624242424242425, + "loss": 2.8541, + "theoretical_loss": 3.423759983019429, + "tokens_seen": 2047082496 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622222222222226, + "loss": 2.6181, + "theoretical_loss": 3.4237508859706116, + "tokens_seen": 2047148032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862020202020202, + "loss": 2.9649, + "theoretical_loss": 3.423741789294556, + "tokens_seen": 2047213568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861818181818182, + "loss": 2.5546, + "theoretical_loss": 3.423732692991236, + "tokens_seen": 2047279104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038616161616161613, + "loss": 2.772, + "theoretical_loss": 3.4237235970606243, + "tokens_seen": 2047344640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614141414141414, + "loss": 2.7947, + "theoretical_loss": 3.4237145015026935, + "tokens_seen": 2047410176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038612121212121215, + "loss": 2.7218, + "theoretical_loss": 3.4237054063174166, + "tokens_seen": 2047475712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861010101010101, + "loss": 2.8158, + "theoretical_loss": 3.4236963115047656, + "tokens_seen": 2047541248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038608080808080807, + "loss": 2.7878, + "theoretical_loss": 3.423687217064715, + "tokens_seen": 2047606784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860606060606061, + "loss": 2.6168, + "theoretical_loss": 3.4236781229972353, + "tokens_seen": 2047672320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038604040404040404, + "loss": 2.8668, + "theoretical_loss": 3.4236690293023018, + "tokens_seen": 2047737856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602020202020205, + "loss": 2.4426, + "theoretical_loss": 3.4236599359798854, + "tokens_seen": 2047803392 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386, + "loss": 2.4617, + "theoretical_loss": 3.42365084302996, + "tokens_seen": 2047868928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038597979797979796, + "loss": 2.7245, + "theoretical_loss": 3.4236417504524974, + "tokens_seen": 2047934464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038595959595959597, + "loss": 2.6945, + "theoretical_loss": 3.4236326582474716, + "tokens_seen": 2048000000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593939393939393, + "loss": 2.4532, + "theoretical_loss": 3.4236235664148547, + "tokens_seen": 2048065536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591919191919194, + "loss": 2.7428, + "theoretical_loss": 3.42361447495462, + "tokens_seen": 2048131072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858989898989899, + "loss": 2.487, + "theoretical_loss": 3.4236053838667395, + "tokens_seen": 2048196608 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1159006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.661281108856201, + "objective/train/theoretical_loss": 3.4235962931511867, + "objective/train/tokens_used": 407121376, + "theoretical_loss": 3.4235962931511867, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858787878787879, + "loss": 2.8251, + "theoretical_loss": 3.4235962931511867, + "tokens_seen": 2048262144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585858585858586, + "loss": 2.7517, + "theoretical_loss": 3.4235872028079353, + "tokens_seen": 2048327680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858383838383838, + "loss": 2.7318, + "theoretical_loss": 3.423578112836956, + "tokens_seen": 2048393216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581818181818183, + "loss": 2.7951, + "theoretical_loss": 3.4235690232382234, + "tokens_seen": 2048458752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857979797979798, + "loss": 2.718, + "theoretical_loss": 3.4235599340117098, + "tokens_seen": 2048524288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857777777777778, + "loss": 2.6979, + "theoretical_loss": 3.4235508451573873, + "tokens_seen": 2048589824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038575757575757576, + "loss": 2.6459, + "theoretical_loss": 3.42354175667523, + "tokens_seen": 2048655360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857373737373737, + "loss": 2.7458, + "theoretical_loss": 3.42353266856521, + "tokens_seen": 2048720896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171717171717, + "loss": 2.591, + "theoretical_loss": 3.4235235808273012, + "tokens_seen": 2048786432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569696969696974, + "loss": 2.6198, + "theoretical_loss": 3.4235144934614747, + "tokens_seen": 2048851968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856767676767677, + "loss": 2.5969, + "theoretical_loss": 3.423505406467705, + "tokens_seen": 2048917504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565656565656565, + "loss": 2.4446, + "theoretical_loss": 3.4234963198459636, + "tokens_seen": 2048983040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856363636363636, + "loss": 2.7546, + "theoretical_loss": 3.4234872335962248, + "tokens_seen": 2049048576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856161616161616, + "loss": 2.8734, + "theoretical_loss": 3.42347814771846, + "tokens_seen": 2049114112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038559595959595963, + "loss": 2.5708, + "theoretical_loss": 3.423469062212643, + "tokens_seen": 2049179648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855757575757576, + "loss": 2.8648, + "theoretical_loss": 3.4234599770787466, + "tokens_seen": 2049245184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038555555555555554, + "loss": 2.4264, + "theoretical_loss": 3.4234508923167435, + "tokens_seen": 2049310720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855353535353535, + "loss": 2.7387, + "theoretical_loss": 3.423441807926607, + "tokens_seen": 2049376256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038551515151515156, + "loss": 2.3965, + "theoretical_loss": 3.423432723908309, + "tokens_seen": 2049441792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854949494949495, + "loss": 2.5871, + "theoretical_loss": 3.423423640261823, + "tokens_seen": 2049507328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854747474747475, + "loss": 2.9884, + "theoretical_loss": 3.4234145569871224, + "tokens_seen": 2049572864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545454545454544, + "loss": 2.756, + "theoretical_loss": 3.4234054740841797, + "tokens_seen": 2049638400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854343434343434, + "loss": 2.6088, + "theoretical_loss": 3.4233963915529673, + "tokens_seen": 2049703936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038541414141414146, + "loss": 2.7773, + "theoretical_loss": 3.4233873093934584, + "tokens_seen": 2049769472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853939393939394, + "loss": 2.7383, + "theoretical_loss": 3.4233782276056264, + "tokens_seen": 2049835008 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1159734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0860424041748047, + "objective/train/theoretical_loss": 3.4233691461894438, + "objective/train/tokens_used": 408759776, + "theoretical_loss": 3.4233691461894438, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537373737373737, + "loss": 2.9101, + "theoretical_loss": 3.4233691461894438, + "tokens_seen": 2049900544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535353535353533, + "loss": 2.5485, + "theoretical_loss": 3.4233600651448834, + "tokens_seen": 2049966080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853333333333334, + "loss": 2.5889, + "theoretical_loss": 3.4233509844719183, + "tokens_seen": 2050031616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531313131313135, + "loss": 2.7695, + "theoretical_loss": 3.4233419041705213, + "tokens_seen": 2050097152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852929292929293, + "loss": 2.796, + "theoretical_loss": 3.4233328242406653, + "tokens_seen": 2050162688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527272727272726, + "loss": 2.6361, + "theoretical_loss": 3.4233237446823237, + "tokens_seen": 2050228224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852525252525252, + "loss": 2.6807, + "theoretical_loss": 3.4233146654954685, + "tokens_seen": 2050293760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852323232323233, + "loss": 2.6836, + "theoretical_loss": 3.4233055866800735, + "tokens_seen": 2050359296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038521212121212124, + "loss": 2.6654, + "theoretical_loss": 3.4232965082361106, + "tokens_seen": 2050424832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851919191919192, + "loss": 2.5232, + "theoretical_loss": 3.4232874301635543, + "tokens_seen": 2050490368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038517171717171716, + "loss": 2.9194, + "theoretical_loss": 3.4232783524623764, + "tokens_seen": 2050555904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515151515151517, + "loss": 2.3964, + "theoretical_loss": 3.42326927513255, + "tokens_seen": 2050621440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851313131313132, + "loss": 2.7513, + "theoretical_loss": 3.423260198174048, + "tokens_seen": 2050686976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038511111111111114, + "loss": 3.0081, + "theoretical_loss": 3.4232511215868433, + "tokens_seen": 2050752512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850909090909091, + "loss": 2.4918, + "theoretical_loss": 3.42324204537091, + "tokens_seen": 2050818048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038507070707070705, + "loss": 2.7725, + "theoretical_loss": 3.423232969526219, + "tokens_seen": 2050883584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038505050505050506, + "loss": 2.6275, + "theoretical_loss": 3.423223894052745, + "tokens_seen": 2050949120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038503030303030307, + "loss": 2.8104, + "theoretical_loss": 3.4232148189504596, + "tokens_seen": 2051014656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501010101010103, + "loss": 2.5709, + "theoretical_loss": 3.423205744219337, + "tokens_seen": 2051080192 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384989898989899, + "loss": 2.6706, + "theoretical_loss": 3.4231966698593492, + "tokens_seen": 2051145728 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384969696969697, + "loss": 2.8282, + "theoretical_loss": 3.42318759587047, + "tokens_seen": 2051211264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494949494949495, + "loss": 2.6249, + "theoretical_loss": 3.423178522252672, + "tokens_seen": 2051276800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038492929292929296, + "loss": 2.6344, + "theoretical_loss": 3.4231694490059272, + "tokens_seen": 2051342336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849090909090909, + "loss": 2.4384, + "theoretical_loss": 3.4231603761302107, + "tokens_seen": 2051407872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848888888888889, + "loss": 2.6242, + "theoretical_loss": 3.4231513036254935, + "tokens_seen": 2051473408 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1160859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.563382625579834, + "objective/train/theoretical_loss": 3.4231422314917497, + "objective/train/tokens_used": 410398176, + "theoretical_loss": 3.4231422314917497, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848686868686869, + "loss": 2.7752, + "theoretical_loss": 3.4231422314917497, + "tokens_seen": 2051538944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038484848484848485, + "loss": 2.8138, + "theoretical_loss": 3.423133159728951, + "tokens_seen": 2051604480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482828282828286, + "loss": 2.7121, + "theoretical_loss": 3.4231240883370724, + "tokens_seen": 2051670016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848080808080808, + "loss": 2.6529, + "theoretical_loss": 3.423115017316085, + "tokens_seen": 2051735552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478787878787877, + "loss": 2.595, + "theoretical_loss": 3.423105946665963, + "tokens_seen": 2051801088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847676767676768, + "loss": 2.7051, + "theoretical_loss": 3.423096876386679, + "tokens_seen": 2051866624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474747474747474, + "loss": 2.6216, + "theoretical_loss": 3.4230878064782058, + "tokens_seen": 2051932160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038472727272727275, + "loss": 2.6483, + "theoretical_loss": 3.4230787369405165, + "tokens_seen": 2051997696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847070707070707, + "loss": 2.5571, + "theoretical_loss": 3.4230696677735843, + "tokens_seen": 2052063232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846868686868687, + "loss": 2.7188, + "theoretical_loss": 3.423060598977382, + "tokens_seen": 2052128768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846666666666667, + "loss": 2.8119, + "theoretical_loss": 3.4230515305518825, + "tokens_seen": 2052194304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038464646464646463, + "loss": 2.74, + "theoretical_loss": 3.423042462497059, + "tokens_seen": 2052259840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462626262626264, + "loss": 2.8119, + "theoretical_loss": 3.423033394812885, + "tokens_seen": 2052325376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846060606060606, + "loss": 2.7423, + "theoretical_loss": 3.4230243274993324, + "tokens_seen": 2052390912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845858585858586, + "loss": 2.7735, + "theoretical_loss": 3.423015260556375, + "tokens_seen": 2052456448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456565656565657, + "loss": 2.6011, + "theoretical_loss": 3.4230061939839858, + "tokens_seen": 2052521984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845454545454545, + "loss": 2.6725, + "theoretical_loss": 3.4229971277821374, + "tokens_seen": 2052587520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038452525252525254, + "loss": 2.6808, + "theoretical_loss": 3.4229880619508033, + "tokens_seen": 2052653056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450505050505055, + "loss": 2.6267, + "theoretical_loss": 3.422978996489956, + "tokens_seen": 2052718592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844848484848485, + "loss": 2.9926, + "theoretical_loss": 3.422969931399569, + "tokens_seen": 2052784128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038446464646464646, + "loss": 2.5455, + "theoretical_loss": 3.422960866679616, + "tokens_seen": 2052849664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844444444444444, + "loss": 2.7958, + "theoretical_loss": 3.422951802330068, + "tokens_seen": 2052915200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038442424242424243, + "loss": 2.7347, + "theoretical_loss": 3.4229427383508995, + "tokens_seen": 2052980736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440404040404044, + "loss": 2.5218, + "theoretical_loss": 3.422933674742084, + "tokens_seen": 2053046272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843838383838384, + "loss": 2.6591, + "theoretical_loss": 3.422924611503593, + "tokens_seen": 2053111808 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1161710, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.472203493118286, + "objective/train/theoretical_loss": 3.422915548635401, + "objective/train/tokens_used": 412036576, + "theoretical_loss": 3.422915548635401, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436363636363635, + "loss": 2.6554, + "theoretical_loss": 3.422915548635401, + "tokens_seen": 2053177344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843434343434343, + "loss": 2.5544, + "theoretical_loss": 3.4229064861374803, + "tokens_seen": 2053242880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843232323232324, + "loss": 2.7006, + "theoretical_loss": 3.422897424009804, + "tokens_seen": 2053308416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430303030303033, + "loss": 2.777, + "theoretical_loss": 3.422888362252345, + "tokens_seen": 2053373952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828282828283, + "loss": 2.8403, + "theoretical_loss": 3.4228793008650773, + "tokens_seen": 2053439488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426262626262625, + "loss": 2.6706, + "theoretical_loss": 3.422870239847973, + "tokens_seen": 2053505024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842424242424242, + "loss": 2.6209, + "theoretical_loss": 3.4228611792010053, + "tokens_seen": 2053570560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422222222222227, + "loss": 2.8026, + "theoretical_loss": 3.4228521189241476, + "tokens_seen": 2053636096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842020202020202, + "loss": 2.687, + "theoretical_loss": 3.4228430590173726, + "tokens_seen": 2053701632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841818181818182, + "loss": 2.614, + "theoretical_loss": 3.4228339994806536, + "tokens_seen": 2053767168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038416161616161614, + "loss": 2.84, + "theoretical_loss": 3.422824940313964, + "tokens_seen": 2053832704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414141414141415, + "loss": 2.4825, + "theoretical_loss": 3.422815881517276, + "tokens_seen": 2053898240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038412121212121216, + "loss": 2.617, + "theoretical_loss": 3.4228068230905633, + "tokens_seen": 2053963776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841010101010101, + "loss": 2.6991, + "theoretical_loss": 3.422797765033799, + "tokens_seen": 2054029312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840808080808081, + "loss": 2.6298, + "theoretical_loss": 3.4227887073469563, + "tokens_seen": 2054094848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406060606060603, + "loss": 2.5942, + "theoretical_loss": 3.422779650030008, + "tokens_seen": 2054160384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404040404040404, + "loss": 2.5978, + "theoretical_loss": 3.422770593082927, + "tokens_seen": 2054225920 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038402020202020205, + "loss": 2.8414, + "theoretical_loss": 3.422761536505687, + "tokens_seen": 2054291456 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384, + "loss": 2.7483, + "theoretical_loss": 3.4227524802982607, + "tokens_seen": 2054356992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038397979797979797, + "loss": 2.6539, + "theoretical_loss": 3.422743424460621, + "tokens_seen": 2054422528 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383959595959596, + "loss": 2.6057, + "theoretical_loss": 3.4227343689927414, + "tokens_seen": 2054488064 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038393939393939393, + "loss": 2.6712, + "theoretical_loss": 3.4227253138945946, + "tokens_seen": 2054553600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391919191919195, + "loss": 2.5268, + "theoretical_loss": 3.4227162591661546, + "tokens_seen": 2054619136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838989898989899, + "loss": 2.7206, + "theoretical_loss": 3.422707204807393, + "tokens_seen": 2054684672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038387878787878786, + "loss": 2.4685, + "theoretical_loss": 3.4226981508182845, + "tokens_seen": 2054750208 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1163069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820699453353882, + "objective/train/theoretical_loss": 3.422689097198802, + "objective/train/tokens_used": 413674976, + "theoretical_loss": 3.422689097198802, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038385858585858587, + "loss": 2.7206, + "theoretical_loss": 3.422689097198802, + "tokens_seen": 2054815744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383838383838383, + "loss": 2.5569, + "theoretical_loss": 3.422680043948917, + "tokens_seen": 2054881280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038381818181818184, + "loss": 2.8311, + "theoretical_loss": 3.422670991068605, + "tokens_seen": 2054946816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837979797979798, + "loss": 2.8243, + "theoretical_loss": 3.422661938557837, + "tokens_seen": 2055012352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837777777777778, + "loss": 2.8747, + "theoretical_loss": 3.4226528864165875, + "tokens_seen": 2055077888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375757575757576, + "loss": 2.8282, + "theoretical_loss": 3.4226438346448287, + "tokens_seen": 2055143424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837373737373737, + "loss": 2.4812, + "theoretical_loss": 3.422634783242535, + "tokens_seen": 2055208960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371717171717173, + "loss": 2.7346, + "theoretical_loss": 3.422625732209678, + "tokens_seen": 2055274496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836969696969697, + "loss": 2.5909, + "theoretical_loss": 3.4226166815462316, + "tokens_seen": 2055340032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836767676767677, + "loss": 2.5944, + "theoretical_loss": 3.42260763125217, + "tokens_seen": 2055405568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038365656565656566, + "loss": 2.681, + "theoretical_loss": 3.422598581327464, + "tokens_seen": 2055471104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836363636363636, + "loss": 2.6753, + "theoretical_loss": 3.4225895317720885, + "tokens_seen": 2055536640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836161616161616, + "loss": 2.6082, + "theoretical_loss": 3.4225804825860164, + "tokens_seen": 2055602176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038359595959595963, + "loss": 2.678, + "theoretical_loss": 3.42257143376922, + "tokens_seen": 2055667712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835757575757576, + "loss": 2.6929, + "theoretical_loss": 3.422562385321674, + "tokens_seen": 2055733248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355555555555555, + "loss": 2.9027, + "theoretical_loss": 3.4225533372433503, + "tokens_seen": 2055798784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038353535353535356, + "loss": 2.6816, + "theoretical_loss": 3.422544289534222, + "tokens_seen": 2055864320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835151515151515, + "loss": 2.663, + "theoretical_loss": 3.422535242194263, + "tokens_seen": 2055929856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349494949494953, + "loss": 2.6116, + "theoretical_loss": 3.4225261952234467, + "tokens_seen": 2055995392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834747474747475, + "loss": 2.8447, + "theoretical_loss": 3.422517148621745, + "tokens_seen": 2056060928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038345454545454544, + "loss": 2.7536, + "theoretical_loss": 3.4225081023891324, + "tokens_seen": 2056126464 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343434343434345, + "loss": 2.9071, + "theoretical_loss": 3.422499056525581, + "tokens_seen": 2056192000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341414141414146, + "loss": 2.2917, + "theoretical_loss": 3.4224900110310648, + "tokens_seen": 2056257536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833939393939394, + "loss": 2.6824, + "theoretical_loss": 3.422480965905556, + "tokens_seen": 2056323072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833737373737374, + "loss": 2.6659, + "theoretical_loss": 3.4224719211490293, + "tokens_seen": 2056388608 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1163822, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.876322031021118, + "objective/train/theoretical_loss": 3.4224628767614567, + "objective/train/tokens_used": 415313376, + "theoretical_loss": 3.4224628767614567, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335353535353533, + "loss": 2.8335, + "theoretical_loss": 3.4224628767614567, + "tokens_seen": 2056454144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038333333333333334, + "loss": 2.7979, + "theoretical_loss": 3.4224538327428116, + "tokens_seen": 2056519680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038331313131313136, + "loss": 2.4081, + "theoretical_loss": 3.4224447890930674, + "tokens_seen": 2056585216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832929292929293, + "loss": 2.6375, + "theoretical_loss": 3.422435745812197, + "tokens_seen": 2056650752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038327272727272727, + "loss": 2.9092, + "theoretical_loss": 3.4224267029001743, + "tokens_seen": 2056716288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832525252525252, + "loss": 2.7532, + "theoretical_loss": 3.422417660356972, + "tokens_seen": 2056781824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832323232323233, + "loss": 2.3631, + "theoretical_loss": 3.422408618182563, + "tokens_seen": 2056847360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321212121212125, + "loss": 2.6525, + "theoretical_loss": 3.422399576376921, + "tokens_seen": 2056912896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831919191919192, + "loss": 2.6081, + "theoretical_loss": 3.422390534940019, + "tokens_seen": 2056978432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317171717171716, + "loss": 2.5723, + "theoretical_loss": 3.42238149387183, + "tokens_seen": 2057043968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831515151515151, + "loss": 3.0293, + "theoretical_loss": 3.4223724531723283, + "tokens_seen": 2057109504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831313131313132, + "loss": 2.652, + "theoretical_loss": 3.4223634128414853, + "tokens_seen": 2057175040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311111111111114, + "loss": 2.7144, + "theoretical_loss": 3.422354372879276, + "tokens_seen": 2057240576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830909090909091, + "loss": 2.6724, + "theoretical_loss": 3.4223453332856724, + "tokens_seen": 2057306112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038307070707070706, + "loss": 2.729, + "theoretical_loss": 3.4223362940606483, + "tokens_seen": 2057371648 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383050505050505, + "loss": 2.7526, + "theoretical_loss": 3.422327255204177, + "tokens_seen": 2057437184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830303030303031, + "loss": 2.5234, + "theoretical_loss": 3.4223182167162314, + "tokens_seen": 2057502720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301010101010103, + "loss": 2.702, + "theoretical_loss": 3.4223091785967847, + "tokens_seen": 2057568256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382989898989899, + "loss": 2.6828, + "theoretical_loss": 3.4223001408458105, + "tokens_seen": 2057633792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038296969696969695, + "loss": 2.5764, + "theoretical_loss": 3.422291103463282, + "tokens_seen": 2057699328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038294949494949496, + "loss": 2.5912, + "theoretical_loss": 3.422282066449172, + "tokens_seen": 2057764864 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292929292929297, + "loss": 2.7335, + "theoretical_loss": 3.4222730298034545, + "tokens_seen": 2057830400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829090909090909, + "loss": 2.6719, + "theoretical_loss": 3.4222639935261014, + "tokens_seen": 2057895936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828888888888889, + "loss": 2.7827, + "theoretical_loss": 3.4222549576170875, + "tokens_seen": 2057961472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038286868686868684, + "loss": 2.7703, + "theoretical_loss": 3.422245922076386, + "tokens_seen": 2058027008 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1164494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.458921194076538, + "objective/train/theoretical_loss": 3.4222368869039688, + "objective/train/tokens_used": 416951776, + "theoretical_loss": 3.4222368869039688, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284848484848485, + "loss": 2.4617, + "theoretical_loss": 3.4222368869039688, + "tokens_seen": 2058092544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038282828282828286, + "loss": 2.456, + "theoretical_loss": 3.42222785209981, + "tokens_seen": 2058158080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828080808080808, + "loss": 2.7494, + "theoretical_loss": 3.4222188176638833, + "tokens_seen": 2058223616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827878787878788, + "loss": 2.767, + "theoretical_loss": 3.422209783596161, + "tokens_seen": 2058289152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827676767676768, + "loss": 2.9014, + "theoretical_loss": 3.422200749896617, + "tokens_seen": 2058354688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038274747474747474, + "loss": 2.4356, + "theoretical_loss": 3.4221917165652243, + "tokens_seen": 2058420224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272727272727276, + "loss": 2.6735, + "theoretical_loss": 3.4221826836019567, + "tokens_seen": 2058485760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827070707070707, + "loss": 2.7723, + "theoretical_loss": 3.4221736510067866, + "tokens_seen": 2058551296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268686868686867, + "loss": 2.7285, + "theoretical_loss": 3.422164618779688, + "tokens_seen": 2058616832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826666666666667, + "loss": 2.7043, + "theoretical_loss": 3.422155586920634, + "tokens_seen": 2058682368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264646464646464, + "loss": 2.6052, + "theoretical_loss": 3.4221465554295976, + "tokens_seen": 2058747904 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262626262626265, + "loss": 2.7988, + "theoretical_loss": 3.422137524306552, + "tokens_seen": 2058813440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826060606060606, + "loss": 2.6319, + "theoretical_loss": 3.4221284935514715, + "tokens_seen": 2058878976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825858585858586, + "loss": 2.4577, + "theoretical_loss": 3.4221194631643286, + "tokens_seen": 2058944512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038256565656565657, + "loss": 2.394, + "theoretical_loss": 3.4221104331450967, + "tokens_seen": 2059010048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254545454545453, + "loss": 2.5179, + "theoretical_loss": 3.4221014034937487, + "tokens_seen": 2059075584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252525252525254, + "loss": 2.5771, + "theoretical_loss": 3.4220923742102585, + "tokens_seen": 2059141120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825050505050505, + "loss": 2.6255, + "theoretical_loss": 3.4220833452945993, + "tokens_seen": 2059206656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824848484848485, + "loss": 2.6996, + "theoretical_loss": 3.4220743167467442, + "tokens_seen": 2059272192 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246464646464647, + "loss": 2.8315, + "theoretical_loss": 3.422065288566667, + "tokens_seen": 2059337728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824444444444444, + "loss": 2.6656, + "theoretical_loss": 3.4220562607543403, + "tokens_seen": 2059403264 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038242424242424243, + "loss": 2.771, + "theoretical_loss": 3.422047233309738, + "tokens_seen": 2059468800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038240404040404044, + "loss": 2.8466, + "theoretical_loss": 3.422038206232833, + "tokens_seen": 2059534336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823838383838384, + "loss": 2.6629, + "theoretical_loss": 3.422029179523599, + "tokens_seen": 2059599872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038236363636363636, + "loss": 2.7048, + "theoretical_loss": 3.422020153182009, + "tokens_seen": 2059665408 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1165816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6828415393829346, + "objective/train/theoretical_loss": 3.4220111272080365, + "objective/train/tokens_used": 418590176, + "theoretical_loss": 3.4220111272080365, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823434343434343, + "loss": 2.5248, + "theoretical_loss": 3.4220111272080365, + "tokens_seen": 2059730944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823232323232323, + "loss": 2.8162, + "theoretical_loss": 3.4220021016016546, + "tokens_seen": 2059796480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230303030303034, + "loss": 2.7292, + "theoretical_loss": 3.4219930763628374, + "tokens_seen": 2059862016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822828282828283, + "loss": 2.5979, + "theoretical_loss": 3.421984051491557, + "tokens_seen": 2059927552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226262626262625, + "loss": 2.6414, + "theoretical_loss": 3.4219750269877878, + "tokens_seen": 2059993088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822424242424242, + "loss": 2.7057, + "theoretical_loss": 3.4219660028515024, + "tokens_seen": 2060058624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822222222222223, + "loss": 2.6483, + "theoretical_loss": 3.421956979082675, + "tokens_seen": 2060124160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038220202020202023, + "loss": 2.5842, + "theoretical_loss": 3.4219479556812784, + "tokens_seen": 2060189696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821818181818182, + "loss": 2.603, + "theoretical_loss": 3.421938932647286, + "tokens_seen": 2060255232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038216161616161614, + "loss": 2.8263, + "theoretical_loss": 3.421929909980671, + "tokens_seen": 2060320768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821414141414141, + "loss": 2.7983, + "theoretical_loss": 3.4219208876814067, + "tokens_seen": 2060386304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212121212121217, + "loss": 2.3082, + "theoretical_loss": 3.4219118657494674, + "tokens_seen": 2060451840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821010101010101, + "loss": 2.6501, + "theoretical_loss": 3.421902844184825, + "tokens_seen": 2060517376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820808080808081, + "loss": 2.5932, + "theoretical_loss": 3.421893822987454, + "tokens_seen": 2060582912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038206060606060604, + "loss": 2.6664, + "theoretical_loss": 3.421884802157327, + "tokens_seen": 2060648448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204040404040405, + "loss": 2.6397, + "theoretical_loss": 3.4218757816944185, + "tokens_seen": 2060713984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202020202020206, + "loss": 2.6347, + "theoretical_loss": 3.4218667615987, + "tokens_seen": 2060779520 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382, + "loss": 2.6067, + "theoretical_loss": 3.421857741870147, + "tokens_seen": 2060845056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197979797979797, + "loss": 2.7393, + "theoretical_loss": 3.4218487225087317, + "tokens_seen": 2060910592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038195959595959593, + "loss": 2.6498, + "theoretical_loss": 3.4218397035144275, + "tokens_seen": 2060976128 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038193939393939394, + "loss": 2.7838, + "theoretical_loss": 3.4218306848872073, + "tokens_seen": 2061041664 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038191919191919195, + "loss": 2.5641, + "theoretical_loss": 3.421821666627046, + "tokens_seen": 2061107200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818989898989899, + "loss": 2.6807, + "theoretical_loss": 3.4218126487339156, + "tokens_seen": 2061172736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038187878787878786, + "loss": 2.9635, + "theoretical_loss": 3.4218036312077906, + "tokens_seen": 2061238272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818585858585859, + "loss": 2.6212, + "theoretical_loss": 3.4217946140486433, + "tokens_seen": 2061303808 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1166525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.661501407623291, + "objective/train/theoretical_loss": 3.4217855972564477, + "objective/train/tokens_used": 420228576, + "theoretical_loss": 3.4217855972564477, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818383838383839, + "loss": 2.6494, + "theoretical_loss": 3.4217855972564477, + "tokens_seen": 2061369344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181818181818184, + "loss": 2.6022, + "theoretical_loss": 3.4217765808311773, + "tokens_seen": 2061434880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817979797979798, + "loss": 2.6889, + "theoretical_loss": 3.4217675647728054, + "tokens_seen": 2061500416 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177777777777776, + "loss": 2.7851, + "theoretical_loss": 3.4217585490813045, + "tokens_seen": 2061565952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038175757575757577, + "loss": 2.7207, + "theoretical_loss": 3.4217495337566497, + "tokens_seen": 2061631488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817373737373738, + "loss": 2.6481, + "theoretical_loss": 3.4217405187988135, + "tokens_seen": 2061697024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038171717171717174, + "loss": 2.5882, + "theoretical_loss": 3.421731504207769, + "tokens_seen": 2061762560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816969696969697, + "loss": 2.6914, + "theoretical_loss": 3.42172248998349, + "tokens_seen": 2061828096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816767676767677, + "loss": 2.6684, + "theoretical_loss": 3.42171347612595, + "tokens_seen": 2061893632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165656565656566, + "loss": 2.7583, + "theoretical_loss": 3.4217044626351223, + "tokens_seen": 2061959168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038163636363636367, + "loss": 2.7088, + "theoretical_loss": 3.42169544951098, + "tokens_seen": 2062024704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038161616161616163, + "loss": 2.6417, + "theoretical_loss": 3.4216864367534976, + "tokens_seen": 2062090240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815959595959596, + "loss": 2.5028, + "theoretical_loss": 3.4216774243626475, + "tokens_seen": 2062155776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815757575757576, + "loss": 2.661, + "theoretical_loss": 3.4216684123384034, + "tokens_seen": 2062221312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038155555555555555, + "loss": 2.7357, + "theoretical_loss": 3.421659400680739, + "tokens_seen": 2062286848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038153535353535357, + "loss": 2.8299, + "theoretical_loss": 3.421650389389627, + "tokens_seen": 2062352384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815151515151515, + "loss": 2.6463, + "theoretical_loss": 3.4216413784650417, + "tokens_seen": 2062417920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814949494949495, + "loss": 2.6475, + "theoretical_loss": 3.4216323679069562, + "tokens_seen": 2062483456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814747474747475, + "loss": 2.6553, + "theoretical_loss": 3.421623357715344, + "tokens_seen": 2062548992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038145454545454545, + "loss": 2.6585, + "theoretical_loss": 3.421614347890178, + "tokens_seen": 2062614528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143434343434346, + "loss": 2.7044, + "theoretical_loss": 3.4216053384314327, + "tokens_seen": 2062680064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814141414141414, + "loss": 2.6449, + "theoretical_loss": 3.4215963293390805, + "tokens_seen": 2062745600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813939393939394, + "loss": 2.5879, + "theoretical_loss": 3.4215873206130962, + "tokens_seen": 2062811136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813737373737374, + "loss": 2.5786, + "theoretical_loss": 3.4215783122534518, + "tokens_seen": 2062876672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038135353535353534, + "loss": 2.6952, + "theoretical_loss": 3.421569304260122, + "tokens_seen": 2062942208 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 1167825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8658223152160645, + "objective/train/theoretical_loss": 3.421560296633079, + "objective/train/tokens_used": 421866976, + "theoretical_loss": 3.421560296633079, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038133333333333335, + "loss": 2.7143, + "theoretical_loss": 3.421560296633079, + "tokens_seen": 2063007744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813131313131313, + "loss": 2.5177, + "theoretical_loss": 3.4215512893722972, + "tokens_seen": 2063073280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812929292929293, + "loss": 2.7136, + "theoretical_loss": 3.4215422824777497, + "tokens_seen": 2063138816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812727272727273, + "loss": 2.4694, + "theoretical_loss": 3.42153327594941, + "tokens_seen": 2063204352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125252525252523, + "loss": 2.8111, + "theoretical_loss": 3.421524269787252, + "tokens_seen": 2063269888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123232323232324, + "loss": 2.5254, + "theoretical_loss": 3.4215152639912487, + "tokens_seen": 2063335424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121212121212125, + "loss": 2.8271, + "theoretical_loss": 3.421506258561374, + "tokens_seen": 2063400960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811919191919192, + "loss": 2.6786, + "theoretical_loss": 3.421497253497601, + "tokens_seen": 2063466496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117171717171717, + "loss": 2.5401, + "theoretical_loss": 3.4214882487999034, + "tokens_seen": 2063532032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811515151515151, + "loss": 2.4325, + "theoretical_loss": 3.4214792444682542, + "tokens_seen": 2063597568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113131313131314, + "loss": 2.7128, + "theoretical_loss": 3.421470240502628, + "tokens_seen": 2063663104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038111111111111115, + "loss": 2.8178, + "theoretical_loss": 3.4214612369029966, + "tokens_seen": 2063728640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810909090909091, + "loss": 2.5088, + "theoretical_loss": 3.421452233669335, + "tokens_seen": 2063794176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107070707070706, + "loss": 2.6171, + "theoretical_loss": 3.421443230801616, + "tokens_seen": 2063859712 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381050505050505, + "loss": 2.6763, + "theoretical_loss": 3.421434228299814, + "tokens_seen": 2063925248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810303030303031, + "loss": 2.5894, + "theoretical_loss": 3.421425226163901, + "tokens_seen": 2063990784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038101010101010104, + "loss": 2.583, + "theoretical_loss": 3.4214162243938517, + "tokens_seen": 2064056320 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380989898989899, + "loss": 2.5012, + "theoretical_loss": 3.4214072229896395, + "tokens_seen": 2064121856 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038096969696969695, + "loss": 2.4929, + "theoretical_loss": 3.4213982219512373, + "tokens_seen": 2064187392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809494949494949, + "loss": 2.3919, + "theoretical_loss": 3.421389221278619, + "tokens_seen": 2064252928 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380929292929293, + "loss": 2.7227, + "theoretical_loss": 3.4213802209717583, + "tokens_seen": 2064318464 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038090909090909093, + "loss": 2.5093, + "theoretical_loss": 3.4213712210306286, + "tokens_seen": 2064384000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808888888888889, + "loss": 2.5109, + "theoretical_loss": 3.421362221455203, + "tokens_seen": 2064449536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086868686868685, + "loss": 2.6902, + "theoretical_loss": 3.4213532222454557, + "tokens_seen": 2064515072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038084848484848486, + "loss": 2.547, + "theoretical_loss": 3.42134422340136, + "tokens_seen": 2064580608 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1168450, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9453647136688232, + "objective/train/theoretical_loss": 3.421335224922889, + "objective/train/tokens_used": 423505376, + "theoretical_loss": 3.421335224922889, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038082828282828287, + "loss": 2.8839, + "theoretical_loss": 3.421335224922889, + "tokens_seen": 2064646144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808080808080808, + "loss": 2.6142, + "theoretical_loss": 3.4213262268100166, + "tokens_seen": 2064711680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807878787878788, + "loss": 2.6974, + "theoretical_loss": 3.4213172290627165, + "tokens_seen": 2064777216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038076767676767674, + "loss": 2.7995, + "theoretical_loss": 3.421308231680962, + "tokens_seen": 2064842752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074747474747475, + "loss": 2.8946, + "theoretical_loss": 3.421299234664727, + "tokens_seen": 2064908288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038072727272727276, + "loss": 2.7596, + "theoretical_loss": 3.421290238013985, + "tokens_seen": 2064973824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807070707070707, + "loss": 2.7274, + "theoretical_loss": 3.4212812417287086, + "tokens_seen": 2065039360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806868686868687, + "loss": 2.5134, + "theoretical_loss": 3.421272245808873, + "tokens_seen": 2065104896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806666666666667, + "loss": 2.6408, + "theoretical_loss": 3.42126325025445, + "tokens_seen": 2065170432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038064646464646464, + "loss": 2.7515, + "theoretical_loss": 3.4212542550654144, + "tokens_seen": 2065235968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038062626262626265, + "loss": 2.6638, + "theoretical_loss": 3.4212452602417396, + "tokens_seen": 2065301504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806060606060606, + "loss": 2.6765, + "theoretical_loss": 3.421236265783399, + "tokens_seen": 2065367040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058585858585857, + "loss": 2.6859, + "theoretical_loss": 3.4212272716903653, + "tokens_seen": 2065432576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805656565656566, + "loss": 2.539, + "theoretical_loss": 3.421218277962614, + "tokens_seen": 2065498112 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038054545454545454, + "loss": 2.5618, + "theoretical_loss": 3.421209284600117, + "tokens_seen": 2065563648 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038052525252525255, + "loss": 2.5601, + "theoretical_loss": 3.4212002916028483, + "tokens_seen": 2065629184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805050505050505, + "loss": 2.6156, + "theoretical_loss": 3.421191298970782, + "tokens_seen": 2065694720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804848484848485, + "loss": 2.7302, + "theoretical_loss": 3.4211823067038907, + "tokens_seen": 2065760256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038046464646464647, + "loss": 2.599, + "theoretical_loss": 3.4211733148021493, + "tokens_seen": 2065825792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038044444444444443, + "loss": 2.626, + "theoretical_loss": 3.42116432326553, + "tokens_seen": 2065891328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042424242424244, + "loss": 2.6711, + "theoretical_loss": 3.421155332094008, + "tokens_seen": 2065956864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804040404040404, + "loss": 2.7242, + "theoretical_loss": 3.4211463412875553, + "tokens_seen": 2066022400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803838383838384, + "loss": 2.5766, + "theoretical_loss": 3.4211373508461462, + "tokens_seen": 2066087936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038036363636363636, + "loss": 2.8162, + "theoretical_loss": 3.4211283607697545, + "tokens_seen": 2066153472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803434343434343, + "loss": 2.3838, + "theoretical_loss": 3.421119371058354, + "tokens_seen": 2066219008 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1169491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6120872497558594, + "objective/train/theoretical_loss": 3.421110381711917, + "objective/train/tokens_used": 425143776, + "theoretical_loss": 3.421110381711917, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038032323232323233, + "loss": 2.6659, + "theoretical_loss": 3.421110381711917, + "tokens_seen": 2066284544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030303030303034, + "loss": 2.5441, + "theoretical_loss": 3.421101392730418, + "tokens_seen": 2066350080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802828282828283, + "loss": 2.6192, + "theoretical_loss": 3.421092404113831, + "tokens_seen": 2066415616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026262626262626, + "loss": 2.6693, + "theoretical_loss": 3.421083415862129, + "tokens_seen": 2066481152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802424242424242, + "loss": 2.8623, + "theoretical_loss": 3.4210744279752863, + "tokens_seen": 2066546688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802222222222222, + "loss": 2.4839, + "theoretical_loss": 3.421065440453276, + "tokens_seen": 2066612224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038020202020202024, + "loss": 2.7548, + "theoretical_loss": 3.421056453296071, + "tokens_seen": 2066677760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801818181818182, + "loss": 2.5181, + "theoretical_loss": 3.4210474665036466, + "tokens_seen": 2066743296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038016161616161615, + "loss": 2.79, + "theoretical_loss": 3.421038480075975, + "tokens_seen": 2066808832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038014141414141416, + "loss": 2.7114, + "theoretical_loss": 3.421029494013031, + "tokens_seen": 2066874368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012121212121217, + "loss": 2.6253, + "theoretical_loss": 3.421020508314787, + "tokens_seen": 2066939904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010101010101013, + "loss": 2.5909, + "theoretical_loss": 3.421011522981217, + "tokens_seen": 2067005440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800808080808081, + "loss": 2.7594, + "theoretical_loss": 3.4210025380122953, + "tokens_seen": 2067070976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038006060606060604, + "loss": 2.6355, + "theoretical_loss": 3.420993553407995, + "tokens_seen": 2067136512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004040404040405, + "loss": 2.7577, + "theoretical_loss": 3.42098456916829, + "tokens_seen": 2067202048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002020202020206, + "loss": 2.7671, + "theoretical_loss": 3.4209755852931534, + "tokens_seen": 2067267584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 2.7799, + "theoretical_loss": 3.4209666017825597, + "tokens_seen": 2067333120 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379979797979798, + "loss": 2.6595, + "theoretical_loss": 3.4209576186364816, + "tokens_seen": 2067398656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995959595959593, + "loss": 2.4569, + "theoretical_loss": 3.4209486358548933, + "tokens_seen": 2067464192 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379939393939394, + "loss": 2.3887, + "theoretical_loss": 3.420939653437769, + "tokens_seen": 2067529728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037991919191919196, + "loss": 2.6406, + "theoretical_loss": 3.420930671385081, + "tokens_seen": 2067595264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798989898989899, + "loss": 2.6678, + "theoretical_loss": 3.4209216896968044, + "tokens_seen": 2067660800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987878787878787, + "loss": 2.7084, + "theoretical_loss": 3.420912708372912, + "tokens_seen": 2067726336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037985858585858583, + "loss": 2.6974, + "theoretical_loss": 3.4209037274133776, + "tokens_seen": 2067791872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798383838383839, + "loss": 2.8875, + "theoretical_loss": 3.420894746818175, + "tokens_seen": 2067857408 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1170265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3831610679626465, + "objective/train/theoretical_loss": 3.4208857665872774, + "objective/train/tokens_used": 426782176, + "theoretical_loss": 3.4208857665872774, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037981818181818185, + "loss": 2.7136, + "theoretical_loss": 3.4208857665872774, + "tokens_seen": 2067922944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797979797979798, + "loss": 2.6824, + "theoretical_loss": 3.4208767867206595, + "tokens_seen": 2067988480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037977777777777776, + "loss": 2.6849, + "theoretical_loss": 3.4208678072182943, + "tokens_seen": 2068054016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797575757575757, + "loss": 2.7496, + "theoretical_loss": 3.4208588280801555, + "tokens_seen": 2068119552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797373737373738, + "loss": 2.613, + "theoretical_loss": 3.420849849306216, + "tokens_seen": 2068185088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971717171717174, + "loss": 2.5932, + "theoretical_loss": 3.4208408708964515, + "tokens_seen": 2068250624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796969696969697, + "loss": 2.8211, + "theoretical_loss": 3.420831892850834, + "tokens_seen": 2068316160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037967676767676766, + "loss": 2.7434, + "theoretical_loss": 3.420822915169338, + "tokens_seen": 2068381696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037965656565656567, + "loss": 2.6198, + "theoretical_loss": 3.4208139378519364, + "tokens_seen": 2068447232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796363636363637, + "loss": 2.5398, + "theoretical_loss": 3.4208049608986038, + "tokens_seen": 2068512768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037961616161616163, + "loss": 2.578, + "theoretical_loss": 3.420795984309313, + "tokens_seen": 2068578304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795959595959596, + "loss": 2.6387, + "theoretical_loss": 3.420787008084039, + "tokens_seen": 2068643840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037957575757575755, + "loss": 3.0227, + "theoretical_loss": 3.4207780322227537, + "tokens_seen": 2068709376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955555555555556, + "loss": 2.639, + "theoretical_loss": 3.4207690567254327, + "tokens_seen": 2068774912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037953535353535357, + "loss": 2.9345, + "theoretical_loss": 3.4207600815920487, + "tokens_seen": 2068840448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037951515151515153, + "loss": 2.6042, + "theoretical_loss": 3.4207511068225753, + "tokens_seen": 2068905984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794949494949495, + "loss": 2.6819, + "theoretical_loss": 3.4207421324169864, + "tokens_seen": 2068971520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794747474747475, + "loss": 2.6466, + "theoretical_loss": 3.4207331583752563, + "tokens_seen": 2069037056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037945454545454545, + "loss": 2.6673, + "theoretical_loss": 3.4207241846973577, + "tokens_seen": 2069102592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037943434343434346, + "loss": 2.897, + "theoretical_loss": 3.420715211383265, + "tokens_seen": 2069168128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794141414141414, + "loss": 2.9113, + "theoretical_loss": 3.4207062384329516, + "tokens_seen": 2069233664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793939393939394, + "loss": 2.608, + "theoretical_loss": 3.4206972658463917, + "tokens_seen": 2069299200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793737373737374, + "loss": 2.7201, + "theoretical_loss": 3.420688293623559, + "tokens_seen": 2069364736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037935353535353535, + "loss": 2.5112, + "theoretical_loss": 3.4206793217644265, + "tokens_seen": 2069430272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933333333333336, + "loss": 2.6224, + "theoretical_loss": 3.420670350268969, + "tokens_seen": 2069495808 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1170942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7669098377227783, + "objective/train/theoretical_loss": 3.420661379137159, + "objective/train/tokens_used": 428420576, + "theoretical_loss": 3.420661379137159, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793131313131313, + "loss": 2.6339, + "theoretical_loss": 3.420661379137159, + "tokens_seen": 2069561344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792929292929293, + "loss": 2.6438, + "theoretical_loss": 3.420652408368971, + "tokens_seen": 2069626880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792727272727273, + "loss": 2.5309, + "theoretical_loss": 3.4206434379643786, + "tokens_seen": 2069692416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925252525252524, + "loss": 2.6332, + "theoretical_loss": 3.420634467923356, + "tokens_seen": 2069757952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037923232323232325, + "loss": 2.5236, + "theoretical_loss": 3.420625498245876, + "tokens_seen": 2069823488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792121212121212, + "loss": 2.5826, + "theoretical_loss": 3.4206165289319133, + "tokens_seen": 2069889024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791919191919192, + "loss": 2.8086, + "theoretical_loss": 3.420607559981441, + "tokens_seen": 2069954560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791717171717172, + "loss": 2.7336, + "theoretical_loss": 3.420598591394433, + "tokens_seen": 2070020096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037915151515151513, + "loss": 2.6061, + "theoretical_loss": 3.420589623170864, + "tokens_seen": 2070085632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037913131313131314, + "loss": 2.6435, + "theoretical_loss": 3.4205806553107063, + "tokens_seen": 2070151168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911111111111115, + "loss": 2.4987, + "theoretical_loss": 3.4205716878139345, + "tokens_seen": 2070216704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790909090909091, + "loss": 2.5315, + "theoretical_loss": 3.420562720680522, + "tokens_seen": 2070282240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037907070707070707, + "loss": 2.8474, + "theoretical_loss": 3.420553753910443, + "tokens_seen": 2070347776 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379050505050505, + "loss": 2.7089, + "theoretical_loss": 3.4205447875036707, + "tokens_seen": 2070413312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903030303030303, + "loss": 2.5325, + "theoretical_loss": 3.42053582146018, + "tokens_seen": 2070478848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037901010101010105, + "loss": 2.6585, + "theoretical_loss": 3.4205268557799435, + "tokens_seen": 2070544384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378989898989899, + "loss": 2.9106, + "theoretical_loss": 3.420517890462935, + "tokens_seen": 2070609920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037896969696969696, + "loss": 2.5265, + "theoretical_loss": 3.420508925509129, + "tokens_seen": 2070675456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789494949494949, + "loss": 2.7775, + "theoretical_loss": 3.420499960918499, + "tokens_seen": 2070740992 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378929292929293, + "loss": 2.5403, + "theoretical_loss": 3.420490996691019, + "tokens_seen": 2070806528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890909090909094, + "loss": 2.7932, + "theoretical_loss": 3.420482032826662, + "tokens_seen": 2070872064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788888888888889, + "loss": 2.67, + "theoretical_loss": 3.4204730693254035, + "tokens_seen": 2070937600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037886868686868685, + "loss": 2.5489, + "theoretical_loss": 3.4204641061872154, + "tokens_seen": 2071003136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788484848484848, + "loss": 2.5774, + "theoretical_loss": 3.420455143412072, + "tokens_seen": 2071068672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788282828282829, + "loss": 2.5615, + "theoretical_loss": 3.4204461809999476, + "tokens_seen": 2071134208 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1172086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8357508182525635, + "objective/train/theoretical_loss": 3.4204372189508163, + "objective/train/tokens_used": 430058976, + "theoretical_loss": 3.4204372189508163, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880808080808083, + "loss": 2.717, + "theoretical_loss": 3.4204372189508163, + "tokens_seen": 2071199744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787878787878788, + "loss": 2.7323, + "theoretical_loss": 3.4204282572646507, + "tokens_seen": 2071265280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876767676767674, + "loss": 2.5918, + "theoretical_loss": 3.420419295941426, + "tokens_seen": 2071330816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037874747474747476, + "loss": 2.7078, + "theoretical_loss": 3.4204103349811152, + "tokens_seen": 2071396352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872727272727277, + "loss": 2.8419, + "theoretical_loss": 3.420401374383692, + "tokens_seen": 2071461888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787070707070707, + "loss": 2.5696, + "theoretical_loss": 3.4203924141491306, + "tokens_seen": 2071527424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786868686868687, + "loss": 2.7442, + "theoretical_loss": 3.4203834542774043, + "tokens_seen": 2071592960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037866666666666664, + "loss": 2.6127, + "theoretical_loss": 3.420374494768488, + "tokens_seen": 2071658496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037864646464646465, + "loss": 2.7584, + "theoretical_loss": 3.420365535622355, + "tokens_seen": 2071724032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037862626262626266, + "loss": 2.5562, + "theoretical_loss": 3.4203565768389783, + "tokens_seen": 2071789568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786060606060606, + "loss": 2.6739, + "theoretical_loss": 3.420347618418333, + "tokens_seen": 2071855104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785858585858586, + "loss": 2.6363, + "theoretical_loss": 3.4203386603603922, + "tokens_seen": 2071920640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656565656566, + "loss": 2.6888, + "theoretical_loss": 3.4203297026651303, + "tokens_seen": 2071986176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037854545454545454, + "loss": 2.8503, + "theoretical_loss": 3.4203207453325204, + "tokens_seen": 2072051712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852525252525255, + "loss": 2.4433, + "theoretical_loss": 3.4203117883625365, + "tokens_seen": 2072117248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785050505050505, + "loss": 2.7281, + "theoretical_loss": 3.420302831755153, + "tokens_seen": 2072182784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037848484848484847, + "loss": 2.7352, + "theoretical_loss": 3.4202938755103434, + "tokens_seen": 2072248320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784646464646465, + "loss": 2.7578, + "theoretical_loss": 3.420284919628082, + "tokens_seen": 2072313856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784444444444445, + "loss": 2.7229, + "theoretical_loss": 3.4202759641083413, + "tokens_seen": 2072379392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842424242424244, + "loss": 2.5451, + "theoretical_loss": 3.4202670089510967, + "tokens_seen": 2072444928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784040404040404, + "loss": 2.6291, + "theoretical_loss": 3.4202580541563217, + "tokens_seen": 2072510464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783838383838384, + "loss": 2.5909, + "theoretical_loss": 3.4202490997239896, + "tokens_seen": 2072576000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836363636363637, + "loss": 2.6526, + "theoretical_loss": 3.4202401456540747, + "tokens_seen": 2072641536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783434343434344, + "loss": 2.6628, + "theoretical_loss": 3.420231191946551, + "tokens_seen": 2072707072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832323232323234, + "loss": 2.8017, + "theoretical_loss": 3.420222238601392, + "tokens_seen": 2072772608 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1172787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9420156478881836, + "objective/train/theoretical_loss": 3.420213285618572, + "objective/train/tokens_used": 431697376, + "theoretical_loss": 3.420213285618572, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783030303030303, + "loss": 2.5389, + "theoretical_loss": 3.420213285618572, + "tokens_seen": 2072838144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782828282828283, + "loss": 2.714, + "theoretical_loss": 3.420204332998064, + "tokens_seen": 2072903680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037826262626262626, + "loss": 2.8283, + "theoretical_loss": 3.420195380739843, + "tokens_seen": 2072969216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782424242424243, + "loss": 2.4924, + "theoretical_loss": 3.4201864288438824, + "tokens_seen": 2073034752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037822222222222223, + "loss": 2.581, + "theoretical_loss": 3.4201774773101556, + "tokens_seen": 2073100288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782020202020202, + "loss": 2.6137, + "theoretical_loss": 3.4201685261386374, + "tokens_seen": 2073165824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781818181818182, + "loss": 2.4754, + "theoretical_loss": 3.4201595753293015, + "tokens_seen": 2073231360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816161616161615, + "loss": 2.8785, + "theoretical_loss": 3.420150624882121, + "tokens_seen": 2073296896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037814141414141417, + "loss": 2.7317, + "theoretical_loss": 3.4201416747970708, + "tokens_seen": 2073362432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781212121212121, + "loss": 2.593, + "theoretical_loss": 3.4201327250741245, + "tokens_seen": 2073427968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037810101010101013, + "loss": 2.5435, + "theoretical_loss": 3.4201237757132557, + "tokens_seen": 2073493504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780808080808081, + "loss": 2.5464, + "theoretical_loss": 3.420114826714438, + "tokens_seen": 2073559040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037806060606060605, + "loss": 2.7212, + "theoretical_loss": 3.4201058780776465, + "tokens_seen": 2073624576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037804040404040406, + "loss": 2.7739, + "theoretical_loss": 3.4200969298028543, + "tokens_seen": 2073690112 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378020202020202, + "loss": 2.657, + "theoretical_loss": 3.4200879818900347, + "tokens_seen": 2073755648 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378, + "loss": 2.8451, + "theoretical_loss": 3.420079034339163, + "tokens_seen": 2073821184 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377979797979798, + "loss": 2.6633, + "theoretical_loss": 3.4200700871502123, + "tokens_seen": 2073886720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795959595959594, + "loss": 2.8986, + "theoretical_loss": 3.420061140323157, + "tokens_seen": 2073952256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793939393939395, + "loss": 2.5848, + "theoretical_loss": 3.4200521938579707, + "tokens_seen": 2074017792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037791919191919196, + "loss": 2.6853, + "theoretical_loss": 3.420043247754627, + "tokens_seen": 2074083328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778989898989899, + "loss": 2.5792, + "theoretical_loss": 3.4200343020131, + "tokens_seen": 2074148864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778787878787879, + "loss": 2.6661, + "theoretical_loss": 3.4200253566333645, + "tokens_seen": 2074214400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785858585858583, + "loss": 2.7249, + "theoretical_loss": 3.4200164116153937, + "tokens_seen": 2074279936 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037783838383838384, + "loss": 2.6617, + "theoretical_loss": 3.420007466959161, + "tokens_seen": 2074345472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781818181818186, + "loss": 2.6679, + "theoretical_loss": 3.419998522664641, + "tokens_seen": 2074411008 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1174027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.937673568725586, + "objective/train/theoretical_loss": 3.4199895787318084, + "objective/train/tokens_used": 433335776, + "theoretical_loss": 3.4199895787318084, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777979797979798, + "loss": 2.8638, + "theoretical_loss": 3.4199895787318084, + "tokens_seen": 2074476544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777777777777777, + "loss": 2.6573, + "theoretical_loss": 3.4199806351606354, + "tokens_seen": 2074542080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777575757575757, + "loss": 2.5862, + "theoretical_loss": 3.4199716919510976, + "tokens_seen": 2074607616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777373737373738, + "loss": 2.64, + "theoretical_loss": 3.4199627491031674, + "tokens_seen": 2074673152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037771717171717175, + "loss": 2.7563, + "theoretical_loss": 3.41995380661682, + "tokens_seen": 2074738688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776969696969697, + "loss": 2.8674, + "theoretical_loss": 3.419944864492029, + "tokens_seen": 2074804224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037767676767676766, + "loss": 2.3904, + "theoretical_loss": 3.4199359227287687, + "tokens_seen": 2074869760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776565656565656, + "loss": 2.609, + "theoretical_loss": 3.4199269813270123, + "tokens_seen": 2074935296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776363636363637, + "loss": 2.7758, + "theoretical_loss": 3.419918040286734, + "tokens_seen": 2075000832 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037761616161616164, + "loss": 2.985, + "theoretical_loss": 3.4199090996079082, + "tokens_seen": 2075066368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775959595959596, + "loss": 2.667, + "theoretical_loss": 3.419900159290509, + "tokens_seen": 2075131904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037757575757575755, + "loss": 2.7865, + "theoretical_loss": 3.419891219334509, + "tokens_seen": 2075197440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037755555555555557, + "loss": 2.7346, + "theoretical_loss": 3.419882279739884, + "tokens_seen": 2075262976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775353535353536, + "loss": 2.7398, + "theoretical_loss": 3.4198733405066064, + "tokens_seen": 2075328512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751515151515153, + "loss": 2.6861, + "theoretical_loss": 3.4198644016346513, + "tokens_seen": 2075394048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774949494949495, + "loss": 2.6403, + "theoretical_loss": 3.4198554631239926, + "tokens_seen": 2075459584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037747474747474745, + "loss": 2.5859, + "theoretical_loss": 3.4198465249746035, + "tokens_seen": 2075525120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745454545454546, + "loss": 2.606, + "theoretical_loss": 3.419837587186459, + "tokens_seen": 2075590656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743434343434347, + "loss": 2.6021, + "theoretical_loss": 3.419828649759532, + "tokens_seen": 2075656192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774141414141414, + "loss": 2.8785, + "theoretical_loss": 3.419819712693797, + "tokens_seen": 2075721728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773939393939394, + "loss": 2.6933, + "theoretical_loss": 3.4198107759892284, + "tokens_seen": 2075787264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773737373737374, + "loss": 2.7137, + "theoretical_loss": 3.4198018396457996, + "tokens_seen": 2075852800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037735353535353535, + "loss": 2.791, + "theoretical_loss": 3.4197929036634855, + "tokens_seen": 2075918336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733333333333336, + "loss": 2.7527, + "theoretical_loss": 3.4197839680422586, + "tokens_seen": 2075983872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773131313131313, + "loss": 2.8114, + "theoretical_loss": 3.419775032782095, + "tokens_seen": 2076049408 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1174812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8366079330444336, + "objective/train/theoretical_loss": 3.4197660978829667, + "objective/train/tokens_used": 434974176, + "theoretical_loss": 3.4197660978829667, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772929292929293, + "loss": 2.7274, + "theoretical_loss": 3.4197660978829667, + "tokens_seen": 2076114944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772727272727273, + "loss": 2.5463, + "theoretical_loss": 3.4197571633448485, + "tokens_seen": 2076180480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725252525252524, + "loss": 2.7296, + "theoretical_loss": 3.4197482291677144, + "tokens_seen": 2076246016 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037723232323232325, + "loss": 2.8419, + "theoretical_loss": 3.4197392953515386, + "tokens_seen": 2076311552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772121212121212, + "loss": 2.6465, + "theoretical_loss": 3.419730361896295, + "tokens_seen": 2076377088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771919191919192, + "loss": 2.7838, + "theoretical_loss": 3.4197214288019575, + "tokens_seen": 2076442624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771717171717172, + "loss": 2.829, + "theoretical_loss": 3.4197124960685006, + "tokens_seen": 2076508160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037715151515151514, + "loss": 2.8514, + "theoretical_loss": 3.419703563695898, + "tokens_seen": 2076573696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713131313131315, + "loss": 2.5954, + "theoretical_loss": 3.4196946316841235, + "tokens_seen": 2076639232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771111111111111, + "loss": 2.6328, + "theoretical_loss": 3.419685700033151, + "tokens_seen": 2076704768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770909090909091, + "loss": 2.6216, + "theoretical_loss": 3.4196767687429555, + "tokens_seen": 2076770304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037707070707070707, + "loss": 2.6327, + "theoretical_loss": 3.41966783781351, + "tokens_seen": 2076835840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705050505050503, + "loss": 2.8442, + "theoretical_loss": 3.4196589072447896, + "tokens_seen": 2076901376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703030303030304, + "loss": 2.7962, + "theoretical_loss": 3.4196499770367668, + "tokens_seen": 2076966912 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037701010101010105, + "loss": 2.6707, + "theoretical_loss": 3.419641047189417, + "tokens_seen": 2077032448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376989898989899, + "loss": 2.578, + "theoretical_loss": 3.4196321177027142, + "tokens_seen": 2077097984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037696969696969696, + "loss": 2.6864, + "theoretical_loss": 3.4196231885766317, + "tokens_seen": 2077163520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769494949494949, + "loss": 2.846, + "theoretical_loss": 3.4196142598111443, + "tokens_seen": 2077229056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037692929292929293, + "loss": 2.8173, + "theoretical_loss": 3.4196053314062254, + "tokens_seen": 2077294592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690909090909094, + "loss": 2.5335, + "theoretical_loss": 3.419596403361849, + "tokens_seen": 2077360128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768888888888889, + "loss": 2.6046, + "theoretical_loss": 3.4195874756779903, + "tokens_seen": 2077425664 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037686868686868686, + "loss": 2.5354, + "theoretical_loss": 3.4195785483546226, + "tokens_seen": 2077491200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768484848484848, + "loss": 2.7129, + "theoretical_loss": 3.4195696213917195, + "tokens_seen": 2077556736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768282828282829, + "loss": 2.6738, + "theoretical_loss": 3.419560694789256, + "tokens_seen": 2077622272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680808080808084, + "loss": 2.6226, + "theoretical_loss": 3.419551768547205, + "tokens_seen": 2077687808 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1175928, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.206481456756592, + "objective/train/theoretical_loss": 3.419542842665542, + "objective/train/tokens_used": 436612576, + "theoretical_loss": 3.419542842665542, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767878787878788, + "loss": 2.5652, + "theoretical_loss": 3.419542842665542, + "tokens_seen": 2077753344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676767676767675, + "loss": 2.7346, + "theoretical_loss": 3.41953391714424, + "tokens_seen": 2077818880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767474747474747, + "loss": 2.7595, + "theoretical_loss": 3.4195249919832738, + "tokens_seen": 2077884416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672727272727277, + "loss": 2.6788, + "theoretical_loss": 3.419516067182617, + "tokens_seen": 2077949952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037670707070707073, + "loss": 2.7646, + "theoretical_loss": 3.419507142742244, + "tokens_seen": 2078015488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766868686868687, + "loss": 2.5201, + "theoretical_loss": 3.4194982186621288, + "tokens_seen": 2078081024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037666666666666664, + "loss": 2.7998, + "theoretical_loss": 3.4194892949422453, + "tokens_seen": 2078146560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766464646464647, + "loss": 2.5621, + "theoretical_loss": 3.419480371582568, + "tokens_seen": 2078212096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662626262626266, + "loss": 2.7562, + "theoretical_loss": 3.419471448583071, + "tokens_seen": 2078277632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766060606060606, + "loss": 2.6984, + "theoretical_loss": 3.419462525943727, + "tokens_seen": 2078343168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765858585858586, + "loss": 2.7178, + "theoretical_loss": 3.419453603664512, + "tokens_seen": 2078408704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656565656565654, + "loss": 2.7322, + "theoretical_loss": 3.4194446817453996, + "tokens_seen": 2078474240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765454545454546, + "loss": 2.606, + "theoretical_loss": 3.4194357601863636, + "tokens_seen": 2078539776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037652525252525256, + "loss": 2.7598, + "theoretical_loss": 3.419426838987378, + "tokens_seen": 2078605312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765050505050505, + "loss": 2.9256, + "theoretical_loss": 3.419417918148417, + "tokens_seen": 2078670848 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648484848484847, + "loss": 2.6658, + "theoretical_loss": 3.4194089976694553, + "tokens_seen": 2078736384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037646464646464643, + "loss": 2.7665, + "theoretical_loss": 3.419400077550466, + "tokens_seen": 2078801920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764444444444445, + "loss": 2.7173, + "theoretical_loss": 3.4193911577914244, + "tokens_seen": 2078867456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037642424242424245, + "loss": 2.6502, + "theoretical_loss": 3.419382238392304, + "tokens_seen": 2078932992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764040404040404, + "loss": 2.8378, + "theoretical_loss": 3.4193733193530784, + "tokens_seen": 2078998528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037638383838383836, + "loss": 2.7648, + "theoretical_loss": 3.419364400673723, + "tokens_seen": 2079064064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763636363636364, + "loss": 2.7014, + "theoretical_loss": 3.4193554823542107, + "tokens_seen": 2079129600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763434343434344, + "loss": 2.8011, + "theoretical_loss": 3.419346564394516, + "tokens_seen": 2079195136 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632323232323234, + "loss": 2.5817, + "theoretical_loss": 3.419337646794614, + "tokens_seen": 2079260672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763030303030303, + "loss": 2.5441, + "theoretical_loss": 3.4193287295544774, + "tokens_seen": 2079326208 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 1176572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.376237154006958, + "objective/train/theoretical_loss": 3.419319812674081, + "objective/train/tokens_used": 438250976, + "theoretical_loss": 3.419319812674081, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037628282828282826, + "loss": 2.7067, + "theoretical_loss": 3.419319812674081, + "tokens_seen": 2079391744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037626262626262627, + "loss": 2.6409, + "theoretical_loss": 3.4193108961533993, + "tokens_seen": 2079457280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762424242424243, + "loss": 2.9311, + "theoretical_loss": 3.4193019799924063, + "tokens_seen": 2079522816 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037622222222222224, + "loss": 2.6408, + "theoretical_loss": 3.4192930641910753, + "tokens_seen": 2079588352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762020202020202, + "loss": 2.5212, + "theoretical_loss": 3.4192841487493815, + "tokens_seen": 2079653888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761818181818182, + "loss": 2.6842, + "theoretical_loss": 3.4192752336672987, + "tokens_seen": 2079719424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616161616161616, + "loss": 2.6743, + "theoretical_loss": 3.4192663189448007, + "tokens_seen": 2079784960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037614141414141417, + "loss": 2.7662, + "theoretical_loss": 3.4192574045818622, + "tokens_seen": 2079850496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612121212121213, + "loss": 2.7729, + "theoretical_loss": 3.4192484905784575, + "tokens_seen": 2079916032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761010101010101, + "loss": 2.7348, + "theoretical_loss": 3.41923957693456, + "tokens_seen": 2079981568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760808080808081, + "loss": 2.6199, + "theoretical_loss": 3.4192306636501444, + "tokens_seen": 2080047104 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606060606060605, + "loss": 2.6684, + "theoretical_loss": 3.4192217507251854, + "tokens_seen": 2080112640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604040404040406, + "loss": 2.8508, + "theoretical_loss": 3.419212838159656, + "tokens_seen": 2080178176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376020202020202, + "loss": 2.6601, + "theoretical_loss": 3.4192039259535307, + "tokens_seen": 2080243712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600000000000003, + "loss": 2.6352, + "theoretical_loss": 3.4191950141067844, + "tokens_seen": 2080309248 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375979797979798, + "loss": 2.652, + "theoretical_loss": 3.419186102619391, + "tokens_seen": 2080374784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037595959595959595, + "loss": 2.7063, + "theoretical_loss": 3.419177191491324, + "tokens_seen": 2080440320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037593939393939396, + "loss": 2.8259, + "theoretical_loss": 3.4191682807225585, + "tokens_seen": 2080505856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759191919191919, + "loss": 2.6831, + "theoretical_loss": 3.419159370313068, + "tokens_seen": 2080571392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758989898989899, + "loss": 2.6214, + "theoretical_loss": 3.419150460262827, + "tokens_seen": 2080636928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758787878787879, + "loss": 2.6691, + "theoretical_loss": 3.41914155057181, + "tokens_seen": 2080702464 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037585858585858584, + "loss": 2.7933, + "theoretical_loss": 3.419132641239991, + "tokens_seen": 2080768000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583838383838385, + "loss": 2.8242, + "theoretical_loss": 3.419123732267344, + "tokens_seen": 2080833536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037581818181818186, + "loss": 2.4951, + "theoretical_loss": 3.4191148236538433, + "tokens_seen": 2080899072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757979797979798, + "loss": 2.5211, + "theoretical_loss": 3.419105915399463, + "tokens_seen": 2080964608 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1177438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.871868133544922, + "objective/train/theoretical_loss": 3.4190970075041776, + "objective/train/tokens_used": 439889376, + "theoretical_loss": 3.4190970075041776, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757777777777778, + "loss": 2.9536, + "theoretical_loss": 3.4190970075041776, + "tokens_seen": 2081030144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575757575757573, + "loss": 2.6978, + "theoretical_loss": 3.4190880999679614, + "tokens_seen": 2081095680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037573737373737374, + "loss": 2.6435, + "theoretical_loss": 3.4190791927907878, + "tokens_seen": 2081161216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571717171717175, + "loss": 2.7349, + "theoretical_loss": 3.419070285972632, + "tokens_seen": 2081226752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756969696969697, + "loss": 2.8035, + "theoretical_loss": 3.419061379513468, + "tokens_seen": 2081292288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567676767676767, + "loss": 2.5141, + "theoretical_loss": 3.4190524734132692, + "tokens_seen": 2081357824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756565656565656, + "loss": 2.8604, + "theoretical_loss": 3.419043567672011, + "tokens_seen": 2081423360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756363636363637, + "loss": 2.4979, + "theoretical_loss": 3.4190346622896675, + "tokens_seen": 2081488896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561616161616165, + "loss": 2.8975, + "theoretical_loss": 3.419025757266212, + "tokens_seen": 2081554432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755959595959596, + "loss": 2.7506, + "theoretical_loss": 3.4190168526016196, + "tokens_seen": 2081619968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037557575757575756, + "loss": 2.5028, + "theoretical_loss": 3.4190079482958637, + "tokens_seen": 2081685504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755555555555555, + "loss": 2.7788, + "theoretical_loss": 3.4189990443489195, + "tokens_seen": 2081751040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755353535353536, + "loss": 2.8914, + "theoretical_loss": 3.4189901407607612, + "tokens_seen": 2081816576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037551515151515154, + "loss": 2.7179, + "theoretical_loss": 3.418981237531362, + "tokens_seen": 2081882112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754949494949495, + "loss": 2.8172, + "theoretical_loss": 3.4189723346606975, + "tokens_seen": 2081947648 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037547474747474745, + "loss": 2.7673, + "theoretical_loss": 3.4189634321487405, + "tokens_seen": 2082013184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545454545454546, + "loss": 2.7692, + "theoretical_loss": 3.4189545299954665, + "tokens_seen": 2082078720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754343434343435, + "loss": 2.7918, + "theoretical_loss": 3.4189456282008495, + "tokens_seen": 2082144256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037541414141414143, + "loss": 2.6638, + "theoretical_loss": 3.4189367267648634, + "tokens_seen": 2082209792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753939393939394, + "loss": 2.806, + "theoretical_loss": 3.4189278256874824, + "tokens_seen": 2082275328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537373737373735, + "loss": 2.8645, + "theoretical_loss": 3.4189189249686813, + "tokens_seen": 2082340864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535353535353536, + "loss": 2.7044, + "theoretical_loss": 3.418910024608434, + "tokens_seen": 2082406400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037533333333333337, + "loss": 2.7592, + "theoretical_loss": 3.4189011246067147, + "tokens_seen": 2082471936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753131313131313, + "loss": 2.5858, + "theoretical_loss": 3.418892224963498, + "tokens_seen": 2082537472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752929292929293, + "loss": 2.7486, + "theoretical_loss": 3.418883325678758, + "tokens_seen": 2082603008 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1177725, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8855385780334473, + "objective/train/theoretical_loss": 3.4188744267524687, + "objective/train/tokens_used": 441527776, + "theoretical_loss": 3.4188744267524687, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752727272727273, + "loss": 2.7633, + "theoretical_loss": 3.4188744267524687, + "tokens_seen": 2082668544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037525252525252525, + "loss": 2.8146, + "theoretical_loss": 3.4188655281846048, + "tokens_seen": 2082734080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523232323232326, + "loss": 2.9182, + "theoretical_loss": 3.4188566299751404, + "tokens_seen": 2082799616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752121212121212, + "loss": 2.7575, + "theoretical_loss": 3.41884773212405, + "tokens_seen": 2082865152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751919191919192, + "loss": 2.7388, + "theoretical_loss": 3.418838834631307, + "tokens_seen": 2082930688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751717171717172, + "loss": 2.8627, + "theoretical_loss": 3.4188299374968874, + "tokens_seen": 2082996224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515151515151514, + "loss": 2.6815, + "theoretical_loss": 3.418821040720764, + "tokens_seen": 2083061760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037513131313131315, + "loss": 2.9676, + "theoretical_loss": 3.418812144302912, + "tokens_seen": 2083127296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751111111111111, + "loss": 2.7674, + "theoretical_loss": 3.418803248243305, + "tokens_seen": 2083192832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750909090909091, + "loss": 3.0655, + "theoretical_loss": 3.4187943525419175, + "tokens_seen": 2083258368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750707070707071, + "loss": 2.7596, + "theoretical_loss": 3.4187854571987244, + "tokens_seen": 2083323904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037505050505050503, + "loss": 2.7171, + "theoretical_loss": 3.418776562213699, + "tokens_seen": 2083389440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037503030303030305, + "loss": 2.8939, + "theoretical_loss": 3.418767667586817, + "tokens_seen": 2083454976 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375010101010101, + "loss": 2.6305, + "theoretical_loss": 3.418758773318051, + "tokens_seen": 2083520512 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374989898989899, + "loss": 2.754, + "theoretical_loss": 3.4187498794073763, + "tokens_seen": 2083586048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496969696969697, + "loss": 2.8511, + "theoretical_loss": 3.4187409858547673, + "tokens_seen": 2083651584 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374949494949495, + "loss": 2.8043, + "theoretical_loss": 3.418732092660198, + "tokens_seen": 2083717120 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037492929292929294, + "loss": 2.5928, + "theoretical_loss": 3.418723199823643, + "tokens_seen": 2083782656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749090909090909, + "loss": 3.0093, + "theoretical_loss": 3.418714307345076, + "tokens_seen": 2083848192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748888888888889, + "loss": 2.8279, + "theoretical_loss": 3.4187054152244727, + "tokens_seen": 2083913728 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037486868686868686, + "loss": 2.6373, + "theoretical_loss": 3.418696523461806, + "tokens_seen": 2083979264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748484848484849, + "loss": 2.755, + "theoretical_loss": 3.4186876320570505, + "tokens_seen": 2084044800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037482828282828283, + "loss": 2.9262, + "theoretical_loss": 3.4186787410101815, + "tokens_seen": 2084110336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480808080808084, + "loss": 2.6747, + "theoretical_loss": 3.418669850321172, + "tokens_seen": 2084175872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747878787878788, + "loss": 2.6181, + "theoretical_loss": 3.4186609599899977, + "tokens_seen": 2084241408 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1178778, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9331679344177246, + "objective/train/theoretical_loss": 3.418652070016632, + "objective/train/tokens_used": 443166176, + "theoretical_loss": 3.418652070016632, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476767676767676, + "loss": 2.9788, + "theoretical_loss": 3.418652070016632, + "tokens_seen": 2084306944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037474747474747477, + "loss": 2.8083, + "theoretical_loss": 3.418643180401049, + "tokens_seen": 2084372480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747272727272727, + "loss": 2.7362, + "theoretical_loss": 3.418634291143224, + "tokens_seen": 2084438016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037470707070707073, + "loss": 2.6497, + "theoretical_loss": 3.418625402243131, + "tokens_seen": 2084503552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746868686868687, + "loss": 2.8272, + "theoretical_loss": 3.4186165137007443, + "tokens_seen": 2084569088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466666666666665, + "loss": 2.8171, + "theoretical_loss": 3.418607625516038, + "tokens_seen": 2084634624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464646464646466, + "loss": 2.9007, + "theoretical_loss": 3.4185987376889866, + "tokens_seen": 2084700160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462626262626267, + "loss": 2.5485, + "theoretical_loss": 3.418589850219565, + "tokens_seen": 2084765696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460606060606063, + "loss": 2.7477, + "theoretical_loss": 3.418580963107746, + "tokens_seen": 2084831232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745858585858586, + "loss": 2.6727, + "theoretical_loss": 3.418572076353506, + "tokens_seen": 2084896768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037456565656565654, + "loss": 2.5061, + "theoretical_loss": 3.4185631899568185, + "tokens_seen": 2084962304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037454545454545455, + "loss": 2.758, + "theoretical_loss": 3.4185543039176576, + "tokens_seen": 2085027840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452525252525256, + "loss": 2.6805, + "theoretical_loss": 3.4185454182359982, + "tokens_seen": 2085093376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745050505050505, + "loss": 2.677, + "theoretical_loss": 3.418536532911814, + "tokens_seen": 2085158912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744848484848485, + "loss": 2.7077, + "theoretical_loss": 3.41852764794508, + "tokens_seen": 2085224448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037446464646464643, + "loss": 2.5764, + "theoretical_loss": 3.4185187633357703, + "tokens_seen": 2085289984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744444444444445, + "loss": 2.6945, + "theoretical_loss": 3.4185098790838593, + "tokens_seen": 2085355520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037442424242424246, + "loss": 2.801, + "theoretical_loss": 3.418500995189322, + "tokens_seen": 2085421056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744040404040404, + "loss": 2.953, + "theoretical_loss": 3.4184921116521316, + "tokens_seen": 2085486592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037438383838383837, + "loss": 2.5469, + "theoretical_loss": 3.4184832284722635, + "tokens_seen": 2085552128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743636363636363, + "loss": 2.7322, + "theoretical_loss": 3.4184743456496918, + "tokens_seen": 2085617664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743434343434344, + "loss": 2.8421, + "theoretical_loss": 3.4184654631843907, + "tokens_seen": 2085683200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432323232323235, + "loss": 2.7056, + "theoretical_loss": 3.4184565810763345, + "tokens_seen": 2085748736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743030303030303, + "loss": 2.7143, + "theoretical_loss": 3.4184476993254984, + "tokens_seen": 2085814272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428282828282826, + "loss": 2.7077, + "theoretical_loss": 3.4184388179318557, + "tokens_seen": 2085879808 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1179467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2576162815093994, + "objective/train/theoretical_loss": 3.4184299368953814, + "objective/train/tokens_used": 444804576, + "theoretical_loss": 3.4184299368953814, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742626262626263, + "loss": 2.4789, + "theoretical_loss": 3.4184299368953814, + "tokens_seen": 2085945344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742424242424243, + "loss": 2.8066, + "theoretical_loss": 3.4184210562160504, + "tokens_seen": 2086010880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422222222222224, + "loss": 2.5599, + "theoretical_loss": 3.4184121758938364, + "tokens_seen": 2086076416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742020202020202, + "loss": 2.8453, + "theoretical_loss": 3.418403295928714, + "tokens_seen": 2086141952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037418181818181815, + "loss": 2.6252, + "theoretical_loss": 3.4183944163206577, + "tokens_seen": 2086207488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037416161616161617, + "loss": 2.8197, + "theoretical_loss": 3.418385537069642, + "tokens_seen": 2086273024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741414141414142, + "loss": 2.8551, + "theoretical_loss": 3.4183766581756405, + "tokens_seen": 2086338560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412121212121213, + "loss": 2.7835, + "theoretical_loss": 3.418367779638629, + "tokens_seen": 2086404096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741010101010101, + "loss": 2.6808, + "theoretical_loss": 3.418358901458581, + "tokens_seen": 2086469632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740808080808081, + "loss": 2.6991, + "theoretical_loss": 3.4183500236354716, + "tokens_seen": 2086535168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406060606060606, + "loss": 2.8759, + "theoretical_loss": 3.4183411461692748, + "tokens_seen": 2086600704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037404040404040407, + "loss": 2.7418, + "theoretical_loss": 3.418332269059965, + "tokens_seen": 2086666240 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374020202020202, + "loss": 2.7262, + "theoretical_loss": 3.4183233923075163, + "tokens_seen": 2086731776 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374, + "loss": 2.7569, + "theoretical_loss": 3.418314515911904, + "tokens_seen": 2086797312 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373979797979798, + "loss": 2.6933, + "theoretical_loss": 3.418305639873102, + "tokens_seen": 2086862848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037395959595959595, + "loss": 2.6108, + "theoretical_loss": 3.4182967641910853, + "tokens_seen": 2086928384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393939393939396, + "loss": 2.6586, + "theoretical_loss": 3.4182878888658275, + "tokens_seen": 2086993920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739191919191919, + "loss": 2.8226, + "theoretical_loss": 3.418279013897304, + "tokens_seen": 2087059456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037389898989898993, + "loss": 2.6484, + "theoretical_loss": 3.418270139285488, + "tokens_seen": 2087124992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738787878787879, + "loss": 2.8433, + "theoretical_loss": 3.418261265030355, + "tokens_seen": 2087190528 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037385858585858584, + "loss": 2.7432, + "theoretical_loss": 3.4182523911318796, + "tokens_seen": 2087256064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383838383838386, + "loss": 2.5268, + "theoretical_loss": 3.418243517590036, + "tokens_seen": 2087321600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738181818181818, + "loss": 2.6601, + "theoretical_loss": 3.418234644404798, + "tokens_seen": 2087387136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737979797979798, + "loss": 2.7135, + "theoretical_loss": 3.4182257715761404, + "tokens_seen": 2087452672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737777777777778, + "loss": 2.5035, + "theoretical_loss": 3.4182168991040385, + "tokens_seen": 2087518208 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1180869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.378899574279785, + "objective/train/theoretical_loss": 3.4182080269884656, + "objective/train/tokens_used": 446442976, + "theoretical_loss": 3.4182080269884656, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375757575757574, + "loss": 2.5145, + "theoretical_loss": 3.4182080269884656, + "tokens_seen": 2087583744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373737373737375, + "loss": 2.6353, + "theoretical_loss": 3.4181991552293973, + "tokens_seen": 2087649280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371717171717176, + "loss": 2.5659, + "theoretical_loss": 3.4181902838268075, + "tokens_seen": 2087714816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736969696969697, + "loss": 2.4987, + "theoretical_loss": 3.4181814127806702, + "tokens_seen": 2087780352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037367676767676767, + "loss": 2.8278, + "theoretical_loss": 3.4181725420909608, + "tokens_seen": 2087845888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365656565656563, + "loss": 2.5232, + "theoretical_loss": 3.4181636717576533, + "tokens_seen": 2087911424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363636363636364, + "loss": 2.8216, + "theoretical_loss": 3.418154801780722, + "tokens_seen": 2087976960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037361616161616165, + "loss": 2.7915, + "theoretical_loss": 3.418145932160142, + "tokens_seen": 2088042496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735959595959596, + "loss": 2.8407, + "theoretical_loss": 3.4181370628958874, + "tokens_seen": 2088108032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357575757575757, + "loss": 2.7613, + "theoretical_loss": 3.4181281939879327, + "tokens_seen": 2088173568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735555555555555, + "loss": 2.6356, + "theoretical_loss": 3.418119325436253, + "tokens_seen": 2088239104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735353535353536, + "loss": 2.6973, + "theoretical_loss": 3.4181104572408216, + "tokens_seen": 2088304640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037351515151515154, + "loss": 2.7795, + "theoretical_loss": 3.4181015894016142, + "tokens_seen": 2088370176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734949494949495, + "loss": 2.641, + "theoretical_loss": 3.4180927219186046, + "tokens_seen": 2088435712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037347474747474746, + "loss": 2.6697, + "theoretical_loss": 3.418083854791768, + "tokens_seen": 2088501248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734545454545454, + "loss": 2.5598, + "theoretical_loss": 3.4180749880210777, + "tokens_seen": 2088566784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734343434343435, + "loss": 2.6919, + "theoretical_loss": 3.4180661216065094, + "tokens_seen": 2088632320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341414141414144, + "loss": 2.7822, + "theoretical_loss": 3.418057255548037, + "tokens_seen": 2088697856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733939393939394, + "loss": 2.5491, + "theoretical_loss": 3.4180483898456355, + "tokens_seen": 2088763392 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337373737373735, + "loss": 2.8278, + "theoretical_loss": 3.418039524499279, + "tokens_seen": 2088828928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733535353535353, + "loss": 2.7731, + "theoretical_loss": 3.418030659508942, + "tokens_seen": 2088894464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733333333333334, + "loss": 2.8359, + "theoretical_loss": 3.4180217948745995, + "tokens_seen": 2088960000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331313131313133, + "loss": 2.8885, + "theoretical_loss": 3.418012930596226, + "tokens_seen": 2089025536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732929292929293, + "loss": 2.9436, + "theoretical_loss": 3.418004066673795, + "tokens_seen": 2089091072 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327272727272724, + "loss": 2.7995, + "theoretical_loss": 3.4179952031072824, + "tokens_seen": 2089156608 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1181528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1214184761047363, + "objective/train/theoretical_loss": 3.417986339896662, + "objective/train/tokens_used": 448081376, + "theoretical_loss": 3.417986339896662, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732525252525253, + "loss": 2.913, + "theoretical_loss": 3.417986339896662, + "tokens_seen": 2089222144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037323232323232327, + "loss": 2.6446, + "theoretical_loss": 3.4179774770419087, + "tokens_seen": 2089287680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732121212121212, + "loss": 2.7521, + "theoretical_loss": 3.417968614542997, + "tokens_seen": 2089353216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731919191919192, + "loss": 2.7329, + "theoretical_loss": 3.4179597523999012, + "tokens_seen": 2089418752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037317171717171714, + "loss": 2.6947, + "theoretical_loss": 3.4179508906125955, + "tokens_seen": 2089484288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731515151515152, + "loss": 2.7646, + "theoretical_loss": 3.417942029181056, + "tokens_seen": 2089549824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037313131313131316, + "loss": 2.8275, + "theoretical_loss": 3.417933168105255, + "tokens_seen": 2089615360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731111111111111, + "loss": 2.8647, + "theoretical_loss": 3.417924307385169, + "tokens_seen": 2089680896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037309090909090907, + "loss": 2.7044, + "theoretical_loss": 3.4179154470207718, + "tokens_seen": 2089746432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730707070707071, + "loss": 2.4999, + "theoretical_loss": 3.4179065870120375, + "tokens_seen": 2089811968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730505050505051, + "loss": 2.8406, + "theoretical_loss": 3.417897727358942, + "tokens_seen": 2089877504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037303030303030305, + "loss": 2.8331, + "theoretical_loss": 3.4178888680614583, + "tokens_seen": 2089943040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373010101010101, + "loss": 2.7843, + "theoretical_loss": 3.4178800091195622, + "tokens_seen": 2090008576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037298989898989896, + "loss": 2.5609, + "theoretical_loss": 3.4178711505332275, + "tokens_seen": 2090074112 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372969696969697, + "loss": 2.7912, + "theoretical_loss": 3.417862292302429, + "tokens_seen": 2090139648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372949494949495, + "loss": 2.7693, + "theoretical_loss": 3.4178534344271414, + "tokens_seen": 2090205184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037292929292929294, + "loss": 2.6715, + "theoretical_loss": 3.4178445769073393, + "tokens_seen": 2090270720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729090909090909, + "loss": 2.8292, + "theoretical_loss": 3.417835719742998, + "tokens_seen": 2090336256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728888888888889, + "loss": 2.8096, + "theoretical_loss": 3.41782686293409, + "tokens_seen": 2090401792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286868686868687, + "loss": 2.7676, + "theoretical_loss": 3.417818006480592, + "tokens_seen": 2090467328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728484848484849, + "loss": 2.7543, + "theoretical_loss": 3.4178091503824777, + "tokens_seen": 2090532864 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282828282828284, + "loss": 2.7279, + "theoretical_loss": 3.4178002946397217, + "tokens_seen": 2090598400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728080808080808, + "loss": 2.7262, + "theoretical_loss": 3.417791439252299, + "tokens_seen": 2090663936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727878787878788, + "loss": 2.5825, + "theoretical_loss": 3.4177825842201837, + "tokens_seen": 2090729472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037276767676767676, + "loss": 2.7866, + "theoretical_loss": 3.4177737295433506, + "tokens_seen": 2090795008 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1182667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.67637300491333, + "objective/train/theoretical_loss": 3.417764875221774, + "objective/train/tokens_used": 449719776, + "theoretical_loss": 3.417764875221774, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037274747474747477, + "loss": 2.6176, + "theoretical_loss": 3.417764875221774, + "tokens_seen": 2090860544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037272727272727273, + "loss": 2.7257, + "theoretical_loss": 3.4177560212554297, + "tokens_seen": 2090926080 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270707070707074, + "loss": 2.7356, + "theoretical_loss": 3.4177471676442908, + "tokens_seen": 2090991616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726868686868687, + "loss": 2.7675, + "theoretical_loss": 3.417738314388333, + "tokens_seen": 2091057152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037266666666666665, + "loss": 2.4553, + "theoretical_loss": 3.4177294614875304, + "tokens_seen": 2091122688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037264646464646466, + "loss": 2.6609, + "theoretical_loss": 3.4177206089418575, + "tokens_seen": 2091188224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726262626262626, + "loss": 2.8409, + "theoretical_loss": 3.4177117567512894, + "tokens_seen": 2091253760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037260606060606063, + "loss": 2.543, + "theoretical_loss": 3.4177029049158003, + "tokens_seen": 2091319296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725858585858586, + "loss": 2.7544, + "theoretical_loss": 3.4176940534353655, + "tokens_seen": 2091384832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256565656565655, + "loss": 2.6978, + "theoretical_loss": 3.417685202309958, + "tokens_seen": 2091450368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254545454545456, + "loss": 2.8062, + "theoretical_loss": 3.417676351539555, + "tokens_seen": 2091515904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037252525252525257, + "loss": 2.7593, + "theoretical_loss": 3.4176675011241286, + "tokens_seen": 2091581440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725050505050505, + "loss": 2.5718, + "theoretical_loss": 3.417658651063655, + "tokens_seen": 2091646976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724848484848485, + "loss": 2.7728, + "theoretical_loss": 3.4176498013581087, + "tokens_seen": 2091712512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246464646464644, + "loss": 2.6721, + "theoretical_loss": 3.4176409520074635, + "tokens_seen": 2091778048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244444444444445, + "loss": 2.7338, + "theoretical_loss": 3.417632103011695, + "tokens_seen": 2091843584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037242424242424246, + "loss": 2.8671, + "theoretical_loss": 3.417623254370777, + "tokens_seen": 2091909120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724040404040404, + "loss": 2.6924, + "theoretical_loss": 3.4176144060846854, + "tokens_seen": 2091974656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723838383838384, + "loss": 2.7202, + "theoretical_loss": 3.4176055581533933, + "tokens_seen": 2092040192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037236363636363633, + "loss": 2.9382, + "theoretical_loss": 3.417596710576876, + "tokens_seen": 2092105728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723434343434344, + "loss": 2.6703, + "theoretical_loss": 3.417587863355109, + "tokens_seen": 2092171264 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037232323232323235, + "loss": 2.688, + "theoretical_loss": 3.417579016488066, + "tokens_seen": 2092236800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723030303030303, + "loss": 2.6267, + "theoretical_loss": 3.417570169975722, + "tokens_seen": 2092302336 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037228282828282827, + "loss": 2.5997, + "theoretical_loss": 3.417561323818051, + "tokens_seen": 2092367872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722626262626262, + "loss": 2.4706, + "theoretical_loss": 3.417552478015029, + "tokens_seen": 2092433408 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1183291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9553709030151367, + "objective/train/theoretical_loss": 3.4175436325666295, + "objective/train/tokens_used": 451358176, + "theoretical_loss": 3.4175436325666295, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722424242424243, + "loss": 2.708, + "theoretical_loss": 3.4175436325666295, + "tokens_seen": 2092498944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037222222222222225, + "loss": 2.6738, + "theoretical_loss": 3.4175347874728277, + "tokens_seen": 2092564480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722020202020202, + "loss": 2.6737, + "theoretical_loss": 3.417525942733598, + "tokens_seen": 2092630016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218181818181816, + "loss": 2.6803, + "theoretical_loss": 3.417517098348916, + "tokens_seen": 2092695552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037216161616161617, + "loss": 2.5649, + "theoretical_loss": 3.4175082543187547, + "tokens_seen": 2092761088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721414141414142, + "loss": 2.7492, + "theoretical_loss": 3.4174994106430905, + "tokens_seen": 2092826624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212121212121214, + "loss": 2.9105, + "theoretical_loss": 3.417490567321897, + "tokens_seen": 2092892160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721010101010101, + "loss": 2.8093, + "theoretical_loss": 3.4174817243551487, + "tokens_seen": 2092957696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037208080808080805, + "loss": 2.8824, + "theoretical_loss": 3.4174728817428215, + "tokens_seen": 2093023232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037206060606060606, + "loss": 2.6329, + "theoretical_loss": 3.417464039484889, + "tokens_seen": 2093088768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720404040404041, + "loss": 2.7667, + "theoretical_loss": 3.4174551975813268, + "tokens_seen": 2093154304 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202020202020203, + "loss": 2.907, + "theoretical_loss": 3.417446356032109, + "tokens_seen": 2093219840 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372, + "loss": 2.7343, + "theoretical_loss": 3.4174375148372103, + "tokens_seen": 2093285376 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371979797979798, + "loss": 2.6522, + "theoretical_loss": 3.4174286739966053, + "tokens_seen": 2093350912 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195959595959596, + "loss": 2.5836, + "theoretical_loss": 3.4174198335102695, + "tokens_seen": 2093416448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037193939393939397, + "loss": 2.5734, + "theoretical_loss": 3.417410993378177, + "tokens_seen": 2093481984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719191919191919, + "loss": 2.9375, + "theoretical_loss": 3.4174021536003023, + "tokens_seen": 2093547520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718989898989899, + "loss": 2.6761, + "theoretical_loss": 3.4173933141766204, + "tokens_seen": 2093613056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718787878787879, + "loss": 2.7522, + "theoretical_loss": 3.417384475107106, + "tokens_seen": 2093678592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037185858585858585, + "loss": 2.5223, + "theoretical_loss": 3.417375636391734, + "tokens_seen": 2093744128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183838383838386, + "loss": 2.886, + "theoretical_loss": 3.417366798030479, + "tokens_seen": 2093809664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718181818181818, + "loss": 2.6464, + "theoretical_loss": 3.4173579600233155, + "tokens_seen": 2093875200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179797979797983, + "loss": 2.6506, + "theoretical_loss": 3.4173491223702186, + "tokens_seen": 2093940736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717777777777778, + "loss": 2.8964, + "theoretical_loss": 3.417340285071163, + "tokens_seen": 2094006272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175757575757574, + "loss": 2.7919, + "theoretical_loss": 3.4173314481261228, + "tokens_seen": 2094071808 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1183994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6880807876586914, + "objective/train/theoretical_loss": 3.4173226115350737, + "objective/train/tokens_used": 452996576, + "theoretical_loss": 3.4173226115350737, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037173737373737375, + "loss": 2.6333, + "theoretical_loss": 3.4173226115350737, + "tokens_seen": 2094137344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717171717171717, + "loss": 2.626, + "theoretical_loss": 3.41731377529799, + "tokens_seen": 2094202880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716969696969697, + "loss": 2.6375, + "theoretical_loss": 3.417304939414846, + "tokens_seen": 2094268416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716767676767677, + "loss": 2.7191, + "theoretical_loss": 3.4172961038856173, + "tokens_seen": 2094333952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037165656565656564, + "loss": 2.4352, + "theoretical_loss": 3.417287268710278, + "tokens_seen": 2094399488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037163636363636365, + "loss": 2.788, + "theoretical_loss": 3.4172784338888036, + "tokens_seen": 2094465024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716161616161616, + "loss": 2.6356, + "theoretical_loss": 3.417269599421168, + "tokens_seen": 2094530560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715959595959596, + "loss": 2.471, + "theoretical_loss": 3.417260765307346, + "tokens_seen": 2094596096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037157575757575757, + "loss": 2.7217, + "theoretical_loss": 3.417251931547313, + "tokens_seen": 2094661632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155555555555553, + "loss": 2.7595, + "theoretical_loss": 3.417243098141043, + "tokens_seen": 2094727168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153535353535354, + "loss": 2.6582, + "theoretical_loss": 3.4172342650885117, + "tokens_seen": 2094792704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037151515151515155, + "loss": 2.7223, + "theoretical_loss": 3.417225432389693, + "tokens_seen": 2094858240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714949494949495, + "loss": 2.4898, + "theoretical_loss": 3.4172166000445623, + "tokens_seen": 2094923776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147474747474746, + "loss": 2.6852, + "theoretical_loss": 3.417207768053094, + "tokens_seen": 2094989312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714545454545455, + "loss": 2.6106, + "theoretical_loss": 3.4171989364152626, + "tokens_seen": 2095054848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143434343434343, + "loss": 2.8534, + "theoretical_loss": 3.417190105131044, + "tokens_seen": 2095120384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037141414141414144, + "loss": 2.5313, + "theoretical_loss": 3.4171812742004115, + "tokens_seen": 2095185920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713939393939394, + "loss": 2.5031, + "theoretical_loss": 3.417172443623341, + "tokens_seen": 2095251456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037137373737373736, + "loss": 2.6554, + "theoretical_loss": 3.417163613399807, + "tokens_seen": 2095316992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135353535353537, + "loss": 2.7411, + "theoretical_loss": 3.417154783529784, + "tokens_seen": 2095382528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713333333333334, + "loss": 2.655, + "theoretical_loss": 3.4171459540132467, + "tokens_seen": 2095448064 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131313131313134, + "loss": 2.5942, + "theoretical_loss": 3.4171371248501705, + "tokens_seen": 2095513600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712929292929293, + "loss": 2.872, + "theoretical_loss": 3.41712829604053, + "tokens_seen": 2095579136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037127272727272725, + "loss": 2.7185, + "theoretical_loss": 3.4171194675842997, + "tokens_seen": 2095644672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037125252525252526, + "loss": 2.7512, + "theoretical_loss": 3.4171106394814545, + "tokens_seen": 2095710208 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 1184873, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4112489223480225, + "objective/train/theoretical_loss": 3.41710181173197, + "objective/train/tokens_used": 454634976, + "theoretical_loss": 3.41710181173197, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123232323232327, + "loss": 2.4741, + "theoretical_loss": 3.41710181173197, + "tokens_seen": 2095775744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037121212121212123, + "loss": 2.6282, + "theoretical_loss": 3.4170929843358193, + "tokens_seen": 2095841280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711919191919192, + "loss": 2.7003, + "theoretical_loss": 3.4170841572929787, + "tokens_seen": 2095906816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037117171717171714, + "loss": 2.7601, + "theoretical_loss": 3.4170753306034225, + "tokens_seen": 2095972352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711515151515152, + "loss": 2.5953, + "theoretical_loss": 3.417066504267125, + "tokens_seen": 2096037888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037113131313131316, + "loss": 2.7331, + "theoretical_loss": 3.4170576782840625, + "tokens_seen": 2096103424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711111111111111, + "loss": 2.5689, + "theoretical_loss": 3.417048852654208, + "tokens_seen": 2096168960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710909090909091, + "loss": 2.6019, + "theoretical_loss": 3.4170400273775376, + "tokens_seen": 2096234496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107070707070703, + "loss": 2.8769, + "theoretical_loss": 3.4170312024540257, + "tokens_seen": 2096300032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710505050505051, + "loss": 2.7009, + "theoretical_loss": 3.4170223778836473, + "tokens_seen": 2096365568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037103030303030306, + "loss": 2.4419, + "theoretical_loss": 3.4170135536663766, + "tokens_seen": 2096431104 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371010101010101, + "loss": 2.5266, + "theoretical_loss": 3.4170047298021893, + "tokens_seen": 2096496640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037098989898989897, + "loss": 2.9366, + "theoretical_loss": 3.41699590629106, + "tokens_seen": 2096562176 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370969696969697, + "loss": 2.8209, + "theoretical_loss": 3.4169870831329625, + "tokens_seen": 2096627712 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370949494949495, + "loss": 2.8343, + "theoretical_loss": 3.4169782603278733, + "tokens_seen": 2096693248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092929292929295, + "loss": 2.6292, + "theoretical_loss": 3.4169694378757662, + "tokens_seen": 2096758784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709090909090909, + "loss": 2.5373, + "theoretical_loss": 3.4169606157766164, + "tokens_seen": 2096824320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088888888888886, + "loss": 2.6785, + "theoretical_loss": 3.416951794030399, + "tokens_seen": 2096889856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708686868686869, + "loss": 2.6523, + "theoretical_loss": 3.416942972637088, + "tokens_seen": 2096955392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708484848484849, + "loss": 2.6102, + "theoretical_loss": 3.4169341515966583, + "tokens_seen": 2097020928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082828282828284, + "loss": 2.6107, + "theoretical_loss": 3.416925330909086, + "tokens_seen": 2097086464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708080808080808, + "loss": 2.6086, + "theoretical_loss": 3.416916510574345, + "tokens_seen": 2097152000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707878787878788, + "loss": 2.6176, + "theoretical_loss": 3.41690769059241, + "tokens_seen": 2097217536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037076767676767677, + "loss": 2.6779, + "theoretical_loss": 3.416898870963257, + "tokens_seen": 2097283072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707474747474748, + "loss": 2.8332, + "theoretical_loss": 3.416890051686859, + "tokens_seen": 2097348608 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1185597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5074687004089355, + "objective/train/theoretical_loss": 3.4168812327631928, + "objective/train/tokens_used": 456273376, + "theoretical_loss": 3.4168812327631928, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037072727272727273, + "loss": 2.578, + "theoretical_loss": 3.4168812327631928, + "tokens_seen": 2097414144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707070707070707, + "loss": 2.7551, + "theoretical_loss": 3.416872414192232, + "tokens_seen": 2097479680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706868686868687, + "loss": 2.6951, + "theoretical_loss": 3.416863595973952, + "tokens_seen": 2097545216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066666666666666, + "loss": 2.7757, + "theoretical_loss": 3.4168547781083274, + "tokens_seen": 2097610752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037064646464646467, + "loss": 2.8056, + "theoretical_loss": 3.4168459605953334, + "tokens_seen": 2097676288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037062626262626263, + "loss": 2.8481, + "theoretical_loss": 3.416837143434945, + "tokens_seen": 2097741824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060606060606064, + "loss": 2.7464, + "theoretical_loss": 3.416828326627136, + "tokens_seen": 2097807360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705858585858586, + "loss": 2.6697, + "theoretical_loss": 3.416819510171883, + "tokens_seen": 2097872896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037056565656565655, + "loss": 2.8409, + "theoretical_loss": 3.4168106940691594, + "tokens_seen": 2097938432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037054545454545456, + "loss": 2.6788, + "theoretical_loss": 3.4168018783189407, + "tokens_seen": 2098003968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705252525252525, + "loss": 2.4458, + "theoretical_loss": 3.4167930629212018, + "tokens_seen": 2098069504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050505050505053, + "loss": 2.7079, + "theoretical_loss": 3.416784247875918, + "tokens_seen": 2098135040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704848484848485, + "loss": 2.7792, + "theoretical_loss": 3.416775433183063, + "tokens_seen": 2098200576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037046464646464644, + "loss": 2.4727, + "theoretical_loss": 3.4167666188426127, + "tokens_seen": 2098266112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044444444444446, + "loss": 2.8683, + "theoretical_loss": 3.4167578048545417, + "tokens_seen": 2098331648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037042424242424247, + "loss": 2.6553, + "theoretical_loss": 3.4167489912188254, + "tokens_seen": 2098397184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704040404040404, + "loss": 2.5184, + "theoretical_loss": 3.4167401779354383, + "tokens_seen": 2098462720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703838383838384, + "loss": 2.825, + "theoretical_loss": 3.416731365004355, + "tokens_seen": 2098528256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036363636363634, + "loss": 2.7729, + "theoretical_loss": 3.4167225524255507, + "tokens_seen": 2098593792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037034343434343435, + "loss": 2.7605, + "theoretical_loss": 3.4167137401990004, + "tokens_seen": 2098659328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032323232323236, + "loss": 2.7298, + "theoretical_loss": 3.4167049283246786, + "tokens_seen": 2098724864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703030303030303, + "loss": 2.6928, + "theoretical_loss": 3.4166961168025614, + "tokens_seen": 2098790400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702828282828283, + "loss": 2.5398, + "theoretical_loss": 3.4166873056326224, + "tokens_seen": 2098855936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037026262626262623, + "loss": 2.6571, + "theoretical_loss": 3.416678494814837, + "tokens_seen": 2098921472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702424242424243, + "loss": 2.7256, + "theoretical_loss": 3.4166696843491806, + "tokens_seen": 2098987008 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1186813, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2527925968170166, + "objective/train/theoretical_loss": 3.4166608742356273, + "objective/train/tokens_used": 457911776, + "theoretical_loss": 3.4166608742356273, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037022222222222225, + "loss": 2.7967, + "theoretical_loss": 3.4166608742356273, + "tokens_seen": 2099052544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702020202020202, + "loss": 2.8061, + "theoretical_loss": 3.4166520644741523, + "tokens_seen": 2099118080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037018181818181817, + "loss": 2.5017, + "theoretical_loss": 3.4166432550647308, + "tokens_seen": 2099183616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701616161616161, + "loss": 2.8703, + "theoretical_loss": 3.4166344460073375, + "tokens_seen": 2099249152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701414141414142, + "loss": 2.9093, + "theoretical_loss": 3.416625637301948, + "tokens_seen": 2099314688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012121212121215, + "loss": 2.7592, + "theoretical_loss": 3.416616828948536, + "tokens_seen": 2099380224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701010101010101, + "loss": 2.6666, + "theoretical_loss": 3.4166080209470775, + "tokens_seen": 2099445760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037008080808080806, + "loss": 2.6004, + "theoretical_loss": 3.416599213297547, + "tokens_seen": 2099511296 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370060606060606, + "loss": 2.4781, + "theoretical_loss": 3.4165904059999193, + "tokens_seen": 2099576832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700404040404041, + "loss": 2.7364, + "theoretical_loss": 3.41658159905417, + "tokens_seen": 2099642368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002020202020204, + "loss": 2.5846, + "theoretical_loss": 3.416572792460274, + "tokens_seen": 2099707904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 2.5038, + "theoretical_loss": 3.4165639862182053, + "tokens_seen": 2099773440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997979797979795, + "loss": 2.6721, + "theoretical_loss": 3.4165551803279395, + "tokens_seen": 2099838976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036995959595959596, + "loss": 2.588, + "theoretical_loss": 3.416546374789452, + "tokens_seen": 2099904512 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369939393939394, + "loss": 2.8662, + "theoretical_loss": 3.416537569602717, + "tokens_seen": 2099970048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036991919191919193, + "loss": 2.8381, + "theoretical_loss": 3.41652876476771, + "tokens_seen": 2100035584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698989898989899, + "loss": 2.699, + "theoretical_loss": 3.4165199602844054, + "tokens_seen": 2100101120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036987878787878784, + "loss": 2.6624, + "theoretical_loss": 3.416511156152779, + "tokens_seen": 2100166656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985858585858586, + "loss": 2.6138, + "theoretical_loss": 3.416502352372805, + "tokens_seen": 2100232192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036983838383838387, + "loss": 2.7946, + "theoretical_loss": 3.416493548944459, + "tokens_seen": 2100297728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698181818181818, + "loss": 2.6193, + "theoretical_loss": 3.4164847458677157, + "tokens_seen": 2100363264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697979797979798, + "loss": 2.6454, + "theoretical_loss": 3.4164759431425495, + "tokens_seen": 2100428800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697777777777778, + "loss": 2.874, + "theoretical_loss": 3.4164671407689364, + "tokens_seen": 2100494336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697575757575758, + "loss": 2.7876, + "theoretical_loss": 3.4164583387468515, + "tokens_seen": 2100559872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973737373737376, + "loss": 2.5905, + "theoretical_loss": 3.4164495370762684, + "tokens_seen": 2100625408 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1187470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0909693241119385, + "objective/train/theoretical_loss": 3.4164407357571633, + "objective/train/tokens_used": 459550176, + "theoretical_loss": 3.4164407357571633, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697171717171717, + "loss": 2.8091, + "theoretical_loss": 3.4164407357571633, + "tokens_seen": 2100690944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969696969696967, + "loss": 2.6742, + "theoretical_loss": 3.416431934789511, + "tokens_seen": 2100756480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696767676767677, + "loss": 2.7516, + "theoretical_loss": 3.416423134173286, + "tokens_seen": 2100822016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696565656565657, + "loss": 2.8025, + "theoretical_loss": 3.416414333908464, + "tokens_seen": 2100887552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963636363636365, + "loss": 2.4464, + "theoretical_loss": 3.4164055339950195, + "tokens_seen": 2100953088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696161616161616, + "loss": 2.5777, + "theoretical_loss": 3.4163967344329276, + "tokens_seen": 2101018624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695959595959596, + "loss": 2.752, + "theoretical_loss": 3.416387935222164, + "tokens_seen": 2101084160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695757575757576, + "loss": 2.5547, + "theoretical_loss": 3.4163791363627025, + "tokens_seen": 2101149696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695555555555556, + "loss": 2.6544, + "theoretical_loss": 3.4163703378545187, + "tokens_seen": 2101215232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036953535353535354, + "loss": 2.5115, + "theoretical_loss": 3.4163615396975877, + "tokens_seen": 2101280768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695151515151515, + "loss": 2.6246, + "theoretical_loss": 3.4163527418918846, + "tokens_seen": 2101346304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694949494949495, + "loss": 2.7929, + "theoretical_loss": 3.4163439444373846, + "tokens_seen": 2101411840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947474747474747, + "loss": 2.6689, + "theoretical_loss": 3.416335147334062, + "tokens_seen": 2101477376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694545454545455, + "loss": 2.5958, + "theoretical_loss": 3.416326350581892, + "tokens_seen": 2101542912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943434343434344, + "loss": 2.6785, + "theoretical_loss": 3.416317554180851, + "tokens_seen": 2101608448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036941414141414145, + "loss": 2.6988, + "theoretical_loss": 3.4163087581309117, + "tokens_seen": 2101673984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693939393939394, + "loss": 2.6749, + "theoretical_loss": 3.416299962432051, + "tokens_seen": 2101739520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036937373737373736, + "loss": 2.4373, + "theoretical_loss": 3.4162911670842435, + "tokens_seen": 2101805056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693535353535354, + "loss": 2.7377, + "theoretical_loss": 3.4162823720874633, + "tokens_seen": 2101870592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933333333333333, + "loss": 2.8838, + "theoretical_loss": 3.416273577441687, + "tokens_seen": 2101936128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036931313131313134, + "loss": 2.872, + "theoretical_loss": 3.416264783146888, + "tokens_seen": 2102001664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692929292929293, + "loss": 2.7968, + "theoretical_loss": 3.4162559892030426, + "tokens_seen": 2102067200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036927272727272725, + "loss": 2.6706, + "theoretical_loss": 3.4162471956101257, + "tokens_seen": 2102132736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036925252525252527, + "loss": 2.5427, + "theoretical_loss": 3.416238402368112, + "tokens_seen": 2102198272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692323232323233, + "loss": 2.849, + "theoretical_loss": 3.4162296094769764, + "tokens_seen": 2102263808 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1188148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2095654010772705, + "objective/train/theoretical_loss": 3.4162208169366943, + "objective/train/tokens_used": 461188576, + "theoretical_loss": 3.4162208169366943, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921212121212123, + "loss": 2.7433, + "theoretical_loss": 3.4162208169366943, + "tokens_seen": 2102329344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691919191919192, + "loss": 2.7125, + "theoretical_loss": 3.416212024747241, + "tokens_seen": 2102394880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036917171717171715, + "loss": 2.6403, + "theoretical_loss": 3.416203232908591, + "tokens_seen": 2102460416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036915151515151516, + "loss": 2.7595, + "theoretical_loss": 3.4161944414207195, + "tokens_seen": 2102525952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913131313131317, + "loss": 2.7929, + "theoretical_loss": 3.4161856502836017, + "tokens_seen": 2102591488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691111111111111, + "loss": 2.8143, + "theoretical_loss": 3.416176859497213, + "tokens_seen": 2102657024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690909090909091, + "loss": 2.7184, + "theoretical_loss": 3.416168069061528, + "tokens_seen": 2102722560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907070707070704, + "loss": 2.5188, + "theoretical_loss": 3.416159278976522, + "tokens_seen": 2102788096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690505050505051, + "loss": 2.6113, + "theoretical_loss": 3.4161504892421695, + "tokens_seen": 2102853632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903030303030306, + "loss": 2.4841, + "theoretical_loss": 3.4161416998584464, + "tokens_seen": 2102919168 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369010101010101, + "loss": 2.8971, + "theoretical_loss": 3.4161329108253278, + "tokens_seen": 2102984704 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368989898989899, + "loss": 2.6184, + "theoretical_loss": 3.4161241221427883, + "tokens_seen": 2103050240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036896969696969693, + "loss": 2.6844, + "theoretical_loss": 3.416115333810803, + "tokens_seen": 2103115776 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368949494949495, + "loss": 2.6864, + "theoretical_loss": 3.416106545829347, + "tokens_seen": 2103181312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036892929292929295, + "loss": 2.6756, + "theoretical_loss": 3.4160977581983962, + "tokens_seen": 2103246848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689090909090909, + "loss": 2.5503, + "theoretical_loss": 3.4160889709179245, + "tokens_seen": 2103312384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036888888888888887, + "loss": 2.7396, + "theoretical_loss": 3.4160801839879076, + "tokens_seen": 2103377920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688686868686869, + "loss": 2.7507, + "theoretical_loss": 3.4160713974083206, + "tokens_seen": 2103443456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688484848484849, + "loss": 2.6596, + "theoretical_loss": 3.4160626111791386, + "tokens_seen": 2103508992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882828282828285, + "loss": 2.7815, + "theoretical_loss": 3.4160538253003367, + "tokens_seen": 2103574528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688080808080808, + "loss": 2.5875, + "theoretical_loss": 3.41604503977189, + "tokens_seen": 2103640064 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878787878787876, + "loss": 2.6728, + "theoretical_loss": 3.4160362545937732, + "tokens_seen": 2103705600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876767676767677, + "loss": 2.7731, + "theoretical_loss": 3.4160274697659623, + "tokens_seen": 2103771136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687474747474748, + "loss": 2.8371, + "theoretical_loss": 3.4160186852884316, + "tokens_seen": 2103836672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036872727272727274, + "loss": 2.5822, + "theoretical_loss": 3.416009901161157, + "tokens_seen": 2103902208 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1189669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7278225421905518, + "objective/train/theoretical_loss": 3.4160011173841127, + "objective/train/tokens_used": 462826976, + "theoretical_loss": 3.4160011173841127, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687070707070707, + "loss": 2.7883, + "theoretical_loss": 3.4160011173841127, + "tokens_seen": 2103967744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686868686868687, + "loss": 2.5314, + "theoretical_loss": 3.4159923339572744, + "tokens_seen": 2104033280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866666666666667, + "loss": 2.8427, + "theoretical_loss": 3.4159835508806173, + "tokens_seen": 2104098816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686464646464647, + "loss": 2.5119, + "theoretical_loss": 3.4159747681541166, + "tokens_seen": 2104164352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036862626262626263, + "loss": 2.7064, + "theoretical_loss": 3.4159659857777465, + "tokens_seen": 2104229888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686060606060606, + "loss": 2.6418, + "theoretical_loss": 3.415957203751484, + "tokens_seen": 2104295424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685858585858586, + "loss": 2.9051, + "theoretical_loss": 3.415948422075302, + "tokens_seen": 2104360960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856565656565656, + "loss": 2.6531, + "theoretical_loss": 3.415939640749177, + "tokens_seen": 2104426496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854545454545457, + "loss": 2.8056, + "theoretical_loss": 3.415930859773084, + "tokens_seen": 2104492032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685252525252525, + "loss": 2.7024, + "theoretical_loss": 3.415922079146998, + "tokens_seen": 2104557568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850505050505054, + "loss": 2.8006, + "theoretical_loss": 3.415913298870894, + "tokens_seen": 2104623104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684848484848485, + "loss": 2.6255, + "theoretical_loss": 3.4159045189447474, + "tokens_seen": 2104688640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846464646464645, + "loss": 2.6814, + "theoretical_loss": 3.4158957393685334, + "tokens_seen": 2104754176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036844444444444446, + "loss": 2.6115, + "theoretical_loss": 3.415886960142227, + "tokens_seen": 2104819712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684242424242424, + "loss": 2.6856, + "theoretical_loss": 3.4158781812658034, + "tokens_seen": 2104885248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036840404040404043, + "loss": 2.6628, + "theoretical_loss": 3.415869402739238, + "tokens_seen": 2104950784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683838383838384, + "loss": 2.526, + "theoretical_loss": 3.415860624562505, + "tokens_seen": 2105016320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036836363636363634, + "loss": 2.6997, + "theoretical_loss": 3.415851846735581, + "tokens_seen": 2105081856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834343434343435, + "loss": 2.6645, + "theoretical_loss": 3.4158430692584405, + "tokens_seen": 2105147392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683232323232323, + "loss": 2.5609, + "theoretical_loss": 3.4158342921310583, + "tokens_seen": 2105212928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683030303030303, + "loss": 2.7897, + "theoretical_loss": 3.4158255153534096, + "tokens_seen": 2105278464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682828282828283, + "loss": 2.6315, + "theoretical_loss": 3.4158167389254706, + "tokens_seen": 2105344000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826262626262624, + "loss": 2.7431, + "theoretical_loss": 3.4158079628472153, + "tokens_seen": 2105409536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824242424242425, + "loss": 2.6709, + "theoretical_loss": 3.415799187118619, + "tokens_seen": 2105475072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822222222222226, + "loss": 2.4026, + "theoretical_loss": 3.415790411739658, + "tokens_seen": 2105540608 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1190193, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776749849319458, + "objective/train/theoretical_loss": 3.4157816367103067, + "objective/train/tokens_used": 464465376, + "theoretical_loss": 3.4157816367103067, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682020202020202, + "loss": 2.7056, + "theoretical_loss": 3.4157816367103067, + "tokens_seen": 2105606144 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818181818181817, + "loss": 2.864, + "theoretical_loss": 3.41577286203054, + "tokens_seen": 2105671680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816161616161613, + "loss": 2.6083, + "theoretical_loss": 3.4157640877003335, + "tokens_seen": 2105737216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036814141414141414, + "loss": 2.772, + "theoretical_loss": 3.415755313719662, + "tokens_seen": 2105802752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036812121212121215, + "loss": 2.7643, + "theoretical_loss": 3.4157465400885014, + "tokens_seen": 2105868288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681010101010101, + "loss": 2.6916, + "theoretical_loss": 3.415737766806826, + "tokens_seen": 2105933824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808080808080806, + "loss": 2.5278, + "theoretical_loss": 3.4157289938746125, + "tokens_seen": 2105999360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680606060606061, + "loss": 2.7007, + "theoretical_loss": 3.415720221291834, + "tokens_seen": 2106064896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680404040404041, + "loss": 2.761, + "theoretical_loss": 3.4157114490584677, + "tokens_seen": 2106130432 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802020202020204, + "loss": 2.8517, + "theoretical_loss": 3.4157026771744876, + "tokens_seen": 2106195968 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368, + "loss": 2.6837, + "theoretical_loss": 3.415693905639869, + "tokens_seen": 2106261504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036797979797979796, + "loss": 2.723, + "theoretical_loss": 3.4156851344545878, + "tokens_seen": 2106327040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795959595959597, + "loss": 2.7641, + "theoretical_loss": 3.4156763636186183, + "tokens_seen": 2106392576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367939393939394, + "loss": 2.7054, + "theoretical_loss": 3.4156675931319365, + "tokens_seen": 2106458112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791919191919194, + "loss": 2.5806, + "theoretical_loss": 3.4156588229945175, + "tokens_seen": 2106523648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678989898989899, + "loss": 2.7311, + "theoretical_loss": 3.415650053206336, + "tokens_seen": 2106589184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787878787878785, + "loss": 2.5394, + "theoretical_loss": 3.4156412837673678, + "tokens_seen": 2106654720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678585858585859, + "loss": 2.699, + "theoretical_loss": 3.415632514677588, + "tokens_seen": 2106720256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036783838383838387, + "loss": 2.7173, + "theoretical_loss": 3.415623745936972, + "tokens_seen": 2106785792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781818181818183, + "loss": 2.6974, + "theoretical_loss": 3.415614977545494, + "tokens_seen": 2106851328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677979797979798, + "loss": 2.7158, + "theoretical_loss": 3.4156062095031308, + "tokens_seen": 2106916864 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777777777777774, + "loss": 2.8261, + "theoretical_loss": 3.4155974418098562, + "tokens_seen": 2106982400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677575757575758, + "loss": 2.8508, + "theoretical_loss": 3.4155886744656465, + "tokens_seen": 2107047936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773737373737376, + "loss": 2.8959, + "theoretical_loss": 3.4155799074704762, + "tokens_seen": 2107113472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677171717171717, + "loss": 2.4589, + "theoretical_loss": 3.415571140824321, + "tokens_seen": 2107179008 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1191719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7232840061187744, + "objective/train/theoretical_loss": 3.4155623745271564, + "objective/train/tokens_used": 466103776, + "theoretical_loss": 3.4155623745271564, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676969696969697, + "loss": 2.6719, + "theoretical_loss": 3.4155623745271564, + "tokens_seen": 2107244544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676767676767677, + "loss": 2.7912, + "theoretical_loss": 3.415553608578957, + "tokens_seen": 2107310080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676565656565657, + "loss": 2.5866, + "theoretical_loss": 3.4155448429796986, + "tokens_seen": 2107375616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036763636363636366, + "loss": 2.6986, + "theoretical_loss": 3.415536077729356, + "tokens_seen": 2107441152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676161616161616, + "loss": 2.8317, + "theoretical_loss": 3.415527312827905, + "tokens_seen": 2107506688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759595959595957, + "loss": 2.6554, + "theoretical_loss": 3.4155185482753203, + "tokens_seen": 2107572224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675757575757576, + "loss": 2.5154, + "theoretical_loss": 3.4155097840715776, + "tokens_seen": 2107637760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675555555555556, + "loss": 2.712, + "theoretical_loss": 3.415501020216652, + "tokens_seen": 2107703296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036753535353535355, + "loss": 2.8242, + "theoretical_loss": 3.415492256710518, + "tokens_seen": 2107768832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675151515151515, + "loss": 2.824, + "theoretical_loss": 3.4154834935531526, + "tokens_seen": 2107834368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674949494949495, + "loss": 2.5284, + "theoretical_loss": 3.41547473074453, + "tokens_seen": 2107899904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674747474747475, + "loss": 2.4567, + "theoretical_loss": 3.415465968284625, + "tokens_seen": 2107965440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674545454545455, + "loss": 2.8097, + "theoretical_loss": 3.4154572061734143, + "tokens_seen": 2108030976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743434343434344, + "loss": 2.5752, + "theoretical_loss": 3.4154484444108717, + "tokens_seen": 2108096512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674141414141414, + "loss": 2.7789, + "theoretical_loss": 3.4154396829969733, + "tokens_seen": 2108162048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673939393939394, + "loss": 2.739, + "theoretical_loss": 3.4154309219316943, + "tokens_seen": 2108227584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036737373737373737, + "loss": 2.615, + "theoretical_loss": 3.4154221612150097, + "tokens_seen": 2108293120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673535353535354, + "loss": 2.4258, + "theoretical_loss": 3.4154134008468953, + "tokens_seen": 2108358656 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036733333333333334, + "loss": 2.6264, + "theoretical_loss": 3.415404640827326, + "tokens_seen": 2108424192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731313131313135, + "loss": 2.6721, + "theoretical_loss": 3.4153958811562775, + "tokens_seen": 2108489728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672929292929293, + "loss": 2.6627, + "theoretical_loss": 3.4153871218337244, + "tokens_seen": 2108555264 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727272727272726, + "loss": 2.6224, + "theoretical_loss": 3.4153783628596424, + "tokens_seen": 2108620800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036725252525252527, + "loss": 2.7675, + "theoretical_loss": 3.415369604234007, + "tokens_seen": 2108686336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723232323232323, + "loss": 2.8422, + "theoretical_loss": 3.4153608459567932, + "tokens_seen": 2108751872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721212121212124, + "loss": 2.6723, + "theoretical_loss": 3.415352088027977, + "tokens_seen": 2108817408 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1192553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.885326862335205, + "objective/train/theoretical_loss": 3.4153433304475325, + "objective/train/tokens_used": 467742176, + "theoretical_loss": 3.4153433304475325, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671919191919192, + "loss": 2.6761, + "theoretical_loss": 3.4153433304475325, + "tokens_seen": 2108882944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036717171717171715, + "loss": 2.631, + "theoretical_loss": 3.415334573215436, + "tokens_seen": 2108948480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036715151515151516, + "loss": 2.6026, + "theoretical_loss": 3.4153258163316624, + "tokens_seen": 2109014016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671313131313132, + "loss": 2.7387, + "theoretical_loss": 3.4153170597961875, + "tokens_seen": 2109079552 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036711111111111113, + "loss": 2.71, + "theoretical_loss": 3.4153083036089855, + "tokens_seen": 2109145088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670909090909091, + "loss": 2.656, + "theoretical_loss": 3.415299547770033, + "tokens_seen": 2109210624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036707070707070705, + "loss": 2.5658, + "theoretical_loss": 3.415290792279305, + "tokens_seen": 2109276160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705050505050506, + "loss": 2.5847, + "theoretical_loss": 3.415282037136776, + "tokens_seen": 2109341696 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036703030303030307, + "loss": 2.5255, + "theoretical_loss": 3.4152732823424223, + "tokens_seen": 2109407232 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367010101010101, + "loss": 2.7802, + "theoretical_loss": 3.415264527896219, + "tokens_seen": 2109472768 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366989898989899, + "loss": 2.7344, + "theoretical_loss": 3.4152557737981413, + "tokens_seen": 2109538304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036696969696969694, + "loss": 2.6606, + "theoretical_loss": 3.4152470200481644, + "tokens_seen": 2109603840 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366949494949495, + "loss": 2.6509, + "theoretical_loss": 3.4152382666462637, + "tokens_seen": 2109669376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036692929292929296, + "loss": 2.4633, + "theoretical_loss": 3.4152295135924153, + "tokens_seen": 2109734912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669090909090909, + "loss": 2.5414, + "theoretical_loss": 3.4152207608865934, + "tokens_seen": 2109800448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668888888888889, + "loss": 2.6675, + "theoretical_loss": 3.4152120085287745, + "tokens_seen": 2109865984 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686868686868683, + "loss": 2.6963, + "theoretical_loss": 3.4152032565189323, + "tokens_seen": 2109931520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668484848484849, + "loss": 2.9166, + "theoretical_loss": 3.415194504857044, + "tokens_seen": 2109997056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682828282828285, + "loss": 2.5394, + "theoretical_loss": 3.415185753543084, + "tokens_seen": 2110062592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668080808080808, + "loss": 2.584, + "theoretical_loss": 3.415177002577028, + "tokens_seen": 2110128128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678787878787877, + "loss": 2.61, + "theoretical_loss": 3.4151682519588507, + "tokens_seen": 2110193664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667676767676767, + "loss": 2.6521, + "theoretical_loss": 3.415159501688528, + "tokens_seen": 2110259200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667474747474748, + "loss": 2.6732, + "theoretical_loss": 3.4151507517660358, + "tokens_seen": 2110324736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672727272727275, + "loss": 2.6576, + "theoretical_loss": 3.4151420021913483, + "tokens_seen": 2110390272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667070707070707, + "loss": 2.5524, + "theoretical_loss": 3.415133252964442, + "tokens_seen": 2110455808 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1194000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4039688110351562, + "objective/train/theoretical_loss": 3.4151245040852913, + "objective/train/tokens_used": 469380576, + "theoretical_loss": 3.4151245040852913, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036668686868686866, + "loss": 2.5653, + "theoretical_loss": 3.4151245040852913, + "tokens_seen": 2110521344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036666666666666667, + "loss": 2.5589, + "theoretical_loss": 3.415115755553872, + "tokens_seen": 2110586880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666464646464647, + "loss": 2.7479, + "theoretical_loss": 3.4151070073701595, + "tokens_seen": 2110652416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036662626262626264, + "loss": 2.7098, + "theoretical_loss": 3.4150982595341297, + "tokens_seen": 2110717952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666060606060606, + "loss": 2.6864, + "theoretical_loss": 3.415089512045757, + "tokens_seen": 2110783488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658585858585855, + "loss": 2.6965, + "theoretical_loss": 3.415080764905017, + "tokens_seen": 2110849024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656565656565656, + "loss": 2.9099, + "theoretical_loss": 3.415072018111886, + "tokens_seen": 2110914560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665454545454546, + "loss": 2.7096, + "theoretical_loss": 3.4150632716663387, + "tokens_seen": 2110980096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036652525252525253, + "loss": 2.8873, + "theoretical_loss": 3.41505452556835, + "tokens_seen": 2111045632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665050505050505, + "loss": 2.6785, + "theoretical_loss": 3.4150457798178966, + "tokens_seen": 2111111168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664848484848485, + "loss": 2.7008, + "theoretical_loss": 3.4150370344149525, + "tokens_seen": 2111176704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036646464646464646, + "loss": 2.3795, + "theoretical_loss": 3.415028289359494, + "tokens_seen": 2111242240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036644444444444447, + "loss": 2.6121, + "theoretical_loss": 3.4150195446514964, + "tokens_seen": 2111307776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664242424242424, + "loss": 2.7535, + "theoretical_loss": 3.415010800290935, + "tokens_seen": 2111373312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664040404040404, + "loss": 2.7429, + "theoretical_loss": 3.4150020562777854, + "tokens_seen": 2111438848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663838383838384, + "loss": 2.6235, + "theoretical_loss": 3.4149933126120224, + "tokens_seen": 2111504384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663636363636364, + "loss": 2.5654, + "theoretical_loss": 3.4149845692936216, + "tokens_seen": 2111569920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036634343434343436, + "loss": 2.6061, + "theoretical_loss": 3.414975826322559, + "tokens_seen": 2111635456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663232323232323, + "loss": 2.7226, + "theoretical_loss": 3.4149670836988095, + "tokens_seen": 2111700992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630303030303033, + "loss": 2.6416, + "theoretical_loss": 3.4149583414223486, + "tokens_seen": 2111766528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662828282828283, + "loss": 2.7217, + "theoretical_loss": 3.414949599493152, + "tokens_seen": 2111832064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662626262626263, + "loss": 2.7004, + "theoretical_loss": 3.414940857911195, + "tokens_seen": 2111897600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036624242424242425, + "loss": 2.6544, + "theoretical_loss": 3.4149321166764532, + "tokens_seen": 2111963136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662222222222222, + "loss": 2.5048, + "theoretical_loss": 3.4149233757889013, + "tokens_seen": 2112028672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662020202020202, + "loss": 2.6972, + "theoretical_loss": 3.4149146352485156, + "tokens_seen": 2112094208 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 1194755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.566965103149414, + "objective/train/theoretical_loss": 3.414905895055271, + "objective/train/tokens_used": 471018976, + "theoretical_loss": 3.414905895055271, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661818181818182, + "loss": 2.5693, + "theoretical_loss": 3.414905895055271, + "tokens_seen": 2112159744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661616161616162, + "loss": 2.5327, + "theoretical_loss": 3.414897155209143, + "tokens_seen": 2112225280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614141414141415, + "loss": 2.5431, + "theoretical_loss": 3.4148884157101076, + "tokens_seen": 2112290816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036612121212121216, + "loss": 2.8957, + "theoretical_loss": 3.4148796765581397, + "tokens_seen": 2112356352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661010101010101, + "loss": 2.4355, + "theoretical_loss": 3.414870937753215, + "tokens_seen": 2112421888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036608080808080807, + "loss": 2.587, + "theoretical_loss": 3.4148621992953085, + "tokens_seen": 2112487424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660606060606061, + "loss": 2.7336, + "theoretical_loss": 3.4148534611843964, + "tokens_seen": 2112552960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036604040404040404, + "loss": 2.7177, + "theoretical_loss": 3.414844723420453, + "tokens_seen": 2112618496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036602020202020205, + "loss": 2.9251, + "theoretical_loss": 3.414835986003455, + "tokens_seen": 2112684032 + }, + { + "epoch": 0.28, + "learning_rate": 0.000366, + "loss": 2.5538, + "theoretical_loss": 3.4148272489333773, + "tokens_seen": 2112749568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036597979797979796, + "loss": 2.6566, + "theoretical_loss": 3.4148185122101955, + "tokens_seen": 2112815104 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365959595959596, + "loss": 2.7379, + "theoretical_loss": 3.414809775833885, + "tokens_seen": 2112880640 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365939393939394, + "loss": 2.5694, + "theoretical_loss": 3.414801039804421, + "tokens_seen": 2112946176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036591919191919194, + "loss": 2.6709, + "theoretical_loss": 3.4147923041217796, + "tokens_seen": 2113011712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658989898989899, + "loss": 2.6926, + "theoretical_loss": 3.4147835687859356, + "tokens_seen": 2113077248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036587878787878786, + "loss": 2.5997, + "theoretical_loss": 3.414774833796865, + "tokens_seen": 2113142784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585858585858587, + "loss": 2.7505, + "theoretical_loss": 3.414766099154543, + "tokens_seen": 2113208320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658383838383839, + "loss": 2.7581, + "theoretical_loss": 3.414757364858945, + "tokens_seen": 2113273856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036581818181818183, + "loss": 2.4997, + "theoretical_loss": 3.414748630910047, + "tokens_seen": 2113339392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657979797979798, + "loss": 2.6156, + "theoretical_loss": 3.414739897307824, + "tokens_seen": 2113404928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577777777777775, + "loss": 2.6472, + "theoretical_loss": 3.4147311640522515, + "tokens_seen": 2113470464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657575757575758, + "loss": 2.541, + "theoretical_loss": 3.4147224311433053, + "tokens_seen": 2113536000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036573737373737377, + "loss": 2.6583, + "theoretical_loss": 3.41471369858096, + "tokens_seen": 2113601536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036571717171717173, + "loss": 2.4608, + "theoretical_loss": 3.4147049663651927, + "tokens_seen": 2113667072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656969696969697, + "loss": 2.4155, + "theoretical_loss": 3.4146962344959775, + "tokens_seen": 2113732608 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1195575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.671170473098755, + "objective/train/theoretical_loss": 3.4146875029732904, + "objective/train/tokens_used": 472657376, + "theoretical_loss": 3.4146875029732904, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036567676767676764, + "loss": 2.5369, + "theoretical_loss": 3.4146875029732904, + "tokens_seen": 2113798144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656565656565657, + "loss": 2.775, + "theoretical_loss": 3.4146787717971074, + "tokens_seen": 2113863680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563636363636366, + "loss": 2.5493, + "theoretical_loss": 3.414670040967403, + "tokens_seen": 2113929216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656161616161616, + "loss": 2.6625, + "theoretical_loss": 3.4146613104841532, + "tokens_seen": 2113994752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655959595959596, + "loss": 2.4653, + "theoretical_loss": 3.414652580347334, + "tokens_seen": 2114060288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655757575757576, + "loss": 2.5514, + "theoretical_loss": 3.41464385055692, + "tokens_seen": 2114125824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655555555555556, + "loss": 2.6365, + "theoretical_loss": 3.4146351211128874, + "tokens_seen": 2114191360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036553535353535356, + "loss": 2.4774, + "theoretical_loss": 3.4146263920152116, + "tokens_seen": 2114256896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655151515151515, + "loss": 2.6103, + "theoretical_loss": 3.4146176632638676, + "tokens_seen": 2114322432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549494949494947, + "loss": 2.9279, + "theoretical_loss": 3.4146089348588315, + "tokens_seen": 2114387968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654747474747475, + "loss": 2.713, + "theoretical_loss": 3.4146002068000785, + "tokens_seen": 2114453504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654545454545455, + "loss": 2.5182, + "theoretical_loss": 3.4145914790875844, + "tokens_seen": 2114519040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036543434343434345, + "loss": 2.5521, + "theoretical_loss": 3.414582751721325, + "tokens_seen": 2114584576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654141414141414, + "loss": 2.6621, + "theoretical_loss": 3.4145740247012752, + "tokens_seen": 2114650112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653939393939394, + "loss": 2.5783, + "theoretical_loss": 3.414565298027411, + "tokens_seen": 2114715648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653737373737374, + "loss": 2.7291, + "theoretical_loss": 3.4145565716997073, + "tokens_seen": 2114781184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653535353535354, + "loss": 2.4412, + "theoretical_loss": 3.41454784571814, + "tokens_seen": 2114846720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533333333333334, + "loss": 2.6825, + "theoretical_loss": 3.4145391200826856, + "tokens_seen": 2114912256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653131313131313, + "loss": 2.7105, + "theoretical_loss": 3.414530394793318, + "tokens_seen": 2114977792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652929292929293, + "loss": 2.5428, + "theoretical_loss": 3.4145216698500134, + "tokens_seen": 2115043328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527272727272727, + "loss": 2.529, + "theoretical_loss": 3.414512945252748, + "tokens_seen": 2115108864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652525252525253, + "loss": 2.8573, + "theoretical_loss": 3.414504221001496, + "tokens_seen": 2115174400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036523232323232323, + "loss": 2.6882, + "theoretical_loss": 3.414495497096235, + "tokens_seen": 2115239936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036521212121212124, + "loss": 2.6847, + "theoretical_loss": 3.4144867735369386, + "tokens_seen": 2115305472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651919191919192, + "loss": 2.6821, + "theoretical_loss": 3.414478050323583, + "tokens_seen": 2115371008 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1196855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.620561122894287, + "objective/train/theoretical_loss": 3.4144693274561444, + "objective/train/tokens_used": 474295776, + "theoretical_loss": 3.4144693274561444, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517171717171716, + "loss": 2.5429, + "theoretical_loss": 3.4144693274561444, + "tokens_seen": 2115436544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036515151515151517, + "loss": 2.6831, + "theoretical_loss": 3.414460604934597, + "tokens_seen": 2115502080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651313131313131, + "loss": 2.3679, + "theoretical_loss": 3.414451882758918, + "tokens_seen": 2115567616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511111111111114, + "loss": 2.9795, + "theoretical_loss": 3.414443160929082, + "tokens_seen": 2115633152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650909090909091, + "loss": 2.7966, + "theoretical_loss": 3.4144344394450643, + "tokens_seen": 2115698688 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036507070707070705, + "loss": 2.6615, + "theoretical_loss": 3.414425718306841, + "tokens_seen": 2115764224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036505050505050506, + "loss": 2.6021, + "theoretical_loss": 3.4144169975143885, + "tokens_seen": 2115829760 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365030303030303, + "loss": 2.5281, + "theoretical_loss": 3.4144082770676807, + "tokens_seen": 2115895296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036501010101010103, + "loss": 2.7996, + "theoretical_loss": 3.4143995569666936, + "tokens_seen": 2115960832 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364989898989899, + "loss": 2.6269, + "theoretical_loss": 3.4143908372114042, + "tokens_seen": 2116026368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036496969696969694, + "loss": 2.4402, + "theoretical_loss": 3.4143821178017864, + "tokens_seen": 2116091904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494949494949496, + "loss": 2.352, + "theoretical_loss": 3.4143733987378164, + "tokens_seen": 2116157440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036492929292929297, + "loss": 2.5383, + "theoretical_loss": 3.41436468001947, + "tokens_seen": 2116222976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649090909090909, + "loss": 2.586, + "theoretical_loss": 3.4143559616467227, + "tokens_seen": 2116288512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648888888888889, + "loss": 2.6429, + "theoretical_loss": 3.41434724361955, + "tokens_seen": 2116354048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036486868686868684, + "loss": 2.6021, + "theoretical_loss": 3.4143385259379277, + "tokens_seen": 2116419584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484848484848485, + "loss": 2.815, + "theoretical_loss": 3.4143298086018308, + "tokens_seen": 2116485120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036482828282828286, + "loss": 2.679, + "theoretical_loss": 3.4143210916112356, + "tokens_seen": 2116550656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648080808080808, + "loss": 2.519, + "theoretical_loss": 3.4143123749661175, + "tokens_seen": 2116616192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478787878787877, + "loss": 2.68, + "theoretical_loss": 3.414303658666452, + "tokens_seen": 2116681728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476767676767673, + "loss": 2.6001, + "theoretical_loss": 3.414294942712215, + "tokens_seen": 2116747264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647474747474748, + "loss": 2.7058, + "theoretical_loss": 3.414286227103381, + "tokens_seen": 2116812800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036472727272727275, + "loss": 2.6293, + "theoretical_loss": 3.4142775118399276, + "tokens_seen": 2116878336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647070707070707, + "loss": 2.7323, + "theoretical_loss": 3.414268796921829, + "tokens_seen": 2116943872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468686868686867, + "loss": 2.5722, + "theoretical_loss": 3.414260082349061, + "tokens_seen": 2117009408 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1197590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6814472675323486, + "objective/train/theoretical_loss": 3.414251368121599, + "objective/train/tokens_used": 475934176, + "theoretical_loss": 3.414251368121599, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646666666666666, + "loss": 2.6538, + "theoretical_loss": 3.414251368121599, + "tokens_seen": 2117074944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646464646464647, + "loss": 2.7146, + "theoretical_loss": 3.4142426542394197, + "tokens_seen": 2117140480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462626262626264, + "loss": 2.5451, + "theoretical_loss": 3.414233940702498, + "tokens_seen": 2117206016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646060606060606, + "loss": 2.731, + "theoretical_loss": 3.4142252275108094, + "tokens_seen": 2117271552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036458585858585856, + "loss": 2.733, + "theoretical_loss": 3.4142165146643295, + "tokens_seen": 2117337088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645656565656566, + "loss": 2.8315, + "theoretical_loss": 3.4142078021630344, + "tokens_seen": 2117402624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645454545454546, + "loss": 2.8919, + "theoretical_loss": 3.4141990900068997, + "tokens_seen": 2117468160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036452525252525254, + "loss": 2.5921, + "theoretical_loss": 3.4141903781959004, + "tokens_seen": 2117533696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645050505050505, + "loss": 2.7449, + "theoretical_loss": 3.4141816667300127, + "tokens_seen": 2117599232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036448484848484845, + "loss": 2.5988, + "theoretical_loss": 3.414172955609212, + "tokens_seen": 2117664768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644646464646465, + "loss": 2.6461, + "theoretical_loss": 3.414164244833475, + "tokens_seen": 2117730304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036444444444444447, + "loss": 2.5471, + "theoretical_loss": 3.4141555344027754, + "tokens_seen": 2117795840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442424242424243, + "loss": 2.749, + "theoretical_loss": 3.41414682431709, + "tokens_seen": 2117861376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644040404040404, + "loss": 2.677, + "theoretical_loss": 3.414138114576395, + "tokens_seen": 2117926912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643838383838384, + "loss": 2.803, + "theoretical_loss": 3.414129405180665, + "tokens_seen": 2117992448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643636363636364, + "loss": 2.7667, + "theoretical_loss": 3.414120696129876, + "tokens_seen": 2118057984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036434343434343437, + "loss": 2.6965, + "theoretical_loss": 3.4141119874240036, + "tokens_seen": 2118123520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643232323232323, + "loss": 2.8425, + "theoretical_loss": 3.414103279063024, + "tokens_seen": 2118189056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643030303030303, + "loss": 2.6937, + "theoretical_loss": 3.4140945710469124, + "tokens_seen": 2118254592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828282828283, + "loss": 2.6797, + "theoretical_loss": 3.414085863375644, + "tokens_seen": 2118320128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642626262626263, + "loss": 2.7755, + "theoretical_loss": 3.4140771560491956, + "tokens_seen": 2118385664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036424242424242426, + "loss": 2.6452, + "theoretical_loss": 3.4140684490675426, + "tokens_seen": 2118451200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642222222222222, + "loss": 2.7216, + "theoretical_loss": 3.41405974243066, + "tokens_seen": 2118516736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642020202020202, + "loss": 2.825, + "theoretical_loss": 3.414051036138524, + "tokens_seen": 2118582272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641818181818182, + "loss": 2.6486, + "theoretical_loss": 3.41404233019111, + "tokens_seen": 2118647808 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1199039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.485259532928467, + "objective/train/theoretical_loss": 3.4140336245883933, + "objective/train/tokens_used": 477572576, + "theoretical_loss": 3.4140336245883933, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641616161616162, + "loss": 2.4256, + "theoretical_loss": 3.4140336245883933, + "tokens_seen": 2118713344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414141414141415, + "loss": 2.9091, + "theoretical_loss": 3.414024919330351, + "tokens_seen": 2118778880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641212121212121, + "loss": 2.5834, + "theoretical_loss": 3.4140162144169577, + "tokens_seen": 2118844416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641010101010101, + "loss": 2.493, + "theoretical_loss": 3.4140075098481892, + "tokens_seen": 2118909952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640808080808081, + "loss": 2.5896, + "theoretical_loss": 3.4139988056240216, + "tokens_seen": 2118975488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640606060606061, + "loss": 2.6177, + "theoretical_loss": 3.41399010174443, + "tokens_seen": 2119041024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404040404040404, + "loss": 2.6466, + "theoretical_loss": 3.4139813982093905, + "tokens_seen": 2119106560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036402020202020205, + "loss": 2.7423, + "theoretical_loss": 3.413972695018879, + "tokens_seen": 2119172096 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364, + "loss": 2.4383, + "theoretical_loss": 3.413963992172871, + "tokens_seen": 2119237632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397979797979797, + "loss": 2.6734, + "theoretical_loss": 3.4139552896713417, + "tokens_seen": 2119303168 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363959595959596, + "loss": 2.6917, + "theoretical_loss": 3.4139465875142676, + "tokens_seen": 2119368704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393939393939394, + "loss": 2.8761, + "theoretical_loss": 3.413937885701624, + "tokens_seen": 2119434240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036391919191919195, + "loss": 2.6195, + "theoretical_loss": 3.4139291842333863, + "tokens_seen": 2119499776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638989898989899, + "loss": 2.5613, + "theoretical_loss": 3.413920483109531, + "tokens_seen": 2119565312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036387878787878786, + "loss": 2.7517, + "theoretical_loss": 3.413911782330034, + "tokens_seen": 2119630848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036385858585858587, + "loss": 2.6375, + "theoretical_loss": 3.41390308189487, + "tokens_seen": 2119696384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638383838383839, + "loss": 2.5275, + "theoretical_loss": 3.413894381804015, + "tokens_seen": 2119761920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381818181818184, + "loss": 2.7012, + "theoretical_loss": 3.413885682057445, + "tokens_seen": 2119827456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637979797979798, + "loss": 2.7584, + "theoretical_loss": 3.413876982655135, + "tokens_seen": 2119892992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036377777777777775, + "loss": 2.5203, + "theoretical_loss": 3.4138682835970626, + "tokens_seen": 2119958528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036375757575757576, + "loss": 2.606, + "theoretical_loss": 3.413859584883202, + "tokens_seen": 2120024064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637373737373738, + "loss": 2.5048, + "theoretical_loss": 3.413850886513529, + "tokens_seen": 2120089600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036371717171717173, + "loss": 2.5433, + "theoretical_loss": 3.4138421884880197, + "tokens_seen": 2120155136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636969696969697, + "loss": 2.6536, + "theoretical_loss": 3.41383349080665, + "tokens_seen": 2120220672 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367676767676765, + "loss": 2.6621, + "theoretical_loss": 3.413824793469395, + "tokens_seen": 2120286208 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1199714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7499430179595947, + "objective/train/theoretical_loss": 3.4138160964762307, + "objective/train/tokens_used": 479210976, + "theoretical_loss": 3.4138160964762307, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636565656565657, + "loss": 2.5486, + "theoretical_loss": 3.4138160964762307, + "tokens_seen": 2120351744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036363636363636367, + "loss": 2.9565, + "theoretical_loss": 3.4138073998271334, + "tokens_seen": 2120417280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636161616161616, + "loss": 2.5025, + "theoretical_loss": 3.4137987035220787, + "tokens_seen": 2120482816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635959595959596, + "loss": 2.3405, + "theoretical_loss": 3.4137900075610412, + "tokens_seen": 2120548352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357575757575754, + "loss": 2.5424, + "theoretical_loss": 3.413781311943999, + "tokens_seen": 2120613888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635555555555556, + "loss": 2.6686, + "theoretical_loss": 3.413772616670925, + "tokens_seen": 2120679424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036353535353535356, + "loss": 2.5854, + "theoretical_loss": 3.413763921741797, + "tokens_seen": 2120744960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635151515151515, + "loss": 2.7862, + "theoretical_loss": 3.4137552271565905, + "tokens_seen": 2120810496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634949494949495, + "loss": 2.646, + "theoretical_loss": 3.4137465329152805, + "tokens_seen": 2120876032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347474747474743, + "loss": 2.6257, + "theoretical_loss": 3.4137378390178434, + "tokens_seen": 2120941568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634545454545455, + "loss": 2.5818, + "theoretical_loss": 3.413729145464255, + "tokens_seen": 2121007104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343434343434345, + "loss": 2.6205, + "theoretical_loss": 3.4137204522544904, + "tokens_seen": 2121072640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634141414141414, + "loss": 2.6543, + "theoretical_loss": 3.413711759388526, + "tokens_seen": 2121138176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339393939393937, + "loss": 2.6024, + "theoretical_loss": 3.4137030668663373, + "tokens_seen": 2121203712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633737373737374, + "loss": 2.4416, + "theoretical_loss": 3.4136943746879007, + "tokens_seen": 2121269248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633535353535354, + "loss": 2.5592, + "theoretical_loss": 3.413685682853191, + "tokens_seen": 2121334784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036333333333333335, + "loss": 2.3951, + "theoretical_loss": 3.413676991362185, + "tokens_seen": 2121400320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633131313131313, + "loss": 2.5591, + "theoretical_loss": 3.413668300214858, + "tokens_seen": 2121465856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036329292929292926, + "loss": 2.5221, + "theoretical_loss": 3.413659609411185, + "tokens_seen": 2121531392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036327272727272727, + "loss": 2.8375, + "theoretical_loss": 3.4136509189511433, + "tokens_seen": 2121596928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632525252525253, + "loss": 2.5514, + "theoretical_loss": 3.4136422288347075, + "tokens_seen": 2121662464 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323232323232324, + "loss": 2.4459, + "theoretical_loss": 3.4136335390618537, + "tokens_seen": 2121728000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632121212121212, + "loss": 2.4067, + "theoretical_loss": 3.4136248496325585, + "tokens_seen": 2121793536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631919191919192, + "loss": 2.6353, + "theoretical_loss": 3.413616160546797, + "tokens_seen": 2121859072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317171717171716, + "loss": 2.6483, + "theoretical_loss": 3.4136074718045446, + "tokens_seen": 2121924608 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1200493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1076231002807617, + "objective/train/theoretical_loss": 3.413598783405778, + "objective/train/tokens_used": 480849376, + "theoretical_loss": 3.413598783405778, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631515151515152, + "loss": 2.4768, + "theoretical_loss": 3.413598783405778, + "tokens_seen": 2121990144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036313131313131313, + "loss": 2.6516, + "theoretical_loss": 3.4135900953504725, + "tokens_seen": 2122055680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631111111111111, + "loss": 2.5288, + "theoretical_loss": 3.4135814076386044, + "tokens_seen": 2122121216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630909090909091, + "loss": 2.4662, + "theoretical_loss": 3.4135727202701487, + "tokens_seen": 2122186752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307070707070706, + "loss": 2.6228, + "theoretical_loss": 3.4135640332450814, + "tokens_seen": 2122252288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305050505050507, + "loss": 2.7582, + "theoretical_loss": 3.4135553465633794, + "tokens_seen": 2122317824 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363030303030303, + "loss": 2.5916, + "theoretical_loss": 3.413546660225017, + "tokens_seen": 2122383360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301010101010104, + "loss": 2.7162, + "theoretical_loss": 3.413537974229971, + "tokens_seen": 2122448896 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362989898989899, + "loss": 2.7891, + "theoretical_loss": 3.413529288578217, + "tokens_seen": 2122514432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296969696969695, + "loss": 2.8207, + "theoretical_loss": 3.4135206032697307, + "tokens_seen": 2122579968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036294949494949496, + "loss": 2.7081, + "theoretical_loss": 3.413511918304488, + "tokens_seen": 2122645504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629292929292929, + "loss": 2.6875, + "theoretical_loss": 3.413503233682465, + "tokens_seen": 2122711040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036290909090909093, + "loss": 2.5109, + "theoretical_loss": 3.4134945494036373, + "tokens_seen": 2122776576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628888888888889, + "loss": 2.5824, + "theoretical_loss": 3.4134858654679805, + "tokens_seen": 2122842112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686868686869, + "loss": 2.8472, + "theoretical_loss": 3.4134771818754714, + "tokens_seen": 2122907648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284848484848485, + "loss": 2.5033, + "theoretical_loss": 3.4134684986260844, + "tokens_seen": 2122973184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036282828282828286, + "loss": 2.7587, + "theoretical_loss": 3.413459815719796, + "tokens_seen": 2123038720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628080808080808, + "loss": 2.4969, + "theoretical_loss": 3.413451133156583, + "tokens_seen": 2123104256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627878787878788, + "loss": 2.4265, + "theoretical_loss": 3.41344245093642, + "tokens_seen": 2123169792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627676767676768, + "loss": 2.485, + "theoretical_loss": 3.4134337690592833, + "tokens_seen": 2123235328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036274747474747475, + "loss": 2.563, + "theoretical_loss": 3.413425087525149, + "tokens_seen": 2123300864 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272727272727276, + "loss": 2.704, + "theoretical_loss": 3.413416406333992, + "tokens_seen": 2123366400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627070707070707, + "loss": 2.4428, + "theoretical_loss": 3.413407725485789, + "tokens_seen": 2123431936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036268686868686867, + "loss": 2.6576, + "theoretical_loss": 3.4133990449805163, + "tokens_seen": 2123497472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626666666666667, + "loss": 2.7086, + "theoretical_loss": 3.4133903648181487, + "tokens_seen": 2123563008 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1201963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.59928297996521, + "objective/train/theoretical_loss": 3.413381684998663, + "objective/train/tokens_used": 482487776, + "theoretical_loss": 3.413381684998663, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626464646464647, + "loss": 2.4795, + "theoretical_loss": 3.413381684998663, + "tokens_seen": 2123628544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262626262626265, + "loss": 2.7145, + "theoretical_loss": 3.4133730055220344, + "tokens_seen": 2123694080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626060606060606, + "loss": 2.6112, + "theoretical_loss": 3.4133643263882387, + "tokens_seen": 2123759616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258585858585856, + "loss": 2.604, + "theoretical_loss": 3.413355647597253, + "tokens_seen": 2123825152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625656565656566, + "loss": 2.5665, + "theoretical_loss": 3.4133469691490514, + "tokens_seen": 2123890688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625454545454546, + "loss": 2.8125, + "theoretical_loss": 3.413338291043611, + "tokens_seen": 2123956224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252525252525254, + "loss": 2.5088, + "theoretical_loss": 3.4133296132809074, + "tokens_seen": 2124021760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625050505050505, + "loss": 2.5655, + "theoretical_loss": 3.4133209358609164, + "tokens_seen": 2124087296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248484848484846, + "loss": 2.6296, + "theoretical_loss": 3.413312258783614, + "tokens_seen": 2124152832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624646464646465, + "loss": 2.5726, + "theoretical_loss": 3.413303582048976, + "tokens_seen": 2124218368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624444444444445, + "loss": 2.7464, + "theoretical_loss": 3.4132949056569784, + "tokens_seen": 2124283904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036242424242424244, + "loss": 2.6354, + "theoretical_loss": 3.4132862296075968, + "tokens_seen": 2124349440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624040404040404, + "loss": 2.7796, + "theoretical_loss": 3.413277553900808, + "tokens_seen": 2124414976 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036238383838383835, + "loss": 2.6997, + "theoretical_loss": 3.413268878536586, + "tokens_seen": 2124480512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623636363636364, + "loss": 2.7913, + "theoretical_loss": 3.413260203514909, + "tokens_seen": 2124546048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036234343434343437, + "loss": 2.6991, + "theoretical_loss": 3.413251528835751, + "tokens_seen": 2124611584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036232323232323233, + "loss": 2.5728, + "theoretical_loss": 3.4132428544990896, + "tokens_seen": 2124677120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623030303030303, + "loss": 2.6904, + "theoretical_loss": 3.4132341805048996, + "tokens_seen": 2124742656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622828282828283, + "loss": 2.6746, + "theoretical_loss": 3.4132255068531574, + "tokens_seen": 2124808192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622626262626263, + "loss": 2.7979, + "theoretical_loss": 3.413216833543838, + "tokens_seen": 2124873728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036224242424242426, + "loss": 2.3464, + "theoretical_loss": 3.413208160576919, + "tokens_seen": 2124939264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622222222222222, + "loss": 2.7043, + "theoretical_loss": 3.413199487952374, + "tokens_seen": 2125004800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622020202020202, + "loss": 2.6063, + "theoretical_loss": 3.4131908156701813, + "tokens_seen": 2125070336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621818181818182, + "loss": 2.5632, + "theoretical_loss": 3.4131821437303156, + "tokens_seen": 2125135872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621616161616162, + "loss": 2.5457, + "theoretical_loss": 3.4131734721327533, + "tokens_seen": 2125201408 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1202675, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4603917598724365, + "objective/train/theoretical_loss": 3.41316480087747, + "objective/train/tokens_used": 484126176, + "theoretical_loss": 3.41316480087747, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214141414141416, + "loss": 2.5287, + "theoretical_loss": 3.41316480087747, + "tokens_seen": 2125266944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621212121212121, + "loss": 2.5412, + "theoretical_loss": 3.4131561299644417, + "tokens_seen": 2125332480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621010101010101, + "loss": 2.6338, + "theoretical_loss": 3.413147459393644, + "tokens_seen": 2125398016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620808080808081, + "loss": 2.8621, + "theoretical_loss": 3.4131387891650533, + "tokens_seen": 2125463552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620606060606061, + "loss": 2.6417, + "theoretical_loss": 3.4131301192786454, + "tokens_seen": 2125529088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036204040404040405, + "loss": 2.5634, + "theoretical_loss": 3.4131214497343967, + "tokens_seen": 2125594624 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362020202020202, + "loss": 2.3666, + "theoretical_loss": 3.4131127805322823, + "tokens_seen": 2125660160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362, + "loss": 2.3897, + "theoretical_loss": 3.4131041116722787, + "tokens_seen": 2125725696 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361979797979798, + "loss": 2.753, + "theoretical_loss": 3.413095443154362, + "tokens_seen": 2125791232 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361959595959596, + "loss": 2.6322, + "theoretical_loss": 3.4130867749785074, + "tokens_seen": 2125856768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193939393939394, + "loss": 2.5107, + "theoretical_loss": 3.4130781071446914, + "tokens_seen": 2125922304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619191919191919, + "loss": 2.6126, + "theoretical_loss": 3.4130694396528902, + "tokens_seen": 2125987840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618989898989899, + "loss": 2.551, + "theoretical_loss": 3.413060772503079, + "tokens_seen": 2126053376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036187878787878787, + "loss": 2.4052, + "theoretical_loss": 3.4130521056952348, + "tokens_seen": 2126118912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618585858585859, + "loss": 2.5747, + "theoretical_loss": 3.413043439229333, + "tokens_seen": 2126184448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036183838383838383, + "loss": 2.3549, + "theoretical_loss": 3.413034773105349, + "tokens_seen": 2126249984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036181818181818185, + "loss": 2.6027, + "theoretical_loss": 3.41302610732326, + "tokens_seen": 2126315520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617979797979798, + "loss": 2.7541, + "theoretical_loss": 3.4130174418830403, + "tokens_seen": 2126381056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036177777777777776, + "loss": 2.8472, + "theoretical_loss": 3.4130087767846677, + "tokens_seen": 2126446592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036175757575757577, + "loss": 2.6581, + "theoretical_loss": 3.4130001120281173, + "tokens_seen": 2126512128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036173737373737373, + "loss": 2.7177, + "theoretical_loss": 3.412991447613365, + "tokens_seen": 2126577664 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171717171717174, + "loss": 2.6759, + "theoretical_loss": 3.412982783540387, + "tokens_seen": 2126643200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616969696969697, + "loss": 2.3634, + "theoretical_loss": 3.412974119809159, + "tokens_seen": 2126708736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036167676767676765, + "loss": 2.8061, + "theoretical_loss": 3.4129654564196574, + "tokens_seen": 2126774272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165656565656566, + "loss": 2.4021, + "theoretical_loss": 3.4129567933718583, + "tokens_seen": 2126839808 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1203810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.557732582092285, + "objective/train/theoretical_loss": 3.412948130665737, + "objective/train/tokens_used": 485764576, + "theoretical_loss": 3.412948130665737, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616363636363637, + "loss": 2.6991, + "theoretical_loss": 3.412948130665737, + "tokens_seen": 2126905344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161616161616163, + "loss": 2.4659, + "theoretical_loss": 3.4129394683012704, + "tokens_seen": 2126970880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615959595959596, + "loss": 2.7751, + "theoretical_loss": 3.4129308062784336, + "tokens_seen": 2127036416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157575757575754, + "loss": 2.5147, + "theoretical_loss": 3.4129221445972027, + "tokens_seen": 2127101952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036155555555555556, + "loss": 2.6824, + "theoretical_loss": 3.4129134832575545, + "tokens_seen": 2127167488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153535353535357, + "loss": 2.8217, + "theoretical_loss": 3.4129048222594642, + "tokens_seen": 2127233024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615151515151515, + "loss": 2.6615, + "theoretical_loss": 3.4128961616029083, + "tokens_seen": 2127298560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614949494949495, + "loss": 2.6656, + "theoretical_loss": 3.4128875012878623, + "tokens_seen": 2127364096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036147474747474744, + "loss": 2.2907, + "theoretical_loss": 3.4128788413143027, + "tokens_seen": 2127429632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614545454545455, + "loss": 2.5334, + "theoretical_loss": 3.412870181682206, + "tokens_seen": 2127495168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036143434343434346, + "loss": 2.7933, + "theoretical_loss": 3.412861522391547, + "tokens_seen": 2127560704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614141414141414, + "loss": 2.5013, + "theoretical_loss": 3.412852863442302, + "tokens_seen": 2127626240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613939393939394, + "loss": 2.8553, + "theoretical_loss": 3.412844204834448, + "tokens_seen": 2127691776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137373737373733, + "loss": 2.5301, + "theoretical_loss": 3.4128355465679596, + "tokens_seen": 2127757312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613535353535354, + "loss": 2.6969, + "theoretical_loss": 3.412826888642814, + "tokens_seen": 2127822848 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133333333333335, + "loss": 2.4444, + "theoretical_loss": 3.4128182310589867, + "tokens_seen": 2127888384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613131313131313, + "loss": 2.7928, + "theoretical_loss": 3.4128095738164537, + "tokens_seen": 2127953920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129292929292927, + "loss": 2.6422, + "theoretical_loss": 3.4128009169151916, + "tokens_seen": 2128019456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612727272727273, + "loss": 2.6889, + "theoretical_loss": 3.4127922603551752, + "tokens_seen": 2128084992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612525252525253, + "loss": 2.4342, + "theoretical_loss": 3.412783604136382, + "tokens_seen": 2128150528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123232323232324, + "loss": 2.5993, + "theoretical_loss": 3.4127749482587872, + "tokens_seen": 2128216064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612121212121212, + "loss": 2.5035, + "theoretical_loss": 3.412766292722367, + "tokens_seen": 2128281600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036119191919191916, + "loss": 2.5513, + "theoretical_loss": 3.412757637527098, + "tokens_seen": 2128347136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611717171717172, + "loss": 2.6494, + "theoretical_loss": 3.412748982672955, + "tokens_seen": 2128412672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611515151515152, + "loss": 2.6164, + "theoretical_loss": 3.412740328159915, + "tokens_seen": 2128478208 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 1204521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7877657413482666, + "objective/train/theoretical_loss": 3.4127316739879543, + "objective/train/tokens_used": 487402976, + "theoretical_loss": 3.4127316739879543, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036113131313131314, + "loss": 2.5352, + "theoretical_loss": 3.4127316739879543, + "tokens_seen": 2128543744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611111111111111, + "loss": 2.4966, + "theoretical_loss": 3.4127230201570478, + "tokens_seen": 2128609280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610909090909091, + "loss": 2.6737, + "theoretical_loss": 3.4127143666671724, + "tokens_seen": 2128674816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610707070707071, + "loss": 2.6963, + "theoretical_loss": 3.4127057135183043, + "tokens_seen": 2128740352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610505050505051, + "loss": 2.6265, + "theoretical_loss": 3.412697060710419, + "tokens_seen": 2128805888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103030303030303, + "loss": 2.7648, + "theoretical_loss": 3.4126884082434934, + "tokens_seen": 2128871424 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361010101010101, + "loss": 2.6673, + "theoretical_loss": 3.412679756117502, + "tokens_seen": 2128936960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360989898989899, + "loss": 2.6957, + "theoretical_loss": 3.4126711043324227, + "tokens_seen": 2129002496 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360969696969697, + "loss": 2.7246, + "theoretical_loss": 3.4126624528882306, + "tokens_seen": 2129068032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036094949494949497, + "loss": 2.6983, + "theoretical_loss": 3.4126538017849017, + "tokens_seen": 2129133568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609292929292929, + "loss": 2.4946, + "theoretical_loss": 3.4126451510224127, + "tokens_seen": 2129199104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036090909090909093, + "loss": 2.8023, + "theoretical_loss": 3.4126365006007386, + "tokens_seen": 2129264640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608888888888889, + "loss": 2.7118, + "theoretical_loss": 3.412627850519857, + "tokens_seen": 2129330176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608686868686869, + "loss": 2.5587, + "theoretical_loss": 3.412619200779743, + "tokens_seen": 2129395712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084848484848486, + "loss": 2.4035, + "theoretical_loss": 3.4126105513803724, + "tokens_seen": 2129461248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608282828282828, + "loss": 2.7073, + "theoretical_loss": 3.4126019023217222, + "tokens_seen": 2129526784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608080808080808, + "loss": 2.7057, + "theoretical_loss": 3.4125932536037675, + "tokens_seen": 2129592320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607878787878788, + "loss": 2.625, + "theoretical_loss": 3.4125846052264857, + "tokens_seen": 2129657856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607676767676768, + "loss": 2.6595, + "theoretical_loss": 3.4125759571898513, + "tokens_seen": 2129723392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074747474747475, + "loss": 2.7104, + "theoretical_loss": 3.412567309493842, + "tokens_seen": 2129788928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036072727272727276, + "loss": 2.5947, + "theoretical_loss": 3.4125586621384327, + "tokens_seen": 2129854464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607070707070707, + "loss": 2.6358, + "theoretical_loss": 3.4125500151236, + "tokens_seen": 2129920000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606868686868687, + "loss": 2.7788, + "theoretical_loss": 3.41254136844932, + "tokens_seen": 2129985536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606666666666667, + "loss": 2.6488, + "theoretical_loss": 3.412532722115569, + "tokens_seen": 2130051072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036064646464646464, + "loss": 2.606, + "theoretical_loss": 3.412524076122322, + "tokens_seen": 2130116608 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1206012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8228116035461426, + "objective/train/theoretical_loss": 3.412515430469557, + "objective/train/tokens_used": 489041376, + "theoretical_loss": 3.412515430469557, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036062626262626266, + "loss": 2.5498, + "theoretical_loss": 3.412515430469557, + "tokens_seen": 2130182144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606060606060606, + "loss": 2.4808, + "theoretical_loss": 3.412506785157248, + "tokens_seen": 2130247680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058585858585857, + "loss": 2.682, + "theoretical_loss": 3.4124981401853733, + "tokens_seen": 2130313216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605656565656566, + "loss": 2.6653, + "theoretical_loss": 3.412489495553908, + "tokens_seen": 2130378752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605454545454546, + "loss": 2.7006, + "theoretical_loss": 3.4124808512628277, + "tokens_seen": 2130444288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036052525252525255, + "loss": 2.653, + "theoretical_loss": 3.412472207312109, + "tokens_seen": 2130509824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605050505050505, + "loss": 2.5744, + "theoretical_loss": 3.4124635637017278, + "tokens_seen": 2130575360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036048484848484846, + "loss": 2.6906, + "theoretical_loss": 3.4124549204316605, + "tokens_seen": 2130640896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046464646464647, + "loss": 2.6013, + "theoretical_loss": 3.412446277501884, + "tokens_seen": 2130706432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604444444444445, + "loss": 2.5518, + "theoretical_loss": 3.4124376349123726, + "tokens_seen": 2130771968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042424242424244, + "loss": 2.7416, + "theoretical_loss": 3.412428992663104, + "tokens_seen": 2130837504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604040404040404, + "loss": 2.9107, + "theoretical_loss": 3.4124203507540534, + "tokens_seen": 2130903040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038383838383835, + "loss": 2.8989, + "theoretical_loss": 3.4124117091851978, + "tokens_seen": 2130968576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603636363636364, + "loss": 2.7332, + "theoretical_loss": 3.4124030679565127, + "tokens_seen": 2131034112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603434343434344, + "loss": 2.5136, + "theoretical_loss": 3.4123944270679742, + "tokens_seen": 2131099648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036032323232323233, + "loss": 2.6632, + "theoretical_loss": 3.4123857865195593, + "tokens_seen": 2131165184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603030303030303, + "loss": 2.5179, + "theoretical_loss": 3.4123771463112433, + "tokens_seen": 2131230720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028282828282825, + "loss": 2.8013, + "theoretical_loss": 3.412368506443002, + "tokens_seen": 2131296256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602626262626263, + "loss": 2.5537, + "theoretical_loss": 3.412359866914813, + "tokens_seen": 2131361792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036024242424242427, + "loss": 2.6948, + "theoretical_loss": 3.4123512277266514, + "tokens_seen": 2131427328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602222222222222, + "loss": 2.4778, + "theoretical_loss": 3.4123425888784933, + "tokens_seen": 2131492864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602020202020202, + "loss": 2.3264, + "theoretical_loss": 3.4123339503703156, + "tokens_seen": 2131558400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036018181818181814, + "loss": 2.5601, + "theoretical_loss": 3.4123253122020936, + "tokens_seen": 2131623936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601616161616162, + "loss": 2.652, + "theoretical_loss": 3.4123166743738045, + "tokens_seen": 2131689472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036014141414141416, + "loss": 2.4514, + "theoretical_loss": 3.412308036885423, + "tokens_seen": 2131755008 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1206751, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1816353797912598, + "objective/train/theoretical_loss": 3.412299399736927, + "objective/train/tokens_used": 490679776, + "theoretical_loss": 3.412299399736927, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601212121212121, + "loss": 2.8666, + "theoretical_loss": 3.412299399736927, + "tokens_seen": 2131820544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601010101010101, + "loss": 2.5589, + "theoretical_loss": 3.4122907629282913, + "tokens_seen": 2131886080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600808080808081, + "loss": 2.6435, + "theoretical_loss": 3.412282126459493, + "tokens_seen": 2131951616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600606060606061, + "loss": 2.5025, + "theoretical_loss": 3.412273490330507, + "tokens_seen": 2132017152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004040404040405, + "loss": 2.8305, + "theoretical_loss": 3.412264854541311, + "tokens_seen": 2132082688 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360020202020202, + "loss": 2.5632, + "theoretical_loss": 3.412256219091881, + "tokens_seen": 2132148224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 2.5814, + "theoretical_loss": 3.412247583982192, + "tokens_seen": 2132213760 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359979797979798, + "loss": 2.4472, + "theoretical_loss": 3.412238949212221, + "tokens_seen": 2132279296 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359959595959596, + "loss": 2.3147, + "theoretical_loss": 3.4122303147819446, + "tokens_seen": 2132344832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035993939393939395, + "loss": 2.7765, + "theoretical_loss": 3.412221680691338, + "tokens_seen": 2132410368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599191919191919, + "loss": 2.6577, + "theoretical_loss": 3.412213046940378, + "tokens_seen": 2132475904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598989898989899, + "loss": 2.7089, + "theoretical_loss": 3.4122044135290412, + "tokens_seen": 2132541440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987878787878787, + "loss": 2.6851, + "theoretical_loss": 3.412195780457303, + "tokens_seen": 2132606976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598585858585859, + "loss": 2.6608, + "theoretical_loss": 3.41218714772514, + "tokens_seen": 2132672512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035983838383838384, + "loss": 2.6809, + "theoretical_loss": 3.412178515332528, + "tokens_seen": 2132738048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598181818181818, + "loss": 2.6247, + "theoretical_loss": 3.412169883279444, + "tokens_seen": 2132803584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597979797979798, + "loss": 2.5995, + "theoretical_loss": 3.412161251565864, + "tokens_seen": 2132869120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035977777777777776, + "loss": 2.5316, + "theoretical_loss": 3.4121526201917636, + "tokens_seen": 2132934656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597575757575758, + "loss": 2.721, + "theoretical_loss": 3.4121439891571193, + "tokens_seen": 2133000192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973737373737373, + "loss": 2.5383, + "theoretical_loss": 3.4121353584619074, + "tokens_seen": 2133065728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971717171717174, + "loss": 2.4691, + "theoretical_loss": 3.412126728106104, + "tokens_seen": 2133131264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596969696969697, + "loss": 2.5957, + "theoretical_loss": 3.412118098089686, + "tokens_seen": 2133196800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035967676767676766, + "loss": 2.6508, + "theoretical_loss": 3.4121094684126287, + "tokens_seen": 2133262336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965656565656567, + "loss": 2.7296, + "theoretical_loss": 3.412100839074909, + "tokens_seen": 2133327872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596363636363636, + "loss": 2.6245, + "theoretical_loss": 3.412092210076503, + "tokens_seen": 2133393408 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1207876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4870247840881348, + "objective/train/theoretical_loss": 3.412083581417386, + "objective/train/tokens_used": 492318176, + "theoretical_loss": 3.412083581417386, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035961616161616164, + "loss": 2.6386, + "theoretical_loss": 3.412083581417386, + "tokens_seen": 2133458944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595959595959596, + "loss": 2.6662, + "theoretical_loss": 3.412074953097536, + "tokens_seen": 2133524480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035957575757575755, + "loss": 2.6256, + "theoretical_loss": 3.412066325116928, + "tokens_seen": 2133590016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955555555555556, + "loss": 2.7474, + "theoretical_loss": 3.4120576974755386, + "tokens_seen": 2133655552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035953535353535357, + "loss": 2.7641, + "theoretical_loss": 3.4120490701733432, + "tokens_seen": 2133721088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951515151515153, + "loss": 2.3861, + "theoretical_loss": 3.4120404432103193, + "tokens_seen": 2133786624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594949494949495, + "loss": 2.7175, + "theoretical_loss": 3.4120318165864427, + "tokens_seen": 2133852160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594747474747475, + "loss": 2.8013, + "theoretical_loss": 3.41202319030169, + "tokens_seen": 2133917696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035945454545454545, + "loss": 2.6592, + "theoretical_loss": 3.4120145643560367, + "tokens_seen": 2133983232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943434343434347, + "loss": 2.5478, + "theoretical_loss": 3.4120059387494592, + "tokens_seen": 2134048768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594141414141414, + "loss": 2.6662, + "theoretical_loss": 3.411997313481934, + "tokens_seen": 2134114304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593939393939394, + "loss": 2.6417, + "theoretical_loss": 3.411988688553438, + "tokens_seen": 2134179840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593737373737374, + "loss": 2.5264, + "theoretical_loss": 3.411980063963946, + "tokens_seen": 2134245376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593535353535354, + "loss": 2.4546, + "theoretical_loss": 3.4119714397134357, + "tokens_seen": 2134310912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035933333333333336, + "loss": 2.5935, + "theoretical_loss": 3.4119628158018824, + "tokens_seen": 2134376448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593131313131313, + "loss": 2.6412, + "theoretical_loss": 3.411954192229263, + "tokens_seen": 2134441984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929292929292927, + "loss": 2.6314, + "theoretical_loss": 3.411945568995553, + "tokens_seen": 2134507520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592727272727273, + "loss": 2.5289, + "theoretical_loss": 3.411936946100729, + "tokens_seen": 2134573056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592525252525253, + "loss": 2.5673, + "theoretical_loss": 3.4119283235447684, + "tokens_seen": 2134638592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035923232323232325, + "loss": 2.6173, + "theoretical_loss": 3.411919701327646, + "tokens_seen": 2134704128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592121212121212, + "loss": 2.5215, + "theoretical_loss": 3.4119110794493386, + "tokens_seen": 2134769664 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035919191919191916, + "loss": 2.6637, + "theoretical_loss": 3.4119024579098225, + "tokens_seen": 2134835200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035917171717171723, + "loss": 2.671, + "theoretical_loss": 3.4118938367090736, + "tokens_seen": 2134900736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591515151515152, + "loss": 2.7781, + "theoretical_loss": 3.411885215847069, + "tokens_seen": 2134966272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913131313131314, + "loss": 2.5759, + "theoretical_loss": 3.4118765953237844, + "tokens_seen": 2135031808 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1208575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6048333644866943, + "objective/train/theoretical_loss": 3.4118679751391965, + "objective/train/tokens_used": 493956576, + "theoretical_loss": 3.4118679751391965, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591111111111111, + "loss": 2.6079, + "theoretical_loss": 3.4118679751391965, + "tokens_seen": 2135097344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035909090909090906, + "loss": 2.9075, + "theoretical_loss": 3.411859355293281, + "tokens_seen": 2135162880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590707070707071, + "loss": 2.8528, + "theoretical_loss": 3.4118507357860146, + "tokens_seen": 2135228416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590505050505051, + "loss": 2.5697, + "theoretical_loss": 3.4118421166173736, + "tokens_seen": 2135293952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035903030303030304, + "loss": 2.7395, + "theoretical_loss": 3.4118334977873346, + "tokens_seen": 2135359488 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359010101010101, + "loss": 2.7642, + "theoretical_loss": 3.4118248792958736, + "tokens_seen": 2135425024 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358989898989899, + "loss": 2.8019, + "theoretical_loss": 3.4118162611429663, + "tokens_seen": 2135490560 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358969696969697, + "loss": 2.6888, + "theoretical_loss": 3.41180764332859, + "tokens_seen": 2135556096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035894949494949497, + "loss": 2.6679, + "theoretical_loss": 3.41179902585272, + "tokens_seen": 2135621632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892929292929293, + "loss": 2.7335, + "theoretical_loss": 3.411790408715334, + "tokens_seen": 2135687168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589090909090909, + "loss": 2.739, + "theoretical_loss": 3.411781791916407, + "tokens_seen": 2135752704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588888888888889, + "loss": 2.8468, + "theoretical_loss": 3.411773175455916, + "tokens_seen": 2135818240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588686868686869, + "loss": 2.7909, + "theoretical_loss": 3.4117645593338377, + "tokens_seen": 2135883776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035884848484848486, + "loss": 2.6535, + "theoretical_loss": 3.411755943550147, + "tokens_seen": 2135949312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588282828282828, + "loss": 2.6627, + "theoretical_loss": 3.4117473281048216, + "tokens_seen": 2136014848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880808080808083, + "loss": 2.6774, + "theoretical_loss": 3.4117387129978374, + "tokens_seen": 2136080384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587878787878788, + "loss": 2.7263, + "theoretical_loss": 3.4117300982291705, + "tokens_seen": 2136145920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587676767676768, + "loss": 2.4225, + "theoretical_loss": 3.4117214837987975, + "tokens_seen": 2136211456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035874747474747476, + "loss": 2.4609, + "theoretical_loss": 3.4117128697066947, + "tokens_seen": 2136276992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587272727272727, + "loss": 2.6177, + "theoretical_loss": 3.4117042559528388, + "tokens_seen": 2136342528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587070707070707, + "loss": 2.7137, + "theoretical_loss": 3.411695642537205, + "tokens_seen": 2136408064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586868686868687, + "loss": 2.7107, + "theoretical_loss": 3.411687029459771, + "tokens_seen": 2136473600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586666666666667, + "loss": 2.7701, + "theoretical_loss": 3.4116784167205125, + "tokens_seen": 2136539136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864646464646465, + "loss": 2.5995, + "theoretical_loss": 3.4116698043194056, + "tokens_seen": 2136604672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586262626262626, + "loss": 2.7015, + "theoretical_loss": 3.411661192256427, + "tokens_seen": 2136670208 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1209997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9265260696411133, + "objective/train/theoretical_loss": 3.4116525805315527, + "objective/train/tokens_used": 495594976, + "theoretical_loss": 3.4116525805315527, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586060606060606, + "loss": 2.7224, + "theoretical_loss": 3.4116525805315527, + "tokens_seen": 2136735744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585858585858586, + "loss": 2.7168, + "theoretical_loss": 3.4116439691447598, + "tokens_seen": 2136801280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585656565656566, + "loss": 2.7696, + "theoretical_loss": 3.4116353580960244, + "tokens_seen": 2136866816 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035854545454545454, + "loss": 2.6547, + "theoretical_loss": 3.411626747385322, + "tokens_seen": 2136932352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852525252525255, + "loss": 2.4603, + "theoretical_loss": 3.4116181370126304, + "tokens_seen": 2136997888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585050505050505, + "loss": 2.5712, + "theoretical_loss": 3.4116095269779247, + "tokens_seen": 2137063424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848484848484847, + "loss": 2.3471, + "theoretical_loss": 3.4116009172811816, + "tokens_seen": 2137128960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584646464646465, + "loss": 2.6517, + "theoretical_loss": 3.411592307922378, + "tokens_seen": 2137194496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844444444444444, + "loss": 2.5216, + "theoretical_loss": 3.41158369890149, + "tokens_seen": 2137260032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035842424242424245, + "loss": 2.8134, + "theoretical_loss": 3.411575090218494, + "tokens_seen": 2137325568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584040404040404, + "loss": 2.5946, + "theoretical_loss": 3.4115664818733658, + "tokens_seen": 2137391104 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035838383838383836, + "loss": 2.3823, + "theoretical_loss": 3.411557873866083, + "tokens_seen": 2137456640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035836363636363637, + "loss": 2.5521, + "theoretical_loss": 3.4115492661966202, + "tokens_seen": 2137522176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583434343434344, + "loss": 2.5064, + "theoretical_loss": 3.411540658864955, + "tokens_seen": 2137587712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832323232323234, + "loss": 2.6341, + "theoretical_loss": 3.411532051871064, + "tokens_seen": 2137653248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583030303030303, + "loss": 2.5746, + "theoretical_loss": 3.4115234452149235, + "tokens_seen": 2137718784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035828282828282825, + "loss": 2.5296, + "theoretical_loss": 3.411514838896509, + "tokens_seen": 2137784320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826262626262626, + "loss": 2.5941, + "theoretical_loss": 3.411506232915798, + "tokens_seen": 2137849856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582424242424243, + "loss": 2.6183, + "theoretical_loss": 3.411497627272766, + "tokens_seen": 2137915392 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035822222222222223, + "loss": 2.5764, + "theoretical_loss": 3.41148902196739, + "tokens_seen": 2137980928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582020202020202, + "loss": 2.6582, + "theoretical_loss": 3.4114804169996464, + "tokens_seen": 2138046464 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818181818181815, + "loss": 2.8897, + "theoretical_loss": 3.411471812369511, + "tokens_seen": 2138112000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581616161616162, + "loss": 2.8066, + "theoretical_loss": 3.4114632080769605, + "tokens_seen": 2138177536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035814141414141417, + "loss": 2.5446, + "theoretical_loss": 3.4114546041219715, + "tokens_seen": 2138243072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581212121212121, + "loss": 2.5022, + "theoretical_loss": 3.4114460005045206, + "tokens_seen": 2138308608 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1210838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.537412166595459, + "objective/train/theoretical_loss": 3.4114373972245837, + "objective/train/tokens_used": 497233376, + "theoretical_loss": 3.4114373972245837, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581010101010101, + "loss": 2.4891, + "theoretical_loss": 3.4114373972245837, + "tokens_seen": 2138374144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035808080808080804, + "loss": 2.4494, + "theoretical_loss": 3.4114287942821377, + "tokens_seen": 2138439680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580606060606061, + "loss": 2.6337, + "theoretical_loss": 3.4114201916771583, + "tokens_seen": 2138505216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035804040404040406, + "loss": 2.7192, + "theoretical_loss": 3.4114115894096226, + "tokens_seen": 2138570752 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358020202020202, + "loss": 2.6658, + "theoretical_loss": 3.411402987479507, + "tokens_seen": 2138636288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358, + "loss": 2.5798, + "theoretical_loss": 3.4113943858867875, + "tokens_seen": 2138701824 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357979797979798, + "loss": 2.6459, + "theoretical_loss": 3.411385784631441, + "tokens_seen": 2138767360 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357959595959596, + "loss": 2.5023, + "theoretical_loss": 3.4113771837134435, + "tokens_seen": 2138832896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035793939393939395, + "loss": 2.4366, + "theoretical_loss": 3.411368583132772, + "tokens_seen": 2138898432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579191919191919, + "loss": 2.8236, + "theoretical_loss": 3.411359982889402, + "tokens_seen": 2138963968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035789898989898987, + "loss": 2.4003, + "theoretical_loss": 3.4113513829833106, + "tokens_seen": 2139029504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578787878787879, + "loss": 2.5155, + "theoretical_loss": 3.411342783414474, + "tokens_seen": 2139095040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578585858585859, + "loss": 2.558, + "theoretical_loss": 3.411334184182869, + "tokens_seen": 2139160576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783838383838385, + "loss": 2.6443, + "theoretical_loss": 3.411325585288472, + "tokens_seen": 2139226112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578181818181818, + "loss": 2.5738, + "theoretical_loss": 3.411316986731259, + "tokens_seen": 2139291648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577979797979798, + "loss": 2.8293, + "theoretical_loss": 3.411308388511207, + "tokens_seen": 2139357184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777777777777777, + "loss": 2.6874, + "theoretical_loss": 3.4112997906282923, + "tokens_seen": 2139422720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577575757575758, + "loss": 2.7321, + "theoretical_loss": 3.4112911930824907, + "tokens_seen": 2139488256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035773737373737374, + "loss": 2.6719, + "theoretical_loss": 3.4112825958737796, + "tokens_seen": 2139553792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577171717171717, + "loss": 2.5723, + "theoretical_loss": 3.411273999002135, + "tokens_seen": 2139619328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576969696969697, + "loss": 2.7431, + "theoretical_loss": 3.4112654024675333, + "tokens_seen": 2139684864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576767676767677, + "loss": 2.8301, + "theoretical_loss": 3.411256806269951, + "tokens_seen": 2139750400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576565656565657, + "loss": 2.5809, + "theoretical_loss": 3.411248210409364, + "tokens_seen": 2139815936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035763636363636363, + "loss": 2.6114, + "theoretical_loss": 3.4112396148857504, + "tokens_seen": 2139881472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761616161616164, + "loss": 2.553, + "theoretical_loss": 3.411231019699086, + "tokens_seen": 2139947008 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1212386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.683521270751953, + "objective/train/theoretical_loss": 3.411222424849346, + "objective/train/tokens_used": 498871776, + "theoretical_loss": 3.411222424849346, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575959595959596, + "loss": 2.7796, + "theoretical_loss": 3.411222424849346, + "tokens_seen": 2140012544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575757575757576, + "loss": 2.7159, + "theoretical_loss": 3.411213830336508, + "tokens_seen": 2140078080 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035755555555555557, + "loss": 2.6193, + "theoretical_loss": 3.4112052361605487, + "tokens_seen": 2140143616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575353535353535, + "loss": 2.7094, + "theoretical_loss": 3.4111966423214435, + "tokens_seen": 2140209152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035751515151515153, + "loss": 2.8075, + "theoretical_loss": 3.41118804881917, + "tokens_seen": 2140274688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574949494949495, + "loss": 2.8164, + "theoretical_loss": 3.4111794556537047, + "tokens_seen": 2140340224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574747474747475, + "loss": 2.818, + "theoretical_loss": 3.411170862825023, + "tokens_seen": 2140405760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745454545454546, + "loss": 2.474, + "theoretical_loss": 3.4111622703331017, + "tokens_seen": 2140471296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035743434343434347, + "loss": 2.463, + "theoretical_loss": 3.4111536781779184, + "tokens_seen": 2140536832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035741414141414143, + "loss": 2.6987, + "theoretical_loss": 3.4111450863594484, + "tokens_seen": 2140602368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573939393939394, + "loss": 2.7485, + "theoretical_loss": 3.4111364948776686, + "tokens_seen": 2140667904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573737373737374, + "loss": 2.6049, + "theoretical_loss": 3.411127903732556, + "tokens_seen": 2140733440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035735353535353535, + "loss": 2.5969, + "theoretical_loss": 3.411119312924086, + "tokens_seen": 2140798976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035733333333333336, + "loss": 2.6445, + "theoretical_loss": 3.4111107224522357, + "tokens_seen": 2140864512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573131313131313, + "loss": 2.5324, + "theoretical_loss": 3.411102132316982, + "tokens_seen": 2140930048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572929292929293, + "loss": 2.4479, + "theoretical_loss": 3.411093542518301, + "tokens_seen": 2140995584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572727272727273, + "loss": 2.6035, + "theoretical_loss": 3.411084953056169, + "tokens_seen": 2141061120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572525252525253, + "loss": 2.504, + "theoretical_loss": 3.411076363930563, + "tokens_seen": 2141126656 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723232323232326, + "loss": 2.4731, + "theoretical_loss": 3.411067775141459, + "tokens_seen": 2141192192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572121212121212, + "loss": 2.6755, + "theoretical_loss": 3.411059186688834, + "tokens_seen": 2141257728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035719191919191917, + "loss": 2.5443, + "theoretical_loss": 3.411050598572664, + "tokens_seen": 2141323264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571717171717172, + "loss": 2.9018, + "theoretical_loss": 3.4110420107929262, + "tokens_seen": 2141388800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571515151515152, + "loss": 2.6439, + "theoretical_loss": 3.4110334233495965, + "tokens_seen": 2141454336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713131313131315, + "loss": 2.6868, + "theoretical_loss": 3.4110248362426514, + "tokens_seen": 2141519872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571111111111111, + "loss": 2.7333, + "theoretical_loss": 3.4110162494720684, + "tokens_seen": 2141585408 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1213144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.250044345855713, + "objective/train/theoretical_loss": 3.4110076630378225, + "objective/train/tokens_used": 500510176, + "theoretical_loss": 3.4110076630378225, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035709090909090906, + "loss": 2.5157, + "theoretical_loss": 3.4110076630378225, + "tokens_seen": 2141650944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707070707070713, + "loss": 2.7978, + "theoretical_loss": 3.410999076939892, + "tokens_seen": 2141716480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570505050505051, + "loss": 2.6807, + "theoretical_loss": 3.410990491178252, + "tokens_seen": 2141782016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703030303030304, + "loss": 2.703, + "theoretical_loss": 3.4109819057528794, + "tokens_seen": 2141847552 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357010101010101, + "loss": 2.5905, + "theoretical_loss": 3.4109733206637514, + "tokens_seen": 2141913088 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035698989898989896, + "loss": 2.5744, + "theoretical_loss": 3.4109647359108437, + "tokens_seen": 2141978624 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356969696969697, + "loss": 2.6741, + "theoretical_loss": 3.4109561514941333, + "tokens_seen": 2142044160 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356949494949495, + "loss": 2.5922, + "theoretical_loss": 3.4109475674135963, + "tokens_seen": 2142109696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035692929292929293, + "loss": 2.6974, + "theoretical_loss": 3.41093898366921, + "tokens_seen": 2142175232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569090909090909, + "loss": 2.6736, + "theoretical_loss": 3.41093040026095, + "tokens_seen": 2142240768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688888888888885, + "loss": 2.5604, + "theoretical_loss": 3.410921817188794, + "tokens_seen": 2142306304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568686868686869, + "loss": 2.6064, + "theoretical_loss": 3.4109132344527175, + "tokens_seen": 2142371840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035684848484848487, + "loss": 2.7621, + "theoretical_loss": 3.4109046520526975, + "tokens_seen": 2142437376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568282828282828, + "loss": 2.8028, + "theoretical_loss": 3.410896069988711, + "tokens_seen": 2142502912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568080808080808, + "loss": 2.7925, + "theoretical_loss": 3.410887488260734, + "tokens_seen": 2142568448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567878787878788, + "loss": 2.5173, + "theoretical_loss": 3.410878906868743, + "tokens_seen": 2142633984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567676767676768, + "loss": 2.6101, + "theoretical_loss": 3.4108703258127147, + "tokens_seen": 2142699520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674747474747476, + "loss": 2.3914, + "theoretical_loss": 3.4108617450926255, + "tokens_seen": 2142765056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567272727272727, + "loss": 2.8712, + "theoretical_loss": 3.4108531647084526, + "tokens_seen": 2142830592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567070707070707, + "loss": 2.7748, + "theoretical_loss": 3.4108445846601727, + "tokens_seen": 2142896128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566868686868687, + "loss": 2.5546, + "theoretical_loss": 3.410836004947761, + "tokens_seen": 2142961664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566666666666667, + "loss": 2.5219, + "theoretical_loss": 3.4108274255711954, + "tokens_seen": 2143027200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664646464646466, + "loss": 2.5668, + "theoretical_loss": 3.410818846530452, + "tokens_seen": 2143092736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566262626262626, + "loss": 2.6748, + "theoretical_loss": 3.4108102678255072, + "tokens_seen": 2143158272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566060606060606, + "loss": 2.3567, + "theoretical_loss": 3.410801689456338, + "tokens_seen": 2143223808 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1213818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4451510906219482, + "objective/train/theoretical_loss": 3.4107931114229206, + "objective/train/tokens_used": 502148576, + "theoretical_loss": 3.4107931114229206, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565858585858586, + "loss": 2.6069, + "theoretical_loss": 3.4107931114229206, + "tokens_seen": 2143289344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565656565656566, + "loss": 2.7142, + "theoretical_loss": 3.410784533725232, + "tokens_seen": 2143354880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654545454545455, + "loss": 2.7508, + "theoretical_loss": 3.4107759563632487, + "tokens_seen": 2143420416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565252525252525, + "loss": 2.7917, + "theoretical_loss": 3.4107673793369466, + "tokens_seen": 2143485952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565050505050505, + "loss": 2.7656, + "theoretical_loss": 3.4107588026463036, + "tokens_seen": 2143551488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564848484848485, + "loss": 2.6508, + "theoretical_loss": 3.410750226291295, + "tokens_seen": 2143617024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564646464646465, + "loss": 2.7786, + "theoretical_loss": 3.4107416502718984, + "tokens_seen": 2143682560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035644444444444444, + "loss": 2.5019, + "theoretical_loss": 3.41073307458809, + "tokens_seen": 2143748096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035642424242424245, + "loss": 2.6667, + "theoretical_loss": 3.4107244992398464, + "tokens_seen": 2143813632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564040404040404, + "loss": 2.6712, + "theoretical_loss": 3.4107159242271443, + "tokens_seen": 2143879168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638383838383837, + "loss": 2.741, + "theoretical_loss": 3.41070734954996, + "tokens_seen": 2143944704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563636363636364, + "loss": 2.7044, + "theoretical_loss": 3.41069877520827, + "tokens_seen": 2144010240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035634343434343433, + "loss": 2.4767, + "theoretical_loss": 3.410690201202052, + "tokens_seen": 2144075776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035632323232323234, + "loss": 2.6963, + "theoretical_loss": 3.4106816275312815, + "tokens_seen": 2144141312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563030303030303, + "loss": 2.6497, + "theoretical_loss": 3.410673054195936, + "tokens_seen": 2144206848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035628282828282826, + "loss": 2.6883, + "theoretical_loss": 3.410664481195991, + "tokens_seen": 2144272384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035626262626262627, + "loss": 2.5278, + "theoretical_loss": 3.4106559085314236, + "tokens_seen": 2144337920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562424242424243, + "loss": 2.6757, + "theoretical_loss": 3.4106473362022114, + "tokens_seen": 2144403456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622222222222224, + "loss": 2.6744, + "theoretical_loss": 3.41063876420833, + "tokens_seen": 2144468992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562020202020202, + "loss": 2.7731, + "theoretical_loss": 3.410630192549756, + "tokens_seen": 2144534528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035618181818181815, + "loss": 2.8538, + "theoretical_loss": 3.4106216212264666, + "tokens_seen": 2144600064 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616161616161616, + "loss": 2.5068, + "theoretical_loss": 3.4106130502384375, + "tokens_seen": 2144665600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561414141414142, + "loss": 2.6404, + "theoretical_loss": 3.410604479585647, + "tokens_seen": 2144731136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612121212121213, + "loss": 2.48, + "theoretical_loss": 3.4105959092680695, + "tokens_seen": 2144796672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561010101010101, + "loss": 2.5883, + "theoretical_loss": 3.410587339285684, + "tokens_seen": 2144862208 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 1215022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0611629486083984, + "objective/train/theoretical_loss": 3.4105787696384655, + "objective/train/tokens_used": 503786976, + "theoretical_loss": 3.4105787696384655, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035608080808080804, + "loss": 2.538, + "theoretical_loss": 3.4105787696384655, + "tokens_seen": 2144927744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560606060606061, + "loss": 2.6096, + "theoretical_loss": 3.4105702003263914, + "tokens_seen": 2144993280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604040404040407, + "loss": 2.6511, + "theoretical_loss": 3.410561631349438, + "tokens_seen": 2145058816 + }, + { + "epoch": 0.3, + "learning_rate": 0.000356020202020202, + "loss": 2.7878, + "theoretical_loss": 3.410553062707582, + "tokens_seen": 2145124352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000356, + "loss": 2.57, + "theoretical_loss": 3.4105444944008, + "tokens_seen": 2145189888 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355979797979798, + "loss": 2.6632, + "theoretical_loss": 3.4105359264290693, + "tokens_seen": 2145255424 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355959595959596, + "loss": 2.8417, + "theoretical_loss": 3.4105273587923657, + "tokens_seen": 2145320960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035593939393939396, + "loss": 2.6209, + "theoretical_loss": 3.4105187914906665, + "tokens_seen": 2145386496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559191919191919, + "loss": 2.5016, + "theoretical_loss": 3.4105102245239483, + "tokens_seen": 2145452032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035589898989898987, + "loss": 2.5932, + "theoretical_loss": 3.410501657892187, + "tokens_seen": 2145517568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587878787878794, + "loss": 2.4399, + "theoretical_loss": 3.4104930915953604, + "tokens_seen": 2145583104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558585858585859, + "loss": 2.5217, + "theoretical_loss": 3.410484525633444, + "tokens_seen": 2145648640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035583838383838385, + "loss": 2.7665, + "theoretical_loss": 3.4104759600064156, + "tokens_seen": 2145714176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558181818181818, + "loss": 2.8009, + "theoretical_loss": 3.410467394714251, + "tokens_seen": 2145779712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035579797979797977, + "loss": 2.6715, + "theoretical_loss": 3.4104588297569283, + "tokens_seen": 2145845248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035577777777777783, + "loss": 2.4093, + "theoretical_loss": 3.410450265134422, + "tokens_seen": 2145910784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557575757575758, + "loss": 2.6383, + "theoretical_loss": 3.41044170084671, + "tokens_seen": 2145976320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573737373737374, + "loss": 2.514, + "theoretical_loss": 3.4104331368937695, + "tokens_seen": 2146041856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171717171717, + "loss": 2.7435, + "theoretical_loss": 3.4104245732755762, + "tokens_seen": 2146107392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556969696969697, + "loss": 2.6746, + "theoretical_loss": 3.4104160099921077, + "tokens_seen": 2146172928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556767676767677, + "loss": 2.7505, + "theoretical_loss": 3.41040744704334, + "tokens_seen": 2146238464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556565656565657, + "loss": 2.7293, + "theoretical_loss": 3.4103988844292497, + "tokens_seen": 2146304000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563636363636364, + "loss": 2.7244, + "theoretical_loss": 3.410390322149814, + "tokens_seen": 2146369536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556161616161616, + "loss": 2.4535, + "theoretical_loss": 3.4103817602050093, + "tokens_seen": 2146435072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555959595959596, + "loss": 2.8017, + "theoretical_loss": 3.4103731985948125, + "tokens_seen": 2146500608 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1215651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.45157790184021, + "objective/train/theoretical_loss": 3.4103646373192005, + "objective/train/tokens_used": 505425376, + "theoretical_loss": 3.4103646373192005, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555757575757576, + "loss": 2.5237, + "theoretical_loss": 3.4103646373192005, + "tokens_seen": 2146566144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555555555555557, + "loss": 2.8013, + "theoretical_loss": 3.4103560763781493, + "tokens_seen": 2146631680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035553535353535353, + "loss": 2.6799, + "theoretical_loss": 3.4103475157716363, + "tokens_seen": 2146697216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551515151515154, + "loss": 2.8402, + "theoretical_loss": 3.410338955499638, + "tokens_seen": 2146762752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554949494949495, + "loss": 2.8186, + "theoretical_loss": 3.4103303955621307, + "tokens_seen": 2146828288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554747474747475, + "loss": 2.9905, + "theoretical_loss": 3.410321835959092, + "tokens_seen": 2146893824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035545454545454547, + "loss": 2.7196, + "theoretical_loss": 3.4103132766904976, + "tokens_seen": 2146959360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554343434343434, + "loss": 2.3586, + "theoretical_loss": 3.4103047177563255, + "tokens_seen": 2147024896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035541414141414143, + "loss": 2.6222, + "theoretical_loss": 3.410296159156551, + "tokens_seen": 2147090432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553939393939394, + "loss": 2.7194, + "theoretical_loss": 3.410287600891152, + "tokens_seen": 2147155968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553737373737374, + "loss": 2.8596, + "theoretical_loss": 3.410279042960104, + "tokens_seen": 2147221504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535353535353536, + "loss": 2.5819, + "theoretical_loss": 3.4102704853633847, + "tokens_seen": 2147287040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553333333333333, + "loss": 2.4564, + "theoretical_loss": 3.4102619281009714, + "tokens_seen": 2147352576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553131313131313, + "loss": 2.603, + "theoretical_loss": 3.410253371172839, + "tokens_seen": 2147418112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552929292929293, + "loss": 2.7806, + "theoretical_loss": 3.4102448145789657, + "tokens_seen": 2147483648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552727272727273, + "loss": 2.7337, + "theoretical_loss": 3.410236258319328, + "tokens_seen": 2147549184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525252525252525, + "loss": 2.7063, + "theoretical_loss": 3.4102277023939025, + "tokens_seen": 2147614720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035523232323232326, + "loss": 2.5397, + "theoretical_loss": 3.4102191468026657, + "tokens_seen": 2147680256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552121212121212, + "loss": 2.8268, + "theoretical_loss": 3.4102105915455945, + "tokens_seen": 2147745792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551919191919192, + "loss": 2.58, + "theoretical_loss": 3.4102020366226657, + "tokens_seen": 2147811328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551717171717172, + "loss": 2.7294, + "theoretical_loss": 3.410193482033856, + "tokens_seen": 2147876864 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035515151515151514, + "loss": 2.5781, + "theoretical_loss": 3.4101849277791425, + "tokens_seen": 2147942400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513131313131315, + "loss": 2.5377, + "theoretical_loss": 3.4101763738585014, + "tokens_seen": 2148007936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551111111111111, + "loss": 2.5904, + "theoretical_loss": 3.4101678202719103, + "tokens_seen": 2148073472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509090909090907, + "loss": 2.675, + "theoretical_loss": 3.4101592670193446, + "tokens_seen": 2148139008 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1216251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9251840114593506, + "objective/train/theoretical_loss": 3.4101507141007827, + "objective/train/tokens_used": 507063776, + "theoretical_loss": 3.4101507141007827, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550707070707071, + "loss": 2.7425, + "theoretical_loss": 3.4101507141007827, + "tokens_seen": 2148204544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550505050505051, + "loss": 2.7111, + "theoretical_loss": 3.4101421615162, + "tokens_seen": 2148270080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035503030303030305, + "loss": 2.7457, + "theoretical_loss": 3.410133609265574, + "tokens_seen": 2148335616 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355010101010101, + "loss": 2.5556, + "theoretical_loss": 3.410125057348881, + "tokens_seen": 2148401152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498989898989896, + "loss": 2.5046, + "theoretical_loss": 3.4101165057660987, + "tokens_seen": 2148466688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035496969696969697, + "loss": 2.5935, + "theoretical_loss": 3.4101079545172026, + "tokens_seen": 2148532224 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354949494949495, + "loss": 2.7432, + "theoretical_loss": 3.41009940360217, + "tokens_seen": 2148597760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035492929292929294, + "loss": 2.6153, + "theoretical_loss": 3.4100908530209786, + "tokens_seen": 2148663296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549090909090909, + "loss": 2.6582, + "theoretical_loss": 3.4100823027736036, + "tokens_seen": 2148728832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035488888888888885, + "loss": 2.645, + "theoretical_loss": 3.410073752860023, + "tokens_seen": 2148794368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548686868686869, + "loss": 2.7012, + "theoretical_loss": 3.4100652032802135, + "tokens_seen": 2148859904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548484848484849, + "loss": 2.4708, + "theoretical_loss": 3.410056654034151, + "tokens_seen": 2148925440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035482828282828283, + "loss": 2.6244, + "theoretical_loss": 3.4100481051218132, + "tokens_seen": 2148990976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548080808080808, + "loss": 2.4667, + "theoretical_loss": 3.4100395565431763, + "tokens_seen": 2149056512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478787878787875, + "loss": 2.6785, + "theoretical_loss": 3.4100310082982173, + "tokens_seen": 2149122048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547676767676768, + "loss": 2.6815, + "theoretical_loss": 3.410022460386913, + "tokens_seen": 2149187584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035474747474747477, + "loss": 2.4119, + "theoretical_loss": 3.4100139128092404, + "tokens_seen": 2149253120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547272727272727, + "loss": 2.5619, + "theoretical_loss": 3.4100053655651763, + "tokens_seen": 2149318656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547070707070707, + "loss": 2.8614, + "theoretical_loss": 3.409996818654697, + "tokens_seen": 2149384192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546868686868687, + "loss": 2.5432, + "theoretical_loss": 3.4099882720777797, + "tokens_seen": 2149449728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546666666666667, + "loss": 2.3276, + "theoretical_loss": 3.4099797258344013, + "tokens_seen": 2149515264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464646464646466, + "loss": 2.3355, + "theoretical_loss": 3.4099711799245385, + "tokens_seen": 2149580800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546262626262626, + "loss": 2.3147, + "theoretical_loss": 3.4099626343481684, + "tokens_seen": 2149646336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546060606060606, + "loss": 2.6664, + "theoretical_loss": 3.409954089105267, + "tokens_seen": 2149711872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545858585858586, + "loss": 2.6788, + "theoretical_loss": 3.4099455441958124, + "tokens_seen": 2149777408 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1217415, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.860327959060669, + "objective/train/theoretical_loss": 3.40993699961978, + "objective/train/tokens_used": 508702176, + "theoretical_loss": 3.40993699961978, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545656565656566, + "loss": 2.6191, + "theoretical_loss": 3.40993699961978, + "tokens_seen": 2149842944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035454545454545455, + "loss": 2.48, + "theoretical_loss": 3.4099284553771474, + "tokens_seen": 2149908480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545252525252525, + "loss": 2.451, + "theoretical_loss": 3.4099199114678918, + "tokens_seen": 2149974016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545050505050505, + "loss": 2.651, + "theoretical_loss": 3.409911367891989, + "tokens_seen": 2150039552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544848484848485, + "loss": 2.7209, + "theoretical_loss": 3.4099028246494165, + "tokens_seen": 2150105088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544646464646465, + "loss": 2.5717, + "theoretical_loss": 3.409894281740151, + "tokens_seen": 2150170624 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444444444444445, + "loss": 2.6104, + "theoretical_loss": 3.40988573916417, + "tokens_seen": 2150236160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544242424242424, + "loss": 2.7196, + "theoretical_loss": 3.4098771969214487, + "tokens_seen": 2150301696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544040404040404, + "loss": 2.6033, + "theoretical_loss": 3.409868655011966, + "tokens_seen": 2150367232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438383838383837, + "loss": 2.5062, + "theoretical_loss": 3.4098601134356965, + "tokens_seen": 2150432768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543636363636364, + "loss": 2.7681, + "theoretical_loss": 3.4098515721926193, + "tokens_seen": 2150498304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035434343434343434, + "loss": 2.5782, + "theoretical_loss": 3.4098430312827097, + "tokens_seen": 2150563840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432323232323235, + "loss": 2.6382, + "theoretical_loss": 3.4098344907059452, + "tokens_seen": 2150629376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543030303030303, + "loss": 2.6521, + "theoretical_loss": 3.4098259504623023, + "tokens_seen": 2150694912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828282828283, + "loss": 2.5514, + "theoretical_loss": 3.4098174105517582, + "tokens_seen": 2150760448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542626262626263, + "loss": 2.4104, + "theoretical_loss": 3.40980887097429, + "tokens_seen": 2150825984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424242424242423, + "loss": 2.6839, + "theoretical_loss": 3.4098003317298735, + "tokens_seen": 2150891520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422222222222224, + "loss": 2.7487, + "theoretical_loss": 3.4097917928184867, + "tokens_seen": 2150957056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542020202020202, + "loss": 2.3781, + "theoretical_loss": 3.4097832542401063, + "tokens_seen": 2151022592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541818181818182, + "loss": 2.592, + "theoretical_loss": 3.4097747159947085, + "tokens_seen": 2151088128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035416161616161617, + "loss": 2.6144, + "theoretical_loss": 3.40976617808227, + "tokens_seen": 2151153664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541414141414142, + "loss": 2.3999, + "theoretical_loss": 3.409757640502769, + "tokens_seen": 2151219200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035412121212121214, + "loss": 2.663, + "theoretical_loss": 3.409749103256181, + "tokens_seen": 2151284736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541010101010101, + "loss": 2.7124, + "theoretical_loss": 3.4097405663424842, + "tokens_seen": 2151350272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540808080808081, + "loss": 2.6075, + "theoretical_loss": 3.4097320297616545, + "tokens_seen": 2151415808 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1217907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8660171031951904, + "objective/train/theoretical_loss": 3.4097234935136687, + "objective/train/tokens_used": 510340576, + "theoretical_loss": 3.4097234935136687, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406060606060606, + "loss": 2.7922, + "theoretical_loss": 3.4097234935136687, + "tokens_seen": 2151481344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035404040404040407, + "loss": 2.6666, + "theoretical_loss": 3.4097149575985037, + "tokens_seen": 2151546880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402020202020203, + "loss": 2.7161, + "theoretical_loss": 3.4097064220161375, + "tokens_seen": 2151612416 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354, + "loss": 2.7292, + "theoretical_loss": 3.409697886766546, + "tokens_seen": 2151677952 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353979797979798, + "loss": 2.6638, + "theoretical_loss": 3.409689351849706, + "tokens_seen": 2151743488 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353959595959596, + "loss": 2.4856, + "theoretical_loss": 3.409680817265595, + "tokens_seen": 2151809024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035393939393939396, + "loss": 2.6592, + "theoretical_loss": 3.4096722830141895, + "tokens_seen": 2151874560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539191919191919, + "loss": 2.8688, + "theoretical_loss": 3.409663749095466, + "tokens_seen": 2151940096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538989898989899, + "loss": 2.4424, + "theoretical_loss": 3.4096552155094026, + "tokens_seen": 2152005632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538787878787879, + "loss": 2.3609, + "theoretical_loss": 3.409646682255975, + "tokens_seen": 2152071168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538585858585859, + "loss": 2.8115, + "theoretical_loss": 3.409638149335161, + "tokens_seen": 2152136704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383838383838386, + "loss": 2.6421, + "theoretical_loss": 3.4096296167469364, + "tokens_seen": 2152202240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538181818181818, + "loss": 2.6877, + "theoretical_loss": 3.4096210844912793, + "tokens_seen": 2152267776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035379797979797977, + "loss": 2.4426, + "theoretical_loss": 3.409612552568166, + "tokens_seen": 2152333312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035377777777777784, + "loss": 2.4288, + "theoretical_loss": 3.4096040209775733, + "tokens_seen": 2152398848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537575757575758, + "loss": 2.569, + "theoretical_loss": 3.4095954897194787, + "tokens_seen": 2152464384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373737373737375, + "loss": 2.7494, + "theoretical_loss": 3.4095869587938585, + "tokens_seen": 2152529920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537171717171717, + "loss": 2.4857, + "theoretical_loss": 3.40957842820069, + "tokens_seen": 2152595456 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035369696969696966, + "loss": 2.537, + "theoretical_loss": 3.4095698979399502, + "tokens_seen": 2152660992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035367676767676773, + "loss": 2.4235, + "theoretical_loss": 3.4095613680116155, + "tokens_seen": 2152726528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536565656565657, + "loss": 2.6732, + "theoretical_loss": 3.4095528384156633, + "tokens_seen": 2152792064 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035363636363636364, + "loss": 2.5112, + "theoretical_loss": 3.4095443091520705, + "tokens_seen": 2152857600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536161616161616, + "loss": 2.5709, + "theoretical_loss": 3.4095357802208133, + "tokens_seen": 2152923136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035359595959595956, + "loss": 2.7176, + "theoretical_loss": 3.4095272516218698, + "tokens_seen": 2152988672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535757575757576, + "loss": 2.5335, + "theoretical_loss": 3.4095187233552164, + "tokens_seen": 2153054208 + }, + { + "debugging/Self-BLEU-5": 0.4342780019633275, + "debugging/distinct-1-grams": 0.8078002938067941, + "debugging/distinct-2-grams": 0.9689768537714897, + "debugging/entropy-1-grams": 5.9515082084159445, + "debugging/entropy-2-grams": 6.74050317919558, + "debugging/length": 469.15384615384613, + "debugging/num_segments": 13, + "epoch": 0.3, + "objective/train/docs_used": 1219413, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.129000425338745, + "objective/train/theoretical_loss": 3.40951019542083, + "objective/train/tokens_used": 511978976, + "theoretical_loss": 3.40951019542083, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535555555555556, + "loss": 2.7561, + "theoretical_loss": 3.40951019542083, + "tokens_seen": 2153119744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353535353535354, + "loss": 2.8054, + "theoretical_loss": 3.409501667818687, + "tokens_seen": 2153185280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535151515151515, + "loss": 2.8238, + "theoretical_loss": 3.409493140548766, + "tokens_seen": 2153250816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534949494949495, + "loss": 2.6083, + "theoretical_loss": 3.409484613611042, + "tokens_seen": 2153316352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534747474747475, + "loss": 2.4594, + "theoretical_loss": 3.409476087005493, + "tokens_seen": 2153381888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345454545454547, + "loss": 2.643, + "theoretical_loss": 3.4094675607320957, + "tokens_seen": 2153447424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343434343434343, + "loss": 2.7579, + "theoretical_loss": 3.4094590347908276, + "tokens_seen": 2153512960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534141414141414, + "loss": 2.6214, + "theoretical_loss": 3.4094505091816645, + "tokens_seen": 2153578496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533939393939394, + "loss": 2.8583, + "theoretical_loss": 3.4094419839045846, + "tokens_seen": 2153644032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533737373737374, + "loss": 2.7049, + "theoretical_loss": 3.409433458959564, + "tokens_seen": 2153709568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335353535353536, + "loss": 2.4196, + "theoretical_loss": 3.40942493434658, + "tokens_seen": 2153775104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533333333333333, + "loss": 2.7241, + "theoretical_loss": 3.4094164100656092, + "tokens_seen": 2153840640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331313131313133, + "loss": 2.6498, + "theoretical_loss": 3.4094078861166297, + "tokens_seen": 2153906176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532929292929293, + "loss": 2.3319, + "theoretical_loss": 3.4093993624996166, + "tokens_seen": 2153971712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532727272727273, + "loss": 2.4931, + "theoretical_loss": 3.4093908392145487, + "tokens_seen": 2154037248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325252525252526, + "loss": 2.6769, + "theoretical_loss": 3.409382316261402, + "tokens_seen": 2154102784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532323232323232, + "loss": 2.687, + "theoretical_loss": 3.4093737936401536, + "tokens_seen": 2154168320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532121212121212, + "loss": 2.9072, + "theoretical_loss": 3.4093652713507803, + "tokens_seen": 2154233856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531919191919192, + "loss": 2.6326, + "theoretical_loss": 3.4093567493932597, + "tokens_seen": 2154299392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531717171717172, + "loss": 2.7865, + "theoretical_loss": 3.4093482277675684, + "tokens_seen": 2154364928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315151515151515, + "loss": 2.6263, + "theoretical_loss": 3.4093397064736832, + "tokens_seen": 2154430464 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035313131313131316, + "loss": 2.754, + "theoretical_loss": 3.4093311855115815, + "tokens_seen": 2154496000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531111111111111, + "loss": 2.5996, + "theoretical_loss": 3.4093226648812403, + "tokens_seen": 2154561536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530909090909091, + "loss": 2.7044, + "theoretical_loss": 3.409314144582636, + "tokens_seen": 2154627072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530707070707071, + "loss": 2.6458, + "theoretical_loss": 3.4093056246157456, + "tokens_seen": 2154692608 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1220074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.247912645339966, + "objective/train/theoretical_loss": 3.4092971049805474, + "objective/train/tokens_used": 513617376, + "theoretical_loss": 3.4092971049805474, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035305050505050504, + "loss": 2.4702, + "theoretical_loss": 3.4092971049805474, + "tokens_seen": 2154758144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303030303030305, + "loss": 2.7031, + "theoretical_loss": 3.4092885856770168, + "tokens_seen": 2154823680 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353010101010101, + "loss": 2.6677, + "theoretical_loss": 3.4092800667051315, + "tokens_seen": 2154889216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298989898989897, + "loss": 2.6133, + "theoretical_loss": 3.4092715480648685, + "tokens_seen": 2154954752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352969696969697, + "loss": 2.7216, + "theoretical_loss": 3.4092630297562048, + "tokens_seen": 2155020288 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352949494949495, + "loss": 2.7852, + "theoretical_loss": 3.4092545117791175, + "tokens_seen": 2155085824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035292929292929295, + "loss": 2.7652, + "theoretical_loss": 3.4092459941335838, + "tokens_seen": 2155151360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529090909090909, + "loss": 2.5486, + "theoretical_loss": 3.4092374768195794, + "tokens_seen": 2155216896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288888888888886, + "loss": 2.7615, + "theoretical_loss": 3.4092289598370833, + "tokens_seen": 2155282432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286868686868687, + "loss": 2.8573, + "theoretical_loss": 3.4092204431860713, + "tokens_seen": 2155347968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528484848484849, + "loss": 2.5325, + "theoretical_loss": 3.4092119268665204, + "tokens_seen": 2155413504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282828282828284, + "loss": 2.7862, + "theoretical_loss": 3.409203410878408, + "tokens_seen": 2155479040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528080808080808, + "loss": 2.7218, + "theoretical_loss": 3.4091948952217113, + "tokens_seen": 2155544576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035278787878787875, + "loss": 2.4323, + "theoretical_loss": 3.4091863798964064, + "tokens_seen": 2155610112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527676767676768, + "loss": 2.7038, + "theoretical_loss": 3.409177864902471, + "tokens_seen": 2155675648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527474747474748, + "loss": 2.7542, + "theoretical_loss": 3.4091693502398828, + "tokens_seen": 2155741184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035272727272727273, + "loss": 2.5759, + "theoretical_loss": 3.409160835908618, + "tokens_seen": 2155806720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527070707070707, + "loss": 2.6184, + "theoretical_loss": 3.4091523219086532, + "tokens_seen": 2155872256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035268686868686864, + "loss": 2.6927, + "theoretical_loss": 3.4091438082399668, + "tokens_seen": 2155937792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526666666666667, + "loss": 2.677, + "theoretical_loss": 3.4091352949025344, + "tokens_seen": 2156003328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035264646464646467, + "loss": 2.4223, + "theoretical_loss": 3.4091267818963336, + "tokens_seen": 2156068864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526262626262626, + "loss": 2.6638, + "theoretical_loss": 3.4091182692213415, + "tokens_seen": 2156134400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526060606060606, + "loss": 2.8622, + "theoretical_loss": 3.4091097568775357, + "tokens_seen": 2156199936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258585858585865, + "loss": 2.547, + "theoretical_loss": 3.4091012448648925, + "tokens_seen": 2156265472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525656565656566, + "loss": 2.6284, + "theoretical_loss": 3.409092733183389, + "tokens_seen": 2156331008 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1221515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.843029022216797, + "objective/train/theoretical_loss": 3.4090842218330026, + "objective/train/tokens_used": 515255776, + "theoretical_loss": 3.4090842218330026, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254545454545456, + "loss": 2.8233, + "theoretical_loss": 3.4090842218330026, + "tokens_seen": 2156396544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525252525252525, + "loss": 2.8085, + "theoretical_loss": 3.40907571081371, + "tokens_seen": 2156462080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525050505050505, + "loss": 2.8961, + "theoretical_loss": 3.409067200125489, + "tokens_seen": 2156527616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035248484848484854, + "loss": 2.6829, + "theoretical_loss": 3.4090586897683153, + "tokens_seen": 2156593152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524646464646465, + "loss": 2.5913, + "theoretical_loss": 3.4090501797421675, + "tokens_seen": 2156658688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244444444444445, + "loss": 2.6793, + "theoretical_loss": 3.4090416700470216, + "tokens_seen": 2156724224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524242424242424, + "loss": 2.4942, + "theoretical_loss": 3.409033160682855, + "tokens_seen": 2156789760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524040404040404, + "loss": 2.5517, + "theoretical_loss": 3.409024651649645, + "tokens_seen": 2156855296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238383838383843, + "loss": 2.5843, + "theoretical_loss": 3.409016142947368, + "tokens_seen": 2156920832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523636363636364, + "loss": 2.5624, + "theoretical_loss": 3.409007634576002, + "tokens_seen": 2156986368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234343434343434, + "loss": 2.5168, + "theoretical_loss": 3.408999126535523, + "tokens_seen": 2157051904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523232323232323, + "loss": 2.7251, + "theoretical_loss": 3.4089906188259094, + "tokens_seen": 2157117440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523030303030303, + "loss": 2.7725, + "theoretical_loss": 3.408982111447137, + "tokens_seen": 2157182976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522828282828283, + "loss": 2.5831, + "theoretical_loss": 3.4089736043991836, + "tokens_seen": 2157248512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522626262626263, + "loss": 2.6272, + "theoretical_loss": 3.4089650976820263, + "tokens_seen": 2157314048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035224242424242424, + "loss": 2.7402, + "theoretical_loss": 3.4089565912956417, + "tokens_seen": 2157379584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035222222222222225, + "loss": 2.356, + "theoretical_loss": 3.4089480852400076, + "tokens_seen": 2157445120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522020202020202, + "loss": 2.6436, + "theoretical_loss": 3.408939579515101, + "tokens_seen": 2157510656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521818181818182, + "loss": 2.5794, + "theoretical_loss": 3.408931074120898, + "tokens_seen": 2157576192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521616161616162, + "loss": 2.709, + "theoretical_loss": 3.4089225690573763, + "tokens_seen": 2157641728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035214141414141413, + "loss": 2.5785, + "theoretical_loss": 3.4089140643245135, + "tokens_seen": 2157707264 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212121212121214, + "loss": 2.6056, + "theoretical_loss": 3.408905559922286, + "tokens_seen": 2157772800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521010101010101, + "loss": 2.6523, + "theoretical_loss": 3.4088970558506717, + "tokens_seen": 2157838336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520808080808081, + "loss": 2.6143, + "theoretical_loss": 3.408888552109647, + "tokens_seen": 2157903872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035206060606060607, + "loss": 2.6492, + "theoretical_loss": 3.408880048699189, + "tokens_seen": 2157969408 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1222265, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4333038330078125, + "objective/train/theoretical_loss": 3.4088715456192755, + "objective/train/tokens_used": 516894176, + "theoretical_loss": 3.4088715456192755, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352040404040404, + "loss": 2.5699, + "theoretical_loss": 3.4088715456192755, + "tokens_seen": 2158034944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035202020202020203, + "loss": 2.7544, + "theoretical_loss": 3.4088630428698825, + "tokens_seen": 2158100480 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352, + "loss": 2.7167, + "theoretical_loss": 3.4088545404509882, + "tokens_seen": 2158166016 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351979797979798, + "loss": 2.5931, + "theoretical_loss": 3.408846038362569, + "tokens_seen": 2158231552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195959595959596, + "loss": 2.7466, + "theoretical_loss": 3.4088375366046026, + "tokens_seen": 2158297088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193939393939397, + "loss": 2.804, + "theoretical_loss": 3.4088290351770656, + "tokens_seen": 2158362624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519191919191919, + "loss": 2.8199, + "theoretical_loss": 3.4088205340799353, + "tokens_seen": 2158428160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518989898989899, + "loss": 2.755, + "theoretical_loss": 3.4088120333131893, + "tokens_seen": 2158493696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518787878787879, + "loss": 2.6693, + "theoretical_loss": 3.408803532876804, + "tokens_seen": 2158559232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185858585858585, + "loss": 2.7391, + "theoretical_loss": 3.408795032770757, + "tokens_seen": 2158624768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035183838383838386, + "loss": 2.6913, + "theoretical_loss": 3.408786532995025, + "tokens_seen": 2158690304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518181818181818, + "loss": 2.6891, + "theoretical_loss": 3.408778033549585, + "tokens_seen": 2158755840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517979797979798, + "loss": 2.4428, + "theoretical_loss": 3.4087695344344153, + "tokens_seen": 2158821376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517777777777778, + "loss": 2.5476, + "theoretical_loss": 3.4087610356494924, + "tokens_seen": 2158886912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517575757575758, + "loss": 2.5615, + "theoretical_loss": 3.408752537194793, + "tokens_seen": 2158952448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035173737373737376, + "loss": 2.467, + "theoretical_loss": 3.408744039070294, + "tokens_seen": 2159017984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517171717171717, + "loss": 2.5136, + "theoretical_loss": 3.408735541275974, + "tokens_seen": 2159083520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035169696969696967, + "loss": 2.6918, + "theoretical_loss": 3.408727043811809, + "tokens_seen": 2159149056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516767676767677, + "loss": 2.8207, + "theoretical_loss": 3.4087185466777763, + "tokens_seen": 2159214592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516565656565657, + "loss": 2.5216, + "theoretical_loss": 3.4087100498738536, + "tokens_seen": 2159280128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163636363636365, + "loss": 2.6291, + "theoretical_loss": 3.408701553400017, + "tokens_seen": 2159345664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516161616161616, + "loss": 2.6409, + "theoretical_loss": 3.4086930572562446, + "tokens_seen": 2159411200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159595959595956, + "loss": 2.6529, + "theoretical_loss": 3.4086845614425134, + "tokens_seen": 2159476736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515757575757576, + "loss": 2.6122, + "theoretical_loss": 3.4086760659588, + "tokens_seen": 2159542272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515555555555556, + "loss": 2.896, + "theoretical_loss": 3.408667570805082, + "tokens_seen": 2159607808 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1223379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8840444087982178, + "objective/train/theoretical_loss": 3.408659075981337, + "objective/train/tokens_used": 518532576, + "theoretical_loss": 3.408659075981337, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035153535353535354, + "loss": 2.7312, + "theoretical_loss": 3.408659075981337, + "tokens_seen": 2159673344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515151515151515, + "loss": 2.6288, + "theoretical_loss": 3.4086505814875414, + "tokens_seen": 2159738880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035149494949494945, + "loss": 2.9764, + "theoretical_loss": 3.408642087323673, + "tokens_seen": 2159804416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514747474747475, + "loss": 2.8438, + "theoretical_loss": 3.4086335934897085, + "tokens_seen": 2159869952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514545454545455, + "loss": 2.6324, + "theoretical_loss": 3.4086250999856254, + "tokens_seen": 2159935488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035143434343434343, + "loss": 2.7814, + "theoretical_loss": 3.4086166068114006, + "tokens_seen": 2160001024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514141414141414, + "loss": 2.705, + "theoretical_loss": 3.408608113967011, + "tokens_seen": 2160066560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513939393939394, + "loss": 2.8045, + "theoretical_loss": 3.408599621452435, + "tokens_seen": 2160132096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513737373737374, + "loss": 2.7567, + "theoretical_loss": 3.4085911292676485, + "tokens_seen": 2160197632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035135353535353537, + "loss": 2.8083, + "theoretical_loss": 3.4085826374126293, + "tokens_seen": 2160263168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513333333333333, + "loss": 2.6017, + "theoretical_loss": 3.4085741458873544, + "tokens_seen": 2160328704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513131313131313, + "loss": 2.4176, + "theoretical_loss": 3.408565654691801, + "tokens_seen": 2160394240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512929292929293, + "loss": 2.6582, + "theoretical_loss": 3.408557163825946, + "tokens_seen": 2160459776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512727272727273, + "loss": 2.3701, + "theoretical_loss": 3.4085486732897676, + "tokens_seen": 2160525312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125252525252526, + "loss": 2.5943, + "theoretical_loss": 3.4085401830832422, + "tokens_seen": 2160590848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512323232323232, + "loss": 2.5969, + "theoretical_loss": 3.408531693206347, + "tokens_seen": 2160656384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035121212121212123, + "loss": 2.5634, + "theoretical_loss": 3.408523203659059, + "tokens_seen": 2160721920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511919191919192, + "loss": 2.6219, + "theoretical_loss": 3.4085147144413566, + "tokens_seen": 2160787456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511717171717172, + "loss": 2.221, + "theoretical_loss": 3.4085062255532157, + "tokens_seen": 2160852992 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035115151515151515, + "loss": 2.5874, + "theoretical_loss": 3.408497736994614, + "tokens_seen": 2160918528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511313131313131, + "loss": 2.6295, + "theoretical_loss": 3.4084892487655285, + "tokens_seen": 2160984064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511111111111111, + "loss": 2.7577, + "theoretical_loss": 3.408480760865937, + "tokens_seen": 2161049600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510909090909091, + "loss": 2.7137, + "theoretical_loss": 3.408472273295816, + "tokens_seen": 2161115136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510707070707071, + "loss": 2.7181, + "theoretical_loss": 3.408463786055143, + "tokens_seen": 2161180672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035105050505050505, + "loss": 2.5587, + "theoretical_loss": 3.408455299143896, + "tokens_seen": 2161246208 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 1223894, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.656575918197632, + "objective/train/theoretical_loss": 3.408446812562051, + "objective/train/tokens_used": 520170976, + "theoretical_loss": 3.408446812562051, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035103030303030306, + "loss": 2.702, + "theoretical_loss": 3.408446812562051, + "tokens_seen": 2161311744 + }, + { + "epoch": 0.31, + "learning_rate": 0.000351010101010101, + "loss": 2.8032, + "theoretical_loss": 3.4084383263095854, + "tokens_seen": 2161377280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098989898989897, + "loss": 2.6752, + "theoretical_loss": 3.4084298403864772, + "tokens_seen": 2161442816 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350969696969697, + "loss": 2.3753, + "theoretical_loss": 3.408421354792703, + "tokens_seen": 2161508352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035094949494949494, + "loss": 2.712, + "theoretical_loss": 3.4084128695282403, + "tokens_seen": 2161573888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035092929292929295, + "loss": 2.6567, + "theoretical_loss": 3.408404384593066, + "tokens_seen": 2161639424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509090909090909, + "loss": 2.7366, + "theoretical_loss": 3.4083958999871578, + "tokens_seen": 2161704960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035088888888888886, + "loss": 2.6625, + "theoretical_loss": 3.408387415710493, + "tokens_seen": 2161770496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508686868686869, + "loss": 2.6209, + "theoretical_loss": 3.408378931763048, + "tokens_seen": 2161836032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508484848484849, + "loss": 2.5494, + "theoretical_loss": 3.4083704481448014, + "tokens_seen": 2161901568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082828282828284, + "loss": 2.3266, + "theoretical_loss": 3.408361964855729, + "tokens_seen": 2161967104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508080808080808, + "loss": 2.7123, + "theoretical_loss": 3.408353481895809, + "tokens_seen": 2162032640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507878787878788, + "loss": 2.5163, + "theoretical_loss": 3.4083449992650183, + "tokens_seen": 2162098176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035076767676767677, + "loss": 2.6918, + "theoretical_loss": 3.408336516963334, + "tokens_seen": 2162163712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507474747474748, + "loss": 2.4214, + "theoretical_loss": 3.408328034990734, + "tokens_seen": 2162229248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035072727272727274, + "loss": 2.6653, + "theoretical_loss": 3.4083195533471953, + "tokens_seen": 2162294784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507070707070707, + "loss": 2.8949, + "theoretical_loss": 3.4083110720326943, + "tokens_seen": 2162360320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506868686868687, + "loss": 2.8116, + "theoretical_loss": 3.4083025910472093, + "tokens_seen": 2162425856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506666666666667, + "loss": 2.7501, + "theoretical_loss": 3.4082941103907176, + "tokens_seen": 2162491392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035064646464646467, + "loss": 2.5605, + "theoretical_loss": 3.4082856300631956, + "tokens_seen": 2162556928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035062626262626263, + "loss": 2.6296, + "theoretical_loss": 3.408277150064621, + "tokens_seen": 2162622464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506060606060606, + "loss": 2.6312, + "theoretical_loss": 3.4082686703949716, + "tokens_seen": 2162688000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505858585858586, + "loss": 2.5903, + "theoretical_loss": 3.4082601910542243, + "tokens_seen": 2162753536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505656565656566, + "loss": 2.5917, + "theoretical_loss": 3.4082517120423557, + "tokens_seen": 2162819072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035054545454545457, + "loss": 2.6061, + "theoretical_loss": 3.408243233359344, + "tokens_seen": 2162884608 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1225280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.806894302368164, + "objective/train/theoretical_loss": 3.408234755005166, + "objective/train/tokens_used": 521809376, + "theoretical_loss": 3.408234755005166, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505252525252525, + "loss": 2.731, + "theoretical_loss": 3.408234755005166, + "tokens_seen": 2162950144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505050505050505, + "loss": 2.8693, + "theoretical_loss": 3.4082262769797995, + "tokens_seen": 2163015680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035048484848484854, + "loss": 2.5909, + "theoretical_loss": 3.4082177992832214, + "tokens_seen": 2163081216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504646464646465, + "loss": 2.5865, + "theoretical_loss": 3.408209321915409, + "tokens_seen": 2163146752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035044444444444446, + "loss": 2.2497, + "theoretical_loss": 3.4082008448763395, + "tokens_seen": 2163212288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504242424242424, + "loss": 2.7407, + "theoretical_loss": 3.40819236816599, + "tokens_seen": 2163277824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035040404040404037, + "loss": 2.8789, + "theoretical_loss": 3.408183891784338, + "tokens_seen": 2163343360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035038383838383844, + "loss": 2.6934, + "theoretical_loss": 3.4081754157313617, + "tokens_seen": 2163408896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503636363636364, + "loss": 2.6123, + "theoretical_loss": 3.408166940007037, + "tokens_seen": 2163474432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035034343434343435, + "loss": 2.8476, + "theoretical_loss": 3.408158464611342, + "tokens_seen": 2163539968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503232323232323, + "loss": 2.7536, + "theoretical_loss": 3.4081499895442535, + "tokens_seen": 2163605504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035030303030303026, + "loss": 2.6602, + "theoretical_loss": 3.4081415148057497, + "tokens_seen": 2163671040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035028282828282833, + "loss": 2.7378, + "theoretical_loss": 3.4081330403958066, + "tokens_seen": 2163736576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502626262626263, + "loss": 2.5273, + "theoretical_loss": 3.4081245663144024, + "tokens_seen": 2163802112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035024242424242424, + "loss": 2.7561, + "theoretical_loss": 3.4081160925615146, + "tokens_seen": 2163867648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502222222222222, + "loss": 2.598, + "theoretical_loss": 3.40810761913712, + "tokens_seen": 2163933184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502020202020202, + "loss": 2.6009, + "theoretical_loss": 3.408099146041196, + "tokens_seen": 2163998720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501818181818182, + "loss": 2.7231, + "theoretical_loss": 3.40809067327372, + "tokens_seen": 2164064256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501616161616162, + "loss": 2.6707, + "theoretical_loss": 3.408082200834669, + "tokens_seen": 2164129792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014141414141414, + "loss": 2.6984, + "theoretical_loss": 3.408073728724021, + "tokens_seen": 2164195328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501212121212121, + "loss": 2.8935, + "theoretical_loss": 3.408065256941753, + "tokens_seen": 2164260864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501010101010101, + "loss": 2.6666, + "theoretical_loss": 3.408056785487842, + "tokens_seen": 2164326400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500808080808081, + "loss": 2.5386, + "theoretical_loss": 3.408048314362266, + "tokens_seen": 2164391936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035006060606060607, + "loss": 2.4911, + "theoretical_loss": 3.408039843565002, + "tokens_seen": 2164457472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035004040404040403, + "loss": 2.6329, + "theoretical_loss": 3.4080313730960268, + "tokens_seen": 2164523008 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1225734, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889841079711914, + "objective/train/theoretical_loss": 3.4080229029553184, + "objective/train/tokens_used": 523447776, + "theoretical_loss": 3.4080229029553184, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035002020202020204, + "loss": 2.8489, + "theoretical_loss": 3.4080229029553184, + "tokens_seen": 2164588544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035, + "loss": 2.7032, + "theoretical_loss": 3.408014433142854, + "tokens_seen": 2164654080 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349979797979798, + "loss": 2.6675, + "theoretical_loss": 3.408005963658611, + "tokens_seen": 2164719616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034995959595959596, + "loss": 2.8904, + "theoretical_loss": 3.407997494502567, + "tokens_seen": 2164785152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499393939393939, + "loss": 2.6968, + "theoretical_loss": 3.4079890256746985, + "tokens_seen": 2164850688 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034991919191919193, + "loss": 2.8173, + "theoretical_loss": 3.4079805571749833, + "tokens_seen": 2164916224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498989898989899, + "loss": 2.6545, + "theoretical_loss": 3.4079720890033993, + "tokens_seen": 2164981760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498787878787879, + "loss": 2.7566, + "theoretical_loss": 3.4079636211599227, + "tokens_seen": 2165047296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034985858585858586, + "loss": 2.7767, + "theoretical_loss": 3.407955153644532, + "tokens_seen": 2165112832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034983838383838387, + "loss": 2.4388, + "theoretical_loss": 3.407946686457204, + "tokens_seen": 2165178368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498181818181818, + "loss": 2.7338, + "theoretical_loss": 3.407938219597916, + "tokens_seen": 2165243904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497979797979798, + "loss": 2.7086, + "theoretical_loss": 3.4079297530666457, + "tokens_seen": 2165309440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497777777777778, + "loss": 2.5206, + "theoretical_loss": 3.40792128686337, + "tokens_seen": 2165374976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034975757575757575, + "loss": 2.7483, + "theoretical_loss": 3.4079128209880665, + "tokens_seen": 2165440512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034973737373737376, + "loss": 2.7345, + "theoretical_loss": 3.4079043554407127, + "tokens_seen": 2165506048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497171717171717, + "loss": 2.7334, + "theoretical_loss": 3.407895890221286, + "tokens_seen": 2165571584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496969696969697, + "loss": 2.4887, + "theoretical_loss": 3.4078874253297635, + "tokens_seen": 2165637120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496767676767677, + "loss": 2.5446, + "theoretical_loss": 3.407878960766123, + "tokens_seen": 2165702656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496565656565657, + "loss": 2.7687, + "theoretical_loss": 3.407870496530341, + "tokens_seen": 2165768192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034963636363636365, + "loss": 2.7172, + "theoretical_loss": 3.4078620326223956, + "tokens_seen": 2165833728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496161616161616, + "loss": 2.5769, + "theoretical_loss": 3.407853569042264, + "tokens_seen": 2165899264 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034959595959595957, + "loss": 2.5433, + "theoretical_loss": 3.4078451057899244, + "tokens_seen": 2165964800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495757575757576, + "loss": 2.7322, + "theoretical_loss": 3.4078366428653526, + "tokens_seen": 2166030336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495555555555556, + "loss": 2.6148, + "theoretical_loss": 3.4078281802685275, + "tokens_seen": 2166095872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034953535353535355, + "loss": 2.8979, + "theoretical_loss": 3.407819717999425, + "tokens_seen": 2166161408 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1227178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.682830810546875, + "objective/train/theoretical_loss": 3.4078112560580243, + "objective/train/tokens_used": 525086176, + "theoretical_loss": 3.4078112560580243, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495151515151515, + "loss": 2.6355, + "theoretical_loss": 3.4078112560580243, + "tokens_seen": 2166226944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034949494949494946, + "loss": 2.6253, + "theoretical_loss": 3.4078027944443012, + "tokens_seen": 2166292480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494747474747475, + "loss": 2.5417, + "theoretical_loss": 3.407794333158234, + "tokens_seen": 2166358016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494545454545455, + "loss": 2.7283, + "theoretical_loss": 3.407785872199799, + "tokens_seen": 2166423552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034943434343434344, + "loss": 2.5577, + "theoretical_loss": 3.407777411568975, + "tokens_seen": 2166489088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494141414141414, + "loss": 2.4701, + "theoretical_loss": 3.407768951265739, + "tokens_seen": 2166554624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034939393939393935, + "loss": 2.7197, + "theoretical_loss": 3.407760491290068, + "tokens_seen": 2166620160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493737373737374, + "loss": 2.7329, + "theoretical_loss": 3.4077520316419396, + "tokens_seen": 2166685696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493535353535354, + "loss": 2.5617, + "theoretical_loss": 3.4077435723213316, + "tokens_seen": 2166751232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034933333333333333, + "loss": 2.8196, + "theoretical_loss": 3.4077351133282203, + "tokens_seen": 2166816768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493131313131313, + "loss": 2.9106, + "theoretical_loss": 3.407726654662585, + "tokens_seen": 2166882304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492929292929293, + "loss": 2.8878, + "theoretical_loss": 3.4077181963244008, + "tokens_seen": 2166947840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492727272727273, + "loss": 2.6179, + "theoretical_loss": 3.4077097383136468, + "tokens_seen": 2167013376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034925252525252527, + "loss": 2.6551, + "theoretical_loss": 3.4077012806302998, + "tokens_seen": 2167078912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492323232323232, + "loss": 2.7653, + "theoretical_loss": 3.4076928232743375, + "tokens_seen": 2167144448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492121212121212, + "loss": 2.6426, + "theoretical_loss": 3.4076843662457375, + "tokens_seen": 2167209984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491919191919192, + "loss": 2.4127, + "theoretical_loss": 3.4076759095444764, + "tokens_seen": 2167275520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491717171717172, + "loss": 2.6452, + "theoretical_loss": 3.407667453170532, + "tokens_seen": 2167341056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034915151515151516, + "loss": 2.7812, + "theoretical_loss": 3.4076589971238826, + "tokens_seen": 2167406592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491313131313131, + "loss": 2.8215, + "theoretical_loss": 3.4076505414045046, + "tokens_seen": 2167472128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034911111111111113, + "loss": 2.7322, + "theoretical_loss": 3.4076420860123755, + "tokens_seen": 2167537664 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034909090909090914, + "loss": 2.6761, + "theoretical_loss": 3.4076336309474735, + "tokens_seen": 2167603200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490707070707071, + "loss": 2.5113, + "theoretical_loss": 3.407625176209775, + "tokens_seen": 2167668736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034905050505050505, + "loss": 2.6113, + "theoretical_loss": 3.407616721799258, + "tokens_seen": 2167734272 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349030303030303, + "loss": 2.4725, + "theoretical_loss": 3.4076082677159003, + "tokens_seen": 2167799808 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1227923, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3699498176574707, + "objective/train/theoretical_loss": 3.4075998139596786, + "objective/train/tokens_used": 526724576, + "theoretical_loss": 3.4075998139596786, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349010101010101, + "loss": 2.6742, + "theoretical_loss": 3.4075998139596786, + "tokens_seen": 2167865344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034898989898989903, + "loss": 2.6633, + "theoretical_loss": 3.407591360530571, + "tokens_seen": 2167930880 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348969696969697, + "loss": 2.7067, + "theoretical_loss": 3.4075829074285546, + "tokens_seen": 2167996416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034894949494949495, + "loss": 2.6283, + "theoretical_loss": 3.4075744546536075, + "tokens_seen": 2168061952 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034892929292929296, + "loss": 2.7668, + "theoretical_loss": 3.407566002205706, + "tokens_seen": 2168127488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489090909090909, + "loss": 2.8268, + "theoretical_loss": 3.407557550084828, + "tokens_seen": 2168193024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488888888888889, + "loss": 2.6721, + "theoretical_loss": 3.4075490982909518, + "tokens_seen": 2168258560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488686868686869, + "loss": 2.539, + "theoretical_loss": 3.4075406468240534, + "tokens_seen": 2168324096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034884848484848484, + "loss": 2.5762, + "theoretical_loss": 3.4075321956841114, + "tokens_seen": 2168389632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882828282828285, + "loss": 2.682, + "theoretical_loss": 3.4075237448711033, + "tokens_seen": 2168455168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488080808080808, + "loss": 2.6132, + "theoretical_loss": 3.4075152943850058, + "tokens_seen": 2168520704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487878787878788, + "loss": 2.793, + "theoretical_loss": 3.407506844225797, + "tokens_seen": 2168586240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487676767676768, + "loss": 2.7703, + "theoretical_loss": 3.4074983943934543, + "tokens_seen": 2168651776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874747474747473, + "loss": 2.8629, + "theoretical_loss": 3.4074899448879545, + "tokens_seen": 2168717312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872727272727274, + "loss": 2.6282, + "theoretical_loss": 3.407481495709276, + "tokens_seen": 2168782848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487070707070707, + "loss": 2.6625, + "theoretical_loss": 3.4074730468573957, + "tokens_seen": 2168848384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486868686868687, + "loss": 2.7749, + "theoretical_loss": 3.4074645983322913, + "tokens_seen": 2168913920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866666666666667, + "loss": 2.6374, + "theoretical_loss": 3.4074561501339407, + "tokens_seen": 2168979456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486464646464647, + "loss": 2.8392, + "theoretical_loss": 3.4074477022623206, + "tokens_seen": 2169044992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862626262626263, + "loss": 2.6692, + "theoretical_loss": 3.407439254717409, + "tokens_seen": 2169110528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486060606060606, + "loss": 2.5859, + "theoretical_loss": 3.407430807499183, + "tokens_seen": 2169176064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485858585858586, + "loss": 2.4564, + "theoretical_loss": 3.4074223606076206, + "tokens_seen": 2169241600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034856565656565656, + "loss": 2.6747, + "theoretical_loss": 3.407413914042699, + "tokens_seen": 2169307136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854545454545457, + "loss": 2.6707, + "theoretical_loss": 3.407405467804396, + "tokens_seen": 2169372672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034852525252525253, + "loss": 2.5795, + "theoretical_loss": 3.407397021892688, + "tokens_seen": 2169438208 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1229408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.836505651473999, + "objective/train/theoretical_loss": 3.407388576307554, + "objective/train/tokens_used": 528362976, + "theoretical_loss": 3.407388576307554, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485050505050505, + "loss": 2.7993, + "theoretical_loss": 3.407388576307554, + "tokens_seen": 2169503744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484848484848485, + "loss": 2.8703, + "theoretical_loss": 3.407380131048971, + "tokens_seen": 2169569280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484646464646465, + "loss": 2.5578, + "theoretical_loss": 3.407371686116916, + "tokens_seen": 2169634816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034844444444444446, + "loss": 2.6906, + "theoretical_loss": 3.4073632415113675, + "tokens_seen": 2169700352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484242424242424, + "loss": 2.6026, + "theoretical_loss": 3.4073547972323017, + "tokens_seen": 2169765888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484040404040404, + "loss": 2.7613, + "theoretical_loss": 3.407346353279697, + "tokens_seen": 2169831424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483838383838384, + "loss": 2.8437, + "theoretical_loss": 3.4073379096535312, + "tokens_seen": 2169896960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483636363636364, + "loss": 2.6246, + "theoretical_loss": 3.4073294663537808, + "tokens_seen": 2169962496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834343434343436, + "loss": 2.6495, + "theoretical_loss": 3.4073210233804243, + "tokens_seen": 2170028032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483232323232323, + "loss": 2.6598, + "theoretical_loss": 3.4073125807334383, + "tokens_seen": 2170093568 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034830303030303027, + "loss": 2.7787, + "theoretical_loss": 3.407304138412801, + "tokens_seen": 2170159104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828282828282834, + "loss": 2.5098, + "theoretical_loss": 3.4072956964184904, + "tokens_seen": 2170224640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482626262626263, + "loss": 2.7703, + "theoretical_loss": 3.4072872547504827, + "tokens_seen": 2170290176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824242424242425, + "loss": 2.7512, + "theoretical_loss": 3.4072788134087566, + "tokens_seen": 2170355712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482222222222222, + "loss": 2.717, + "theoretical_loss": 3.407270372393289, + "tokens_seen": 2170421248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820202020202016, + "loss": 2.4831, + "theoretical_loss": 3.4072619317040576, + "tokens_seen": 2170486784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818181818181823, + "loss": 2.6432, + "theoretical_loss": 3.4072534913410397, + "tokens_seen": 2170552320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481616161616162, + "loss": 2.694, + "theoretical_loss": 3.4072450513042134, + "tokens_seen": 2170617856 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034814141414141414, + "loss": 2.6997, + "theoretical_loss": 3.407236611593556, + "tokens_seen": 2170683392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481212121212121, + "loss": 2.5383, + "theoretical_loss": 3.4072281722090447, + "tokens_seen": 2170748928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481010101010101, + "loss": 2.6068, + "theoretical_loss": 3.4072197331506575, + "tokens_seen": 2170814464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480808080808081, + "loss": 2.5106, + "theoretical_loss": 3.407211294418372, + "tokens_seen": 2170880000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480606060606061, + "loss": 2.8415, + "theoretical_loss": 3.4072028560121654, + "tokens_seen": 2170945536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804040404040403, + "loss": 2.5775, + "theoretical_loss": 3.4071944179320153, + "tokens_seen": 2171011072 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348020202020202, + "loss": 2.5039, + "theoretical_loss": 3.4071859801778994, + "tokens_seen": 2171076608 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1230033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6070539951324463, + "objective/train/theoretical_loss": 3.4071775427497952, + "objective/train/tokens_used": 530001376, + "theoretical_loss": 3.4071775427497952, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348, + "loss": 2.7155, + "theoretical_loss": 3.4071775427497952, + "tokens_seen": 2171142144 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347979797979798, + "loss": 2.6885, + "theoretical_loss": 3.4071691056476805, + "tokens_seen": 2171207680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034795959595959597, + "loss": 2.55, + "theoretical_loss": 3.4071606688715326, + "tokens_seen": 2171273216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479393939393939, + "loss": 2.4353, + "theoretical_loss": 3.4071522324213293, + "tokens_seen": 2171338752 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034791919191919194, + "loss": 2.4385, + "theoretical_loss": 3.4071437962970474, + "tokens_seen": 2171404288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478989898989899, + "loss": 2.6643, + "theoretical_loss": 3.407135360498666, + "tokens_seen": 2171469824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478787878787879, + "loss": 2.537, + "theoretical_loss": 3.4071269250261604, + "tokens_seen": 2171535360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785858585858586, + "loss": 2.5825, + "theoretical_loss": 3.407118489879511, + "tokens_seen": 2171600896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478383838383838, + "loss": 2.531, + "theoretical_loss": 3.407110055058693, + "tokens_seen": 2171666432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034781818181818183, + "loss": 2.5063, + "theoretical_loss": 3.407101620563685, + "tokens_seen": 2171731968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477979797979798, + "loss": 2.7637, + "theoretical_loss": 3.4070931863944645, + "tokens_seen": 2171797504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477777777777778, + "loss": 2.4634, + "theoretical_loss": 3.4070847525510093, + "tokens_seen": 2171863040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034775757575757576, + "loss": 2.6852, + "theoretical_loss": 3.4070763190332967, + "tokens_seen": 2171928576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034773737373737377, + "loss": 2.6487, + "theoretical_loss": 3.407067885841304, + "tokens_seen": 2171994112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477171717171717, + "loss": 2.8468, + "theoretical_loss": 3.4070594529750093, + "tokens_seen": 2172059648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476969696969697, + "loss": 2.5925, + "theoretical_loss": 3.4070510204343902, + "tokens_seen": 2172125184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476767676767677, + "loss": 2.765, + "theoretical_loss": 3.407042588219424, + "tokens_seen": 2172190720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034765656565656565, + "loss": 2.4825, + "theoretical_loss": 3.407034156330088, + "tokens_seen": 2172256256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034763636363636366, + "loss": 2.6407, + "theoretical_loss": 3.4070257247663607, + "tokens_seen": 2172321792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476161616161616, + "loss": 2.6112, + "theoretical_loss": 3.4070172935282192, + "tokens_seen": 2172387328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034759595959595957, + "loss": 2.6788, + "theoretical_loss": 3.407008862615641, + "tokens_seen": 2172452864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475757575757576, + "loss": 2.7729, + "theoretical_loss": 3.4070004320286036, + "tokens_seen": 2172518400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475555555555556, + "loss": 2.6517, + "theoretical_loss": 3.4069920017670854, + "tokens_seen": 2172583936 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034753535353535355, + "loss": 2.6787, + "theoretical_loss": 3.4069835718310633, + "tokens_seen": 2172649472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475151515151515, + "loss": 2.4151, + "theoretical_loss": 3.4069751422205146, + "tokens_seen": 2172715008 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1231439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.449362277984619, + "objective/train/theoretical_loss": 3.406966712935418, + "objective/train/tokens_used": 531639776, + "theoretical_loss": 3.406966712935418, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034749494949494947, + "loss": 2.7193, + "theoretical_loss": 3.406966712935418, + "tokens_seen": 2172780544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474747474747475, + "loss": 2.6406, + "theoretical_loss": 3.40695828397575, + "tokens_seen": 2172846080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474545454545455, + "loss": 2.5107, + "theoretical_loss": 3.406949855341489, + "tokens_seen": 2172911616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034743434343434344, + "loss": 2.3709, + "theoretical_loss": 3.4069414270326126, + "tokens_seen": 2172977152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474141414141414, + "loss": 2.6962, + "theoretical_loss": 3.4069329990490975, + "tokens_seen": 2173042688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473939393939394, + "loss": 2.6196, + "theoretical_loss": 3.4069245713909226, + "tokens_seen": 2173108224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473737373737374, + "loss": 2.397, + "theoretical_loss": 3.406916144058065, + "tokens_seen": 2173173760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473535353535354, + "loss": 2.8197, + "theoretical_loss": 3.4069077170505015, + "tokens_seen": 2173239296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034733333333333334, + "loss": 2.5396, + "theoretical_loss": 3.4068992903682114, + "tokens_seen": 2173304832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473131313131313, + "loss": 2.6576, + "theoretical_loss": 3.406890864011171, + "tokens_seen": 2173370368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472929292929293, + "loss": 2.8122, + "theoretical_loss": 3.406882437979358, + "tokens_seen": 2173435904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472727272727273, + "loss": 2.7792, + "theoretical_loss": 3.406874012272751, + "tokens_seen": 2173501440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472525252525253, + "loss": 2.6757, + "theoretical_loss": 3.4068655868913273, + "tokens_seen": 2173566976 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034723232323232323, + "loss": 2.6625, + "theoretical_loss": 3.406857161835064, + "tokens_seen": 2173632512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472121212121212, + "loss": 2.8901, + "theoretical_loss": 3.406848737103939, + "tokens_seen": 2173698048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034719191919191925, + "loss": 2.4687, + "theoretical_loss": 3.4068403126979296, + "tokens_seen": 2173763584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471717171717172, + "loss": 2.6417, + "theoretical_loss": 3.406831888617014, + "tokens_seen": 2173829120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034715151515151517, + "loss": 2.758, + "theoretical_loss": 3.40682346486117, + "tokens_seen": 2173894656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471313131313131, + "loss": 2.7905, + "theoretical_loss": 3.4068150414303755, + "tokens_seen": 2173960192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471111111111111, + "loss": 2.9186, + "theoretical_loss": 3.406806618324607, + "tokens_seen": 2174025728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034709090909090914, + "loss": 2.599, + "theoretical_loss": 3.4067981955438427, + "tokens_seen": 2174091264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470707070707071, + "loss": 2.5879, + "theoretical_loss": 3.4067897730880605, + "tokens_seen": 2174156800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034705050505050506, + "loss": 2.6794, + "theoretical_loss": 3.406781350957238, + "tokens_seen": 2174222336 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347030303030303, + "loss": 2.5315, + "theoretical_loss": 3.406772929151353, + "tokens_seen": 2174287872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701010101010097, + "loss": 2.6784, + "theoretical_loss": 3.4067645076703825, + "tokens_seen": 2174353408 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1232037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5743825435638428, + "objective/train/theoretical_loss": 3.4067560865143047, + "objective/train/tokens_used": 533278176, + "theoretical_loss": 3.4067560865143047, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698989898989904, + "loss": 2.6061, + "theoretical_loss": 3.4067560865143047, + "tokens_seen": 2174418944 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346969696969697, + "loss": 2.4113, + "theoretical_loss": 3.4067476656830973, + "tokens_seen": 2174484480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694949494949495, + "loss": 2.5026, + "theoretical_loss": 3.406739245176738, + "tokens_seen": 2174550016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469292929292929, + "loss": 2.7866, + "theoretical_loss": 3.406730824995204, + "tokens_seen": 2174615552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469090909090909, + "loss": 2.5488, + "theoretical_loss": 3.406722405138474, + "tokens_seen": 2174681088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688888888888893, + "loss": 2.6746, + "theoretical_loss": 3.4067139856065247, + "tokens_seen": 2174746624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468686868686869, + "loss": 2.6028, + "theoretical_loss": 3.406705566399334, + "tokens_seen": 2174812160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034684848484848484, + "loss": 2.7844, + "theoretical_loss": 3.40669714751688, + "tokens_seen": 2174877696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468282828282828, + "loss": 2.702, + "theoretical_loss": 3.4066887289591397, + "tokens_seen": 2174943232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468080808080808, + "loss": 2.6361, + "theoretical_loss": 3.4066803107260912, + "tokens_seen": 2175008768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467878787878788, + "loss": 2.624, + "theoretical_loss": 3.4066718928177124, + "tokens_seen": 2175074304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467676767676768, + "loss": 2.6517, + "theoretical_loss": 3.4066634752339806, + "tokens_seen": 2175139840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034674747474747474, + "loss": 2.4256, + "theoretical_loss": 3.4066550579748736, + "tokens_seen": 2175205376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672727272727275, + "loss": 2.7991, + "theoretical_loss": 3.4066466410403695, + "tokens_seen": 2175270912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467070707070707, + "loss": 2.6914, + "theoretical_loss": 3.4066382244304454, + "tokens_seen": 2175336448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466868686868687, + "loss": 2.5621, + "theoretical_loss": 3.4066298081450794, + "tokens_seen": 2175401984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034666666666666667, + "loss": 2.5873, + "theoretical_loss": 3.406621392184249, + "tokens_seen": 2175467520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034664646464646463, + "loss": 2.6969, + "theoretical_loss": 3.4066129765479323, + "tokens_seen": 2175533056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034662626262626264, + "loss": 2.3635, + "theoretical_loss": 3.406604561236106, + "tokens_seen": 2175598592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466060606060606, + "loss": 2.6456, + "theoretical_loss": 3.406596146248749, + "tokens_seen": 2175664128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465858585858586, + "loss": 2.6, + "theoretical_loss": 3.4065877315858386, + "tokens_seen": 2175729664 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034656565656565657, + "loss": 2.6447, + "theoretical_loss": 3.4065793172473526, + "tokens_seen": 2175795200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465454545454546, + "loss": 2.6914, + "theoretical_loss": 3.406570903233268, + "tokens_seen": 2175860736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034652525252525253, + "loss": 2.7256, + "theoretical_loss": 3.4065624895435636, + "tokens_seen": 2175926272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465050505050505, + "loss": 2.5548, + "theoretical_loss": 3.4065540761782165, + "tokens_seen": 2175991808 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1233356, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6514344215393066, + "objective/train/theoretical_loss": 3.406545663137204, + "objective/train/tokens_used": 534916576, + "theoretical_loss": 3.406545663137204, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464848484848485, + "loss": 2.7125, + "theoretical_loss": 3.406545663137204, + "tokens_seen": 2176057344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034646464646464646, + "loss": 2.8952, + "theoretical_loss": 3.4065372504205054, + "tokens_seen": 2176122880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034644444444444447, + "loss": 2.6455, + "theoretical_loss": 3.406528838028097, + "tokens_seen": 2176188416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464242424242424, + "loss": 2.4195, + "theoretical_loss": 3.4065204259599566, + "tokens_seen": 2176253952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464040404040404, + "loss": 2.6887, + "theoretical_loss": 3.4065120142160628, + "tokens_seen": 2176319488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463838383838384, + "loss": 2.6733, + "theoretical_loss": 3.4065036027963926, + "tokens_seen": 2176385024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463636363636364, + "loss": 2.6524, + "theoretical_loss": 3.406495191700924, + "tokens_seen": 2176450560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034634343434343436, + "loss": 2.6887, + "theoretical_loss": 3.4064867809296344, + "tokens_seen": 2176516096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463232323232323, + "loss": 2.8438, + "theoretical_loss": 3.4064783704825023, + "tokens_seen": 2176581632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463030303030303, + "loss": 2.7141, + "theoretical_loss": 3.406469960359505, + "tokens_seen": 2176647168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462828282828283, + "loss": 2.8748, + "theoretical_loss": 3.4064615505606195, + "tokens_seen": 2176712704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462626262626263, + "loss": 2.4277, + "theoretical_loss": 3.4064531410858248, + "tokens_seen": 2176778240 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034624242424242425, + "loss": 2.4326, + "theoretical_loss": 3.4064447319350983, + "tokens_seen": 2176843776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462222222222222, + "loss": 2.8426, + "theoretical_loss": 3.4064363231084176, + "tokens_seen": 2176909312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034620202020202017, + "loss": 2.6734, + "theoretical_loss": 3.4064279146057603, + "tokens_seen": 2176974848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034618181818181823, + "loss": 2.5614, + "theoretical_loss": 3.4064195064271043, + "tokens_seen": 2177040384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461616161616162, + "loss": 2.544, + "theoretical_loss": 3.406411098572428, + "tokens_seen": 2177105920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034614141414141415, + "loss": 2.5821, + "theoretical_loss": 3.4064026910417073, + "tokens_seen": 2177171456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461212121212121, + "loss": 2.5533, + "theoretical_loss": 3.4063942838349224, + "tokens_seen": 2177236992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034610101010101006, + "loss": 2.6762, + "theoretical_loss": 3.4063858769520494, + "tokens_seen": 2177302528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460808080808081, + "loss": 2.4442, + "theoretical_loss": 3.4063774703930667, + "tokens_seen": 2177368064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460606060606061, + "loss": 2.5106, + "theoretical_loss": 3.4063690641579516, + "tokens_seen": 2177433600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034604040404040404, + "loss": 2.5638, + "theoretical_loss": 3.4063606582466823, + "tokens_seen": 2177499136 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346020202020202, + "loss": 2.5363, + "theoretical_loss": 3.4063522526592367, + "tokens_seen": 2177564672 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346, + "loss": 2.7244, + "theoretical_loss": 3.4063438473955925, + "tokens_seen": 2177630208 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 1233682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5204880237579346, + "objective/train/theoretical_loss": 3.406335442455727, + "objective/train/tokens_used": 536554976, + "theoretical_loss": 3.406335442455727, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345979797979798, + "loss": 2.6475, + "theoretical_loss": 3.406335442455727, + "tokens_seen": 2177695744 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345959595959596, + "loss": 2.7073, + "theoretical_loss": 3.4063270378396187, + "tokens_seen": 2177761280 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034593939393939393, + "loss": 2.4325, + "theoretical_loss": 3.4063186335472446, + "tokens_seen": 2177826816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003459191919191919, + "loss": 2.6909, + "theoretical_loss": 3.406310229578583, + "tokens_seen": 2177892352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458989898989899, + "loss": 2.6815, + "theoretical_loss": 3.4063018259336117, + "tokens_seen": 2177957888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458787878787879, + "loss": 2.5713, + "theoretical_loss": 3.4062934226123085, + "tokens_seen": 2178023424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034585858585858587, + "loss": 2.578, + "theoretical_loss": 3.406285019614651, + "tokens_seen": 2178088960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458383838383838, + "loss": 2.618, + "theoretical_loss": 3.406276616940617, + "tokens_seen": 2178154496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581818181818184, + "loss": 2.6829, + "theoretical_loss": 3.4062682145901846, + "tokens_seen": 2178220032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457979797979798, + "loss": 2.7427, + "theoretical_loss": 3.406259812563331, + "tokens_seen": 2178285568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457777777777778, + "loss": 2.7221, + "theoretical_loss": 3.4062514108600346, + "tokens_seen": 2178351104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034575757575757576, + "loss": 2.6808, + "theoretical_loss": 3.406243009480273, + "tokens_seen": 2178416640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457373737373737, + "loss": 2.6831, + "theoretical_loss": 3.406234608424024, + "tokens_seen": 2178482176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034571717171717173, + "loss": 2.8424, + "theoretical_loss": 3.4062262076912653, + "tokens_seen": 2178547712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456969696969697, + "loss": 2.8419, + "theoretical_loss": 3.4062178072819753, + "tokens_seen": 2178613248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456767676767677, + "loss": 2.6555, + "theoretical_loss": 3.406209407196131, + "tokens_seen": 2178678784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565656565656565, + "loss": 2.7164, + "theoretical_loss": 3.406201007433711, + "tokens_seen": 2178744320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034563636363636366, + "loss": 2.7442, + "theoretical_loss": 3.406192607994692, + "tokens_seen": 2178809856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456161616161616, + "loss": 2.825, + "theoretical_loss": 3.4061842088790524, + "tokens_seen": 2178875392 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034559595959595963, + "loss": 2.4686, + "theoretical_loss": 3.4061758100867707, + "tokens_seen": 2178940928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455757575757576, + "loss": 2.7424, + "theoretical_loss": 3.4061674116178238, + "tokens_seen": 2179006464 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034555555555555555, + "loss": 2.8691, + "theoretical_loss": 3.40615901347219, + "tokens_seen": 2179072000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553535353535356, + "loss": 2.7381, + "theoretical_loss": 3.4061506156498473, + "tokens_seen": 2179137536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455151515151515, + "loss": 2.7802, + "theoretical_loss": 3.406142218150773, + "tokens_seen": 2179203072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454949494949495, + "loss": 2.8637, + "theoretical_loss": 3.406133820974945, + "tokens_seen": 2179268608 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1234902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7522618770599365, + "objective/train/theoretical_loss": 3.4061254241223415, + "objective/train/tokens_used": 538193376, + "theoretical_loss": 3.4061254241223415, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454747474747475, + "loss": 2.6014, + "theoretical_loss": 3.4061254241223415, + "tokens_seen": 2179334144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545454545454544, + "loss": 2.7235, + "theoretical_loss": 3.4061170275929404, + "tokens_seen": 2179399680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543434343434345, + "loss": 2.6067, + "theoretical_loss": 3.406108631386719, + "tokens_seen": 2179465216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454141414141414, + "loss": 2.8426, + "theoretical_loss": 3.4061002355036556, + "tokens_seen": 2179530752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453939393939394, + "loss": 2.5099, + "theoretical_loss": 3.406091839943728, + "tokens_seen": 2179596288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453737373737374, + "loss": 2.7189, + "theoretical_loss": 3.4060834447069133, + "tokens_seen": 2179661824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453535353535354, + "loss": 2.7749, + "theoretical_loss": 3.4060750497931904, + "tokens_seen": 2179727360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034533333333333334, + "loss": 2.7352, + "theoretical_loss": 3.4060666552025367, + "tokens_seen": 2179792896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453131313131313, + "loss": 2.7149, + "theoretical_loss": 3.4060582609349304, + "tokens_seen": 2179858432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452929292929293, + "loss": 2.6506, + "theoretical_loss": 3.4060498669903483, + "tokens_seen": 2179923968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527272727272727, + "loss": 2.5638, + "theoretical_loss": 3.4060414733687696, + "tokens_seen": 2179989504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452525252525253, + "loss": 2.7419, + "theoretical_loss": 3.4060330800701717, + "tokens_seen": 2180055040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034523232323232324, + "loss": 2.443, + "theoretical_loss": 3.406024687094532, + "tokens_seen": 2180120576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452121212121212, + "loss": 2.5741, + "theoretical_loss": 3.4060162944418284, + "tokens_seen": 2180186112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451919191919192, + "loss": 2.7972, + "theoretical_loss": 3.40600790211204, + "tokens_seen": 2180251648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451717171717172, + "loss": 2.6621, + "theoretical_loss": 3.4059995101051426, + "tokens_seen": 2180317184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515151515151517, + "loss": 2.5355, + "theoretical_loss": 3.405991118421116, + "tokens_seen": 2180382720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034513131313131313, + "loss": 2.7359, + "theoretical_loss": 3.405982727059937, + "tokens_seen": 2180448256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451111111111111, + "loss": 2.6174, + "theoretical_loss": 3.405974336021584, + "tokens_seen": 2180513792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450909090909091, + "loss": 2.868, + "theoretical_loss": 3.4059659453060345, + "tokens_seen": 2180579328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450707070707071, + "loss": 2.5186, + "theoretical_loss": 3.4059575549132663, + "tokens_seen": 2180644864 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034505050505050506, + "loss": 2.4944, + "theoretical_loss": 3.4059491648432574, + "tokens_seen": 2180710400 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345030303030303, + "loss": 2.5574, + "theoretical_loss": 3.4059407750959863, + "tokens_seen": 2180775936 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345010101010101, + "loss": 2.5503, + "theoretical_loss": 3.40593238567143, + "tokens_seen": 2180841472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034498989898989904, + "loss": 2.6515, + "theoretical_loss": 3.405923996569567, + "tokens_seen": 2180907008 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1235717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8756673336029053, + "objective/train/theoretical_loss": 3.4059156077903747, + "objective/train/tokens_used": 539831776, + "theoretical_loss": 3.4059156077903747, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344969696969697, + "loss": 2.7347, + "theoretical_loss": 3.4059156077903747, + "tokens_seen": 2180972544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034494949494949496, + "loss": 2.5519, + "theoretical_loss": 3.405907219333831, + "tokens_seen": 2181038080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449292929292929, + "loss": 2.5041, + "theoretical_loss": 3.4058988311999148, + "tokens_seen": 2181103616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034490909090909087, + "loss": 2.6408, + "theoretical_loss": 3.4058904433886026, + "tokens_seen": 2181169152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034488888888888894, + "loss": 2.6473, + "theoretical_loss": 3.4058820558998733, + "tokens_seen": 2181234688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448686868686869, + "loss": 2.7873, + "theoretical_loss": 3.4058736687337046, + "tokens_seen": 2181300224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034484848484848485, + "loss": 2.7367, + "theoretical_loss": 3.4058652818900734, + "tokens_seen": 2181365760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448282828282828, + "loss": 2.6707, + "theoretical_loss": 3.4058568953689594, + "tokens_seen": 2181431296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448080808080808, + "loss": 2.782, + "theoretical_loss": 3.405848509170339, + "tokens_seen": 2181496832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034478787878787883, + "loss": 2.9103, + "theoretical_loss": 3.405840123294191, + "tokens_seen": 2181562368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447676767676768, + "loss": 2.5568, + "theoretical_loss": 3.4058317377404927, + "tokens_seen": 2181627904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034474747474747474, + "loss": 2.6427, + "theoretical_loss": 3.4058233525092225, + "tokens_seen": 2181693440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447272727272727, + "loss": 2.6007, + "theoretical_loss": 3.405814967600358, + "tokens_seen": 2181758976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447070707070707, + "loss": 2.5476, + "theoretical_loss": 3.4058065830138773, + "tokens_seen": 2181824512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446868686868687, + "loss": 2.6449, + "theoretical_loss": 3.405798198749758, + "tokens_seen": 2181890048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446666666666667, + "loss": 2.7275, + "theoretical_loss": 3.405789814807979, + "tokens_seen": 2181955584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034464646464646463, + "loss": 2.8549, + "theoretical_loss": 3.4057814311885166, + "tokens_seen": 2182021120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034462626262626265, + "loss": 2.6584, + "theoretical_loss": 3.4057730478913504, + "tokens_seen": 2182086656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446060606060606, + "loss": 2.4769, + "theoretical_loss": 3.405764664916457, + "tokens_seen": 2182152192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445858585858586, + "loss": 2.6439, + "theoretical_loss": 3.405756282263815, + "tokens_seen": 2182217728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034456565656565657, + "loss": 2.6713, + "theoretical_loss": 3.4057478999334023, + "tokens_seen": 2182283264 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034454545454545453, + "loss": 2.6658, + "theoretical_loss": 3.405739517925197, + "tokens_seen": 2182348800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034452525252525254, + "loss": 2.6483, + "theoretical_loss": 3.4057311362391767, + "tokens_seen": 2182414336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445050505050505, + "loss": 2.4013, + "theoretical_loss": 3.4057227548753195, + "tokens_seen": 2182479872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444848484848485, + "loss": 2.5209, + "theoretical_loss": 3.405714373833603, + "tokens_seen": 2182545408 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1236933, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6975762844085693, + "objective/train/theoretical_loss": 3.405705993114006, + "objective/train/tokens_used": 541470176, + "theoretical_loss": 3.405705993114006, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034446464646464646, + "loss": 2.7055, + "theoretical_loss": 3.405705993114006, + "tokens_seen": 2182610944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444444444444445, + "loss": 2.5849, + "theoretical_loss": 3.4056976127165055, + "tokens_seen": 2182676480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034442424242424243, + "loss": 2.6522, + "theoretical_loss": 3.40568923264108, + "tokens_seen": 2182742016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444040404040404, + "loss": 2.6934, + "theoretical_loss": 3.405680852887707, + "tokens_seen": 2182807552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443838383838384, + "loss": 2.7759, + "theoretical_loss": 3.405672473456365, + "tokens_seen": 2182873088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034436363636363636, + "loss": 2.6963, + "theoretical_loss": 3.4056640943470318, + "tokens_seen": 2182938624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034434343434343437, + "loss": 2.4129, + "theoretical_loss": 3.405655715559685, + "tokens_seen": 2183004160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443232323232323, + "loss": 2.7185, + "theoretical_loss": 3.405647337094303, + "tokens_seen": 2183069696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443030303030303, + "loss": 2.6357, + "theoretical_loss": 3.4056389589508633, + "tokens_seen": 2183135232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442828282828283, + "loss": 2.7372, + "theoretical_loss": 3.4056305811293446, + "tokens_seen": 2183200768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442626262626263, + "loss": 2.6168, + "theoretical_loss": 3.4056222036297243, + "tokens_seen": 2183266304 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034424242424242426, + "loss": 2.8209, + "theoretical_loss": 3.4056138264519804, + "tokens_seen": 2183331840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442222222222222, + "loss": 2.7616, + "theoretical_loss": 3.405605449596091, + "tokens_seen": 2183397376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442020202020202, + "loss": 2.5749, + "theoretical_loss": 3.405597073062034, + "tokens_seen": 2183462912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441818181818182, + "loss": 2.8905, + "theoretical_loss": 3.4055886968497875, + "tokens_seen": 2183528448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441616161616162, + "loss": 2.6797, + "theoretical_loss": 3.4055803209593294, + "tokens_seen": 2183593984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034414141414141415, + "loss": 2.8417, + "theoretical_loss": 3.4055719453906375, + "tokens_seen": 2183659520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441212121212121, + "loss": 2.612, + "theoretical_loss": 3.40556357014369, + "tokens_seen": 2183725056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034410101010101007, + "loss": 2.7072, + "theoretical_loss": 3.405555195218465, + "tokens_seen": 2183790592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034408080808080813, + "loss": 2.7955, + "theoretical_loss": 3.40554682061494, + "tokens_seen": 2183856128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440606060606061, + "loss": 2.7016, + "theoretical_loss": 3.405538446333094, + "tokens_seen": 2183921664 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034404040404040405, + "loss": 2.7368, + "theoretical_loss": 3.405530072372904, + "tokens_seen": 2183987200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344020202020202, + "loss": 2.7902, + "theoretical_loss": 3.405521698734348, + "tokens_seen": 2184052736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399999999999996, + "loss": 2.6921, + "theoretical_loss": 3.405513325417404, + "tokens_seen": 2184118272 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343979797979798, + "loss": 2.5804, + "theoretical_loss": 3.405504952422051, + "tokens_seen": 2184183808 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1237677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.603099822998047, + "objective/train/theoretical_loss": 3.405496579748266, + "objective/train/tokens_used": 543108576, + "theoretical_loss": 3.405496579748266, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343959595959596, + "loss": 2.7171, + "theoretical_loss": 3.405496579748266, + "tokens_seen": 2184249344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034393939393939394, + "loss": 2.8622, + "theoretical_loss": 3.4054882073960275, + "tokens_seen": 2184314880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439191919191919, + "loss": 2.7167, + "theoretical_loss": 3.405479835365313, + "tokens_seen": 2184380416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389898989898996, + "loss": 2.784, + "theoretical_loss": 3.4054714636561014, + "tokens_seen": 2184445952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438787878787879, + "loss": 2.4514, + "theoretical_loss": 3.4054630922683695, + "tokens_seen": 2184511488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438585858585859, + "loss": 2.6216, + "theoretical_loss": 3.405454721202096, + "tokens_seen": 2184577024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383838383838383, + "loss": 2.8702, + "theoretical_loss": 3.405446350457259, + "tokens_seen": 2184642560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438181818181818, + "loss": 2.8389, + "theoretical_loss": 3.4054379800338364, + "tokens_seen": 2184708096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034379797979797985, + "loss": 2.5713, + "theoretical_loss": 3.405429609931806, + "tokens_seen": 2184773632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437777777777778, + "loss": 2.8758, + "theoretical_loss": 3.4054212401511466, + "tokens_seen": 2184839168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375757575757577, + "loss": 2.6968, + "theoretical_loss": 3.4054128706918347, + "tokens_seen": 2184904704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437373737373737, + "loss": 2.7633, + "theoretical_loss": 3.40540450155385, + "tokens_seen": 2184970240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437171717171717, + "loss": 2.5918, + "theoretical_loss": 3.4053961327371693, + "tokens_seen": 2185035776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034369696969696975, + "loss": 2.6955, + "theoretical_loss": 3.4053877642417714, + "tokens_seen": 2185101312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436767676767677, + "loss": 2.7395, + "theoretical_loss": 3.4053793960676337, + "tokens_seen": 2185166848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034365656565656566, + "loss": 2.7274, + "theoretical_loss": 3.405371028214735, + "tokens_seen": 2185232384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436363636363636, + "loss": 2.6901, + "theoretical_loss": 3.4053626606830525, + "tokens_seen": 2185297920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034361616161616163, + "loss": 2.7284, + "theoretical_loss": 3.405354293472565, + "tokens_seen": 2185363456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359595959595964, + "loss": 2.7762, + "theoretical_loss": 3.4053459265832497, + "tokens_seen": 2185428992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435757575757576, + "loss": 2.6084, + "theoretical_loss": 3.4053375600150853, + "tokens_seen": 2185494528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034355555555555555, + "loss": 2.7035, + "theoretical_loss": 3.4053291937680497, + "tokens_seen": 2185560064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435353535353535, + "loss": 2.61, + "theoretical_loss": 3.4053208278421208, + "tokens_seen": 2185625600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435151515151515, + "loss": 2.6684, + "theoretical_loss": 3.405312462237277, + "tokens_seen": 2185691136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034349494949494953, + "loss": 2.9284, + "theoretical_loss": 3.405304096953496, + "tokens_seen": 2185756672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434747474747475, + "loss": 2.8187, + "theoretical_loss": 3.4052957319907557, + "tokens_seen": 2185822208 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1238885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.666649580001831, + "objective/train/theoretical_loss": 3.4052873673490347, + "objective/train/tokens_used": 544746976, + "theoretical_loss": 3.4052873673490347, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034345454545454544, + "loss": 2.5426, + "theoretical_loss": 3.4052873673490347, + "tokens_seen": 2185887744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343434343434346, + "loss": 2.7339, + "theoretical_loss": 3.4052790030283107, + "tokens_seen": 2185953280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434141414141414, + "loss": 2.8191, + "theoretical_loss": 3.405270639028562, + "tokens_seen": 2186018816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433939393939394, + "loss": 2.8228, + "theoretical_loss": 3.405262275349766, + "tokens_seen": 2186084352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433737373737374, + "loss": 2.6452, + "theoretical_loss": 3.4052539119919016, + "tokens_seen": 2186149888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335353535353534, + "loss": 2.5419, + "theoretical_loss": 3.4052455489549462, + "tokens_seen": 2186215424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034333333333333335, + "loss": 2.7361, + "theoretical_loss": 3.4052371862388786, + "tokens_seen": 2186280960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433131313131313, + "loss": 2.7403, + "theoretical_loss": 3.405228823843676, + "tokens_seen": 2186346496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432929292929293, + "loss": 2.8043, + "theoretical_loss": 3.4052204617693174, + "tokens_seen": 2186412032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432727272727273, + "loss": 2.5705, + "theoretical_loss": 3.4052121000157802, + "tokens_seen": 2186477568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432525252525253, + "loss": 2.8662, + "theoretical_loss": 3.4052037385830425, + "tokens_seen": 2186543104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034323232323232324, + "loss": 2.7916, + "theoretical_loss": 3.405195377471083, + "tokens_seen": 2186608640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432121212121212, + "loss": 2.4098, + "theoretical_loss": 3.4051870166798786, + "tokens_seen": 2186674176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431919191919192, + "loss": 2.7938, + "theoretical_loss": 3.4051786562094084, + "tokens_seen": 2186739712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034317171717171717, + "loss": 2.6087, + "theoretical_loss": 3.4051702960596506, + "tokens_seen": 2186805248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431515151515152, + "loss": 2.6869, + "theoretical_loss": 3.4051619362305825, + "tokens_seen": 2186870784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034313131313131313, + "loss": 2.8161, + "theoretical_loss": 3.4051535767221823, + "tokens_seen": 2186936320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431111111111111, + "loss": 2.5486, + "theoretical_loss": 3.405145217534429, + "tokens_seen": 2187001856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430909090909091, + "loss": 2.7579, + "theoretical_loss": 3.4051368586672996, + "tokens_seen": 2187067392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430707070707071, + "loss": 2.6859, + "theoretical_loss": 3.4051285001207727, + "tokens_seen": 2187132928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034305050505050507, + "loss": 2.6702, + "theoretical_loss": 3.4051201418948263, + "tokens_seen": 2187198464 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343030303030303, + "loss": 2.5928, + "theoretical_loss": 3.4051117839894385, + "tokens_seen": 2187264000 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343010101010101, + "loss": 2.7833, + "theoretical_loss": 3.4051034264045876, + "tokens_seen": 2187329536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342989898989899, + "loss": 2.706, + "theoretical_loss": 3.4050950691402515, + "tokens_seen": 2187395072 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342969696969697, + "loss": 2.7232, + "theoretical_loss": 3.4050867121964083, + "tokens_seen": 2187460608 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1239462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5709733963012695, + "objective/train/theoretical_loss": 3.4050783555730364, + "objective/train/tokens_used": 546385376, + "theoretical_loss": 3.4050783555730364, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034294949494949496, + "loss": 2.5966, + "theoretical_loss": 3.4050783555730364, + "tokens_seen": 2187526144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429292929292929, + "loss": 2.32, + "theoretical_loss": 3.4050699992701134, + "tokens_seen": 2187591680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429090909090909, + "loss": 2.7091, + "theoretical_loss": 3.4050616432876177, + "tokens_seen": 2187657216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034288888888888894, + "loss": 2.6115, + "theoretical_loss": 3.4050532876255275, + "tokens_seen": 2187722752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686868686869, + "loss": 2.8025, + "theoretical_loss": 3.405044932283821, + "tokens_seen": 2187788288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284848484848486, + "loss": 2.4501, + "theoretical_loss": 3.405036577262476, + "tokens_seen": 2187853824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428282828282828, + "loss": 2.7463, + "theoretical_loss": 3.4050282225614703, + "tokens_seen": 2187919360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034280808080808077, + "loss": 2.7788, + "theoretical_loss": 3.405019868180783, + "tokens_seen": 2187984896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034278787878787883, + "loss": 2.7287, + "theoretical_loss": 3.4050115141203916, + "tokens_seen": 2188050432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427676767676768, + "loss": 2.4953, + "theoretical_loss": 3.405003160380274, + "tokens_seen": 2188115968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034274747474747475, + "loss": 2.441, + "theoretical_loss": 3.4049948069604095, + "tokens_seen": 2188181504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427272727272727, + "loss": 2.7249, + "theoretical_loss": 3.404986453860775, + "tokens_seen": 2188247040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427070707070707, + "loss": 2.6952, + "theoretical_loss": 3.4049781010813485, + "tokens_seen": 2188312576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426868686868687, + "loss": 2.522, + "theoretical_loss": 3.404969748622109, + "tokens_seen": 2188378112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426666666666667, + "loss": 2.8709, + "theoretical_loss": 3.404961396483034, + "tokens_seen": 2188443648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264646464646464, + "loss": 2.8045, + "theoretical_loss": 3.4049530446641025, + "tokens_seen": 2188509184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426262626262626, + "loss": 2.638, + "theoretical_loss": 3.404944693165292, + "tokens_seen": 2188574720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426060606060606, + "loss": 2.6747, + "theoretical_loss": 3.4049363419865806, + "tokens_seen": 2188640256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425858585858586, + "loss": 2.6258, + "theoretical_loss": 3.404927991127946, + "tokens_seen": 2188705792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425656565656566, + "loss": 2.7652, + "theoretical_loss": 3.404919640589368, + "tokens_seen": 2188771328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034254545454545453, + "loss": 2.4383, + "theoretical_loss": 3.4049112903708227, + "tokens_seen": 2188836864 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252525252525254, + "loss": 2.5544, + "theoretical_loss": 3.4049029404722897, + "tokens_seen": 2188902400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425050505050505, + "loss": 2.8324, + "theoretical_loss": 3.404894590893747, + "tokens_seen": 2188967936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424848484848485, + "loss": 2.6492, + "theoretical_loss": 3.4048862416351717, + "tokens_seen": 2189033472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034246464646464647, + "loss": 2.7162, + "theoretical_loss": 3.4048778926965433, + "tokens_seen": 2189099008 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1240823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4131510257720947, + "objective/train/theoretical_loss": 3.404869544077839, + "objective/train/tokens_used": 548023776, + "theoretical_loss": 3.404869544077839, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424444444444444, + "loss": 2.7889, + "theoretical_loss": 3.404869544077839, + "tokens_seen": 2189164544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034242424242424244, + "loss": 3.0324, + "theoretical_loss": 3.404861195779038, + "tokens_seen": 2189230080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424040404040404, + "loss": 2.7456, + "theoretical_loss": 3.4048528478001168, + "tokens_seen": 2189295616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423838383838384, + "loss": 2.4465, + "theoretical_loss": 3.404844500141055, + "tokens_seen": 2189361152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236363636363636, + "loss": 2.869, + "theoretical_loss": 3.4048361528018303, + "tokens_seen": 2189426688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423434343434343, + "loss": 2.4686, + "theoretical_loss": 3.404827805782421, + "tokens_seen": 2189492224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034232323232323233, + "loss": 2.6974, + "theoretical_loss": 3.404819459082805, + "tokens_seen": 2189557760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423030303030303, + "loss": 2.9051, + "theoretical_loss": 3.4048111127029603, + "tokens_seen": 2189623296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422828282828283, + "loss": 2.734, + "theoretical_loss": 3.404802766642866, + "tokens_seen": 2189688832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226262626262625, + "loss": 2.623, + "theoretical_loss": 3.4047944209025, + "tokens_seen": 2189754368 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034224242424242427, + "loss": 2.7182, + "theoretical_loss": 3.4047860754818395, + "tokens_seen": 2189819904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422222222222222, + "loss": 2.8321, + "theoretical_loss": 3.4047777303808635, + "tokens_seen": 2189885440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034220202020202023, + "loss": 2.8568, + "theoretical_loss": 3.4047693855995504, + "tokens_seen": 2189950976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421818181818182, + "loss": 2.4199, + "theoretical_loss": 3.4047610411378777, + "tokens_seen": 2190016512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034216161616161615, + "loss": 2.5611, + "theoretical_loss": 3.404752696995824, + "tokens_seen": 2190082048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214141414141416, + "loss": 2.5096, + "theoretical_loss": 3.404744353173368, + "tokens_seen": 2190147584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421212121212121, + "loss": 2.5818, + "theoretical_loss": 3.4047360096704864, + "tokens_seen": 2190213120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421010101010101, + "loss": 2.5034, + "theoretical_loss": 3.404727666487159, + "tokens_seen": 2190278656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420808080808081, + "loss": 2.932, + "theoretical_loss": 3.4047193236233633, + "tokens_seen": 2190344192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420606060606061, + "loss": 2.6613, + "theoretical_loss": 3.404710981079077, + "tokens_seen": 2190409728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034204040404040405, + "loss": 2.5669, + "theoretical_loss": 3.4047026388542796, + "tokens_seen": 2190475264 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342020202020202, + "loss": 2.782, + "theoretical_loss": 3.404694296948948, + "tokens_seen": 2190540800 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342, + "loss": 2.5158, + "theoretical_loss": 3.4046859553630613, + "tokens_seen": 2190606336 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341979797979798, + "loss": 2.6814, + "theoretical_loss": 3.4046776140965975, + "tokens_seen": 2190671872 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341959595959596, + "loss": 2.5512, + "theoretical_loss": 3.4046692731495343, + "tokens_seen": 2190737408 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1241523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.635329246520996, + "objective/train/theoretical_loss": 3.40466093252185, + "objective/train/tokens_used": 549662176, + "theoretical_loss": 3.40466093252185, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034193939393939394, + "loss": 2.7838, + "theoretical_loss": 3.40466093252185, + "tokens_seen": 2190802944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419191919191919, + "loss": 2.5098, + "theoretical_loss": 3.404652592213524, + "tokens_seen": 2190868480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418989898989899, + "loss": 2.671, + "theoretical_loss": 3.404644252224533, + "tokens_seen": 2190934016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418787878787879, + "loss": 2.5111, + "theoretical_loss": 3.404635912554856, + "tokens_seen": 2190999552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418585858585859, + "loss": 2.5697, + "theoretical_loss": 3.404627573204471, + "tokens_seen": 2191065088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034183838383838384, + "loss": 2.5092, + "theoretical_loss": 3.4046192341733565, + "tokens_seen": 2191130624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418181818181818, + "loss": 2.6565, + "theoretical_loss": 3.4046108954614906, + "tokens_seen": 2191196160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417979797979798, + "loss": 2.7515, + "theoretical_loss": 3.404602557068851, + "tokens_seen": 2191261696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417777777777778, + "loss": 2.7243, + "theoretical_loss": 3.404594218995417, + "tokens_seen": 2191327232 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034175757575757577, + "loss": 2.5706, + "theoretical_loss": 3.4045858812411662, + "tokens_seen": 2191392768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034173737373737373, + "loss": 2.6257, + "theoretical_loss": 3.4045775438060764, + "tokens_seen": 2191458304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417171717171717, + "loss": 2.7811, + "theoretical_loss": 3.4045692066901263, + "tokens_seen": 2191523840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034169696969696975, + "loss": 2.93, + "theoretical_loss": 3.4045608698932943, + "tokens_seen": 2191589376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416767676767677, + "loss": 2.606, + "theoretical_loss": 3.4045525334155586, + "tokens_seen": 2191654912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034165656565656566, + "loss": 2.435, + "theoretical_loss": 3.4045441972568975, + "tokens_seen": 2191720448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416363636363636, + "loss": 2.331, + "theoretical_loss": 3.4045358614172887, + "tokens_seen": 2191785984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416161616161616, + "loss": 2.8078, + "theoretical_loss": 3.404527525896711, + "tokens_seen": 2191851520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034159595959595964, + "loss": 2.6476, + "theoretical_loss": 3.404519190695143, + "tokens_seen": 2191917056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415757575757576, + "loss": 3.0974, + "theoretical_loss": 3.4045108558125614, + "tokens_seen": 2191982592 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034155555555555556, + "loss": 2.8261, + "theoretical_loss": 3.404502521248946, + "tokens_seen": 2192048128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415353535353535, + "loss": 2.8123, + "theoretical_loss": 3.404494187004275, + "tokens_seen": 2192113664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415151515151515, + "loss": 2.6853, + "theoretical_loss": 3.4044858530785254, + "tokens_seen": 2192179200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034149494949494954, + "loss": 2.6155, + "theoretical_loss": 3.4044775194716768, + "tokens_seen": 2192244736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414747474747475, + "loss": 2.6432, + "theoretical_loss": 3.404469186183707, + "tokens_seen": 2192310272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034145454545454545, + "loss": 2.7912, + "theoretical_loss": 3.404460853214594, + "tokens_seen": 2192375808 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1242696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.719667673110962, + "objective/train/theoretical_loss": 3.404452520564316, + "objective/train/tokens_used": 551300576, + "theoretical_loss": 3.404452520564316, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414343434343434, + "loss": 2.7431, + "theoretical_loss": 3.404452520564316, + "tokens_seen": 2192441344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414141414141414, + "loss": 2.7562, + "theoretical_loss": 3.4044441882328518, + "tokens_seen": 2192506880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034139393939393943, + "loss": 2.5431, + "theoretical_loss": 3.4044358562201795, + "tokens_seen": 2192572416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413737373737374, + "loss": 2.6624, + "theoretical_loss": 3.404427524526277, + "tokens_seen": 2192637952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034135353535353534, + "loss": 2.8204, + "theoretical_loss": 3.4044191931511234, + "tokens_seen": 2192703488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034133333333333335, + "loss": 2.7791, + "theoretical_loss": 3.404410862094696, + "tokens_seen": 2192769024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413131313131313, + "loss": 2.6378, + "theoretical_loss": 3.4044025313569737, + "tokens_seen": 2192834560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412929292929293, + "loss": 2.7199, + "theoretical_loss": 3.4043942009379347, + "tokens_seen": 2192900096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412727272727273, + "loss": 2.3532, + "theoretical_loss": 3.404385870837557, + "tokens_seen": 2192965632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034125252525252524, + "loss": 2.775, + "theoretical_loss": 3.404377541055819, + "tokens_seen": 2193031168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123232323232325, + "loss": 2.6993, + "theoretical_loss": 3.4043692115926993, + "tokens_seen": 2193096704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412121212121212, + "loss": 2.4824, + "theoretical_loss": 3.4043608824481764, + "tokens_seen": 2193162240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411919191919192, + "loss": 2.6641, + "theoretical_loss": 3.4043525536222274, + "tokens_seen": 2193227776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034117171717171717, + "loss": 2.8446, + "theoretical_loss": 3.4043442251148317, + "tokens_seen": 2193293312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411515151515152, + "loss": 2.8156, + "theoretical_loss": 3.404335896925967, + "tokens_seen": 2193358848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034113131313131314, + "loss": 2.6791, + "theoretical_loss": 3.4043275690556123, + "tokens_seen": 2193424384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411111111111111, + "loss": 2.6703, + "theoretical_loss": 3.404319241503745, + "tokens_seen": 2193489920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410909090909091, + "loss": 2.616, + "theoretical_loss": 3.404310914270344, + "tokens_seen": 2193555456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034107070707070706, + "loss": 2.6034, + "theoretical_loss": 3.404302587355388, + "tokens_seen": 2193620992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410505050505051, + "loss": 2.8216, + "theoretical_loss": 3.404294260758854, + "tokens_seen": 2193686528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103030303030303, + "loss": 2.5961, + "theoretical_loss": 3.4042859344807215, + "tokens_seen": 2193752064 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341010101010101, + "loss": 2.6561, + "theoretical_loss": 3.4042776085209683, + "tokens_seen": 2193817600 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340989898989899, + "loss": 2.6839, + "theoretical_loss": 3.404269282879573, + "tokens_seen": 2193883136 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340969696969697, + "loss": 2.7443, + "theoretical_loss": 3.4042609575565135, + "tokens_seen": 2193948672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034094949494949497, + "loss": 2.4787, + "theoretical_loss": 3.404252632551769, + "tokens_seen": 2194014208 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 1243185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6270833015441895, + "objective/train/theoretical_loss": 3.404244307865316, + "objective/train/tokens_used": 552938976, + "theoretical_loss": 3.404244307865316, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409292929292929, + "loss": 2.6248, + "theoretical_loss": 3.404244307865316, + "tokens_seen": 2194079744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409090909090909, + "loss": 2.7138, + "theoretical_loss": 3.4042359834971347, + "tokens_seen": 2194145280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408888888888889, + "loss": 2.6982, + "theoretical_loss": 3.4042276594472027, + "tokens_seen": 2194210816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408686868686869, + "loss": 2.4854, + "theoretical_loss": 3.4042193357154984, + "tokens_seen": 2194276352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084848484848486, + "loss": 2.9251, + "theoretical_loss": 3.404211012302, + "tokens_seen": 2194341888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408282828282828, + "loss": 2.716, + "theoretical_loss": 3.404202689206686, + "tokens_seen": 2194407424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408080808080808, + "loss": 2.777, + "theoretical_loss": 3.4041943664295347, + "tokens_seen": 2194472960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078787878787884, + "loss": 2.637, + "theoretical_loss": 3.4041860439705243, + "tokens_seen": 2194538496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407676767676768, + "loss": 2.664, + "theoretical_loss": 3.404177721829633, + "tokens_seen": 2194604032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074747474747475, + "loss": 2.7516, + "theoretical_loss": 3.4041694000068397, + "tokens_seen": 2194669568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407272727272727, + "loss": 2.6627, + "theoretical_loss": 3.4041610785021224, + "tokens_seen": 2194735104 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070707070707067, + "loss": 2.6524, + "theoretical_loss": 3.4041527573154595, + "tokens_seen": 2194800640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034068686868686873, + "loss": 2.6659, + "theoretical_loss": 3.404144436446829, + "tokens_seen": 2194866176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406666666666667, + "loss": 2.6906, + "theoretical_loss": 3.40413611589621, + "tokens_seen": 2194931712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034064646464646465, + "loss": 2.6576, + "theoretical_loss": 3.40412779566358, + "tokens_seen": 2194997248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406262626262626, + "loss": 2.6619, + "theoretical_loss": 3.4041194757489177, + "tokens_seen": 2195062784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034060606060606056, + "loss": 2.8917, + "theoretical_loss": 3.4041111561522017, + "tokens_seen": 2195128320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405858585858586, + "loss": 2.7788, + "theoretical_loss": 3.4041028368734105, + "tokens_seen": 2195193856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405656565656566, + "loss": 2.8097, + "theoretical_loss": 3.4040945179125215, + "tokens_seen": 2195259392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054545454545454, + "loss": 2.6775, + "theoretical_loss": 3.404086199269514, + "tokens_seen": 2195324928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405252525252525, + "loss": 2.5273, + "theoretical_loss": 3.4040778809443664, + "tokens_seen": 2195390464 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050505050505056, + "loss": 2.6819, + "theoretical_loss": 3.4040695629370563, + "tokens_seen": 2195456000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404848484848485, + "loss": 2.885, + "theoretical_loss": 3.4040612452475627, + "tokens_seen": 2195521536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404646464646465, + "loss": 2.8131, + "theoretical_loss": 3.4040529278758638, + "tokens_seen": 2195587072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044444444444443, + "loss": 2.4385, + "theoretical_loss": 3.4040446108219378, + "tokens_seen": 2195652608 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1244493, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8833208084106445, + "objective/train/theoretical_loss": 3.4040362940857634, + "objective/train/tokens_used": 554577376, + "theoretical_loss": 3.4040362940857634, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404242424242424, + "loss": 2.6212, + "theoretical_loss": 3.4040362940857634, + "tokens_seen": 2195718144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034040404040404045, + "loss": 2.7972, + "theoretical_loss": 3.4040279776673183, + "tokens_seen": 2195783680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403838383838384, + "loss": 2.6596, + "theoretical_loss": 3.404019661566582, + "tokens_seen": 2195849216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036363636363637, + "loss": 2.741, + "theoretical_loss": 3.4040113457835317, + "tokens_seen": 2195914752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403434343434343, + "loss": 2.5948, + "theoretical_loss": 3.4040030303181465, + "tokens_seen": 2195980288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034032323232323234, + "loss": 2.7051, + "theoretical_loss": 3.403994715170405, + "tokens_seen": 2196045824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034030303030303035, + "loss": 2.6982, + "theoretical_loss": 3.403986400340285, + "tokens_seen": 2196111360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402828282828283, + "loss": 2.7021, + "theoretical_loss": 3.403978085827765, + "tokens_seen": 2196176896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034026262626262626, + "loss": 2.6547, + "theoretical_loss": 3.4039697716328234, + "tokens_seen": 2196242432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402424242424242, + "loss": 2.4682, + "theoretical_loss": 3.4039614577554387, + "tokens_seen": 2196307968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034022222222222223, + "loss": 2.8194, + "theoretical_loss": 3.4039531441955897, + "tokens_seen": 2196373504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020202020202024, + "loss": 2.8059, + "theoretical_loss": 3.403944830953254, + "tokens_seen": 2196439040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401818181818182, + "loss": 2.6499, + "theoretical_loss": 3.4039365180284102, + "tokens_seen": 2196504576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016161616161615, + "loss": 2.4833, + "theoretical_loss": 3.403928205421037, + "tokens_seen": 2196570112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034014141414141416, + "loss": 2.8045, + "theoretical_loss": 3.403919893131113, + "tokens_seen": 2196635648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401212121212121, + "loss": 2.6248, + "theoretical_loss": 3.4039115811586163, + "tokens_seen": 2196701184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034010101010101013, + "loss": 2.793, + "theoretical_loss": 3.403903269503525, + "tokens_seen": 2196766720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400808080808081, + "loss": 2.8219, + "theoretical_loss": 3.403894958165818, + "tokens_seen": 2196832256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034006060606060605, + "loss": 2.6189, + "theoretical_loss": 3.4038866471454736, + "tokens_seen": 2196897792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034004040404040406, + "loss": 2.6393, + "theoretical_loss": 3.4038783364424696, + "tokens_seen": 2196963328 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340020202020202, + "loss": 2.7649, + "theoretical_loss": 3.4038700260567856, + "tokens_seen": 2197028864 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 2.7635, + "theoretical_loss": 3.4038617159883993, + "tokens_seen": 2197094400 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339979797979798, + "loss": 2.7123, + "theoretical_loss": 3.403853406237289, + "tokens_seen": 2197159936 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339959595959596, + "loss": 2.6196, + "theoretical_loss": 3.4038450968034333, + "tokens_seen": 2197225472 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993939393939395, + "loss": 2.7518, + "theoretical_loss": 3.4038367876868105, + "tokens_seen": 2197291008 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1244898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.95704984664917, + "objective/train/theoretical_loss": 3.4038284788873994, + "objective/train/tokens_used": 556215776, + "theoretical_loss": 3.4038284788873994, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399191919191919, + "loss": 2.9447, + "theoretical_loss": 3.4038284788873994, + "tokens_seen": 2197356544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398989898989899, + "loss": 2.7473, + "theoretical_loss": 3.403820170405178, + "tokens_seen": 2197422080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398787878787879, + "loss": 2.6428, + "theoretical_loss": 3.403811862240125, + "tokens_seen": 2197487616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398585858585859, + "loss": 2.8691, + "theoretical_loss": 3.403803554392219, + "tokens_seen": 2197553152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983838383838384, + "loss": 2.81, + "theoretical_loss": 3.403795246861438, + "tokens_seen": 2197618688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398181818181818, + "loss": 2.735, + "theoretical_loss": 3.4037869396477607, + "tokens_seen": 2197684224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397979797979798, + "loss": 2.7578, + "theoretical_loss": 3.4037786327511657, + "tokens_seen": 2197749760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397777777777778, + "loss": 2.6188, + "theoretical_loss": 3.4037703261716308, + "tokens_seen": 2197815296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397575757575758, + "loss": 2.6121, + "theoretical_loss": 3.403762019909135, + "tokens_seen": 2197880832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033973737373737373, + "loss": 2.8401, + "theoretical_loss": 3.4037537139636567, + "tokens_seen": 2197946368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397171717171717, + "loss": 2.6574, + "theoretical_loss": 3.4037454083351744, + "tokens_seen": 2198011904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396969696969697, + "loss": 2.6947, + "theoretical_loss": 3.403737103023666, + "tokens_seen": 2198077440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396767676767677, + "loss": 2.5166, + "theoretical_loss": 3.403728798029111, + "tokens_seen": 2198142976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033965656565656567, + "loss": 2.6456, + "theoretical_loss": 3.4037204933514866, + "tokens_seen": 2198208512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963636363636363, + "loss": 2.6432, + "theoretical_loss": 3.403712188990772, + "tokens_seen": 2198274048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396161616161616, + "loss": 2.5895, + "theoretical_loss": 3.403703884946946, + "tokens_seen": 2198339584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959595959595965, + "loss": 2.6302, + "theoretical_loss": 3.4036955812199863, + "tokens_seen": 2198405120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395757575757576, + "loss": 2.7111, + "theoretical_loss": 3.4036872778098717, + "tokens_seen": 2198470656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955555555555556, + "loss": 2.544, + "theoretical_loss": 3.4036789747165805, + "tokens_seen": 2198536192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395353535353535, + "loss": 2.7501, + "theoretical_loss": 3.403670671940091, + "tokens_seen": 2198601728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395151515151515, + "loss": 2.9635, + "theoretical_loss": 3.403662369480383, + "tokens_seen": 2198667264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033949494949494954, + "loss": 2.5738, + "theoretical_loss": 3.403654067337433, + "tokens_seen": 2198732800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394747474747475, + "loss": 2.5807, + "theoretical_loss": 3.4036457655112207, + "tokens_seen": 2198798336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945454545454546, + "loss": 2.6312, + "theoretical_loss": 3.403637464001724, + "tokens_seen": 2198863872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394343434343434, + "loss": 2.6822, + "theoretical_loss": 3.4036291628089224, + "tokens_seen": 2198929408 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1246280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.800483465194702, + "objective/train/theoretical_loss": 3.403620861932793, + "objective/train/tokens_used": 557854176, + "theoretical_loss": 3.403620861932793, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394141414141414, + "loss": 2.6293, + "theoretical_loss": 3.403620861932793, + "tokens_seen": 2198994944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939393939393943, + "loss": 2.3907, + "theoretical_loss": 3.4036125613733152, + "tokens_seen": 2199060480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393737373737374, + "loss": 2.6814, + "theoretical_loss": 3.403604261130467, + "tokens_seen": 2199126016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033935353535353535, + "loss": 2.5397, + "theoretical_loss": 3.4035959612042275, + "tokens_seen": 2199191552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393333333333333, + "loss": 2.7217, + "theoretical_loss": 3.4035876615945746, + "tokens_seen": 2199257088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393131313131313, + "loss": 2.4775, + "theoretical_loss": 3.4035793623014867, + "tokens_seen": 2199322624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929292929292933, + "loss": 2.7982, + "theoretical_loss": 3.403571063324943, + "tokens_seen": 2199388160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392727272727273, + "loss": 2.5282, + "theoretical_loss": 3.403562764664921, + "tokens_seen": 2199453696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925252525252524, + "loss": 2.7277, + "theoretical_loss": 3.4035544663214004, + "tokens_seen": 2199519232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033923232323232325, + "loss": 2.5653, + "theoretical_loss": 3.403546168294359, + "tokens_seen": 2199584768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392121212121212, + "loss": 2.6559, + "theoretical_loss": 3.403537870583775, + "tokens_seen": 2199650304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391919191919192, + "loss": 2.9012, + "theoretical_loss": 3.4035295731896276, + "tokens_seen": 2199715840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391717171717172, + "loss": 2.5003, + "theoretical_loss": 3.4035212761118947, + "tokens_seen": 2199781376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033915151515151513, + "loss": 2.5964, + "theoretical_loss": 3.4035129793505554, + "tokens_seen": 2199846912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913131313131315, + "loss": 2.6389, + "theoretical_loss": 3.403504682905588, + "tokens_seen": 2199912448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391111111111111, + "loss": 2.856, + "theoretical_loss": 3.40349638677697, + "tokens_seen": 2199977984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390909090909091, + "loss": 2.6462, + "theoretical_loss": 3.403488090964682, + "tokens_seen": 2200043520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907070707070707, + "loss": 2.7316, + "theoretical_loss": 3.4034797954687006, + "tokens_seen": 2200109056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339050505050505, + "loss": 2.7066, + "theoretical_loss": 3.403471500289005, + "tokens_seen": 2200174592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033903030303030304, + "loss": 2.636, + "theoretical_loss": 3.4034632054255742, + "tokens_seen": 2200240128 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339010101010101, + "loss": 2.7101, + "theoretical_loss": 3.403454910878386, + "tokens_seen": 2200305664 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338989898989899, + "loss": 2.6676, + "theoretical_loss": 3.4034466166474195, + "tokens_seen": 2200371200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033896969696969696, + "loss": 2.6766, + "theoretical_loss": 3.4034383227326526, + "tokens_seen": 2200436736 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338949494949495, + "loss": 2.5151, + "theoretical_loss": 3.4034300291340647, + "tokens_seen": 2200502272 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033892929292929293, + "loss": 2.637, + "theoretical_loss": 3.403421735851633, + "tokens_seen": 2200567808 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1246987, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.628159761428833, + "objective/train/theoretical_loss": 3.4034134428853378, + "objective/train/tokens_used": 559492576, + "theoretical_loss": 3.4034134428853378, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389090909090909, + "loss": 2.6306, + "theoretical_loss": 3.4034134428853378, + "tokens_seen": 2200633344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388888888888889, + "loss": 2.8061, + "theoretical_loss": 3.403405150235156, + "tokens_seen": 2200698880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033886868686868686, + "loss": 2.8063, + "theoretical_loss": 3.403396857901067, + "tokens_seen": 2200764416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033884848484848487, + "loss": 2.6143, + "theoretical_loss": 3.4033885658830494, + "tokens_seen": 2200829952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388282828282828, + "loss": 2.8345, + "theoretical_loss": 3.403380274181081, + "tokens_seen": 2200895488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388080808080808, + "loss": 2.5417, + "theoretical_loss": 3.403371982795141, + "tokens_seen": 2200961024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387878787878788, + "loss": 2.6964, + "theoretical_loss": 3.403363691725208, + "tokens_seen": 2201026560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387676767676768, + "loss": 2.7414, + "theoretical_loss": 3.40335540097126, + "tokens_seen": 2201092096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033874747474747476, + "loss": 2.7266, + "theoretical_loss": 3.403347110533276, + "tokens_seen": 2201157632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387272727272727, + "loss": 2.9091, + "theoretical_loss": 3.4033388204112347, + "tokens_seen": 2201223168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387070707070707, + "loss": 2.6621, + "theoretical_loss": 3.403330530605114, + "tokens_seen": 2201288704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386868686868687, + "loss": 2.5328, + "theoretical_loss": 3.403322241114893, + "tokens_seen": 2201354240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386666666666667, + "loss": 2.5677, + "theoretical_loss": 3.40331395194055, + "tokens_seen": 2201419776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033864646464646465, + "loss": 2.4195, + "theoretical_loss": 3.403305663082064, + "tokens_seen": 2201485312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386262626262626, + "loss": 2.5897, + "theoretical_loss": 3.403297374539413, + "tokens_seen": 2201550848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386060606060606, + "loss": 2.8267, + "theoretical_loss": 3.4032890863125758, + "tokens_seen": 2201616384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858585858585863, + "loss": 2.7257, + "theoretical_loss": 3.403280798401531, + "tokens_seen": 2201681920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385656565656566, + "loss": 2.7219, + "theoretical_loss": 3.403272510806257, + "tokens_seen": 2201747456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033854545454545454, + "loss": 2.6558, + "theoretical_loss": 3.4032642235267327, + "tokens_seen": 2201812992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385252525252525, + "loss": 2.5848, + "theoretical_loss": 3.4032559365629362, + "tokens_seen": 2201878528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385050505050505, + "loss": 2.9078, + "theoretical_loss": 3.4032476499148467, + "tokens_seen": 2201944064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384848484848485, + "loss": 2.5846, + "theoretical_loss": 3.4032393635824416, + "tokens_seen": 2202009600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384646464646465, + "loss": 2.4924, + "theoretical_loss": 3.4032310775657013, + "tokens_seen": 2202075136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033844444444444444, + "loss": 2.7365, + "theoretical_loss": 3.4032227918646027, + "tokens_seen": 2202140672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384242424242424, + "loss": 2.6661, + "theoretical_loss": 3.4032145064791255, + "tokens_seen": 2202206208 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1247624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.782517910003662, + "objective/train/theoretical_loss": 3.4032062214092473, + "objective/train/tokens_used": 561130976, + "theoretical_loss": 3.4032062214092473, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033840404040404046, + "loss": 2.7028, + "theoretical_loss": 3.4032062214092473, + "tokens_seen": 2202271744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383838383838384, + "loss": 2.794, + "theoretical_loss": 3.403197936654948, + "tokens_seen": 2202337280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383636363636364, + "loss": 2.7229, + "theoretical_loss": 3.403189652216205, + "tokens_seen": 2202402816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033834343434343433, + "loss": 2.8342, + "theoretical_loss": 3.403181368092997, + "tokens_seen": 2202468352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383232323232323, + "loss": 2.7979, + "theoretical_loss": 3.4031730842853034, + "tokens_seen": 2202533888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033830303030303035, + "loss": 2.7148, + "theoretical_loss": 3.403164800793102, + "tokens_seen": 2202599424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382828282828283, + "loss": 2.8192, + "theoretical_loss": 3.4031565176163716, + "tokens_seen": 2202664960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826262626262627, + "loss": 2.8765, + "theoretical_loss": 3.403148234755091, + "tokens_seen": 2202730496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382424242424242, + "loss": 2.6936, + "theoretical_loss": 3.403139952209239, + "tokens_seen": 2202796032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033822222222222223, + "loss": 2.6362, + "theoretical_loss": 3.4031316699787935, + "tokens_seen": 2202861568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033820202020202024, + "loss": 2.6899, + "theoretical_loss": 3.403123388063734, + "tokens_seen": 2202927104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381818181818182, + "loss": 2.7122, + "theoretical_loss": 3.403115106464038, + "tokens_seen": 2202992640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033816161616161616, + "loss": 2.7071, + "theoretical_loss": 3.403106825179685, + "tokens_seen": 2203058176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381414141414141, + "loss": 2.7303, + "theoretical_loss": 3.403098544210654, + "tokens_seen": 2203123712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381212121212121, + "loss": 2.5712, + "theoretical_loss": 3.403090263556922, + "tokens_seen": 2203189248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033810101010101014, + "loss": 2.8629, + "theoretical_loss": 3.403081983218469, + "tokens_seen": 2203254784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380808080808081, + "loss": 2.7465, + "theoretical_loss": 3.403073703195273, + "tokens_seen": 2203320320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033806060606060605, + "loss": 2.5964, + "theoretical_loss": 3.403065423487313, + "tokens_seen": 2203385856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033804040404040406, + "loss": 2.6684, + "theoretical_loss": 3.4030571440945674, + "tokens_seen": 2203451392 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338020202020202, + "loss": 2.4571, + "theoretical_loss": 3.4030488650170145, + "tokens_seen": 2203516928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033800000000000003, + "loss": 2.8148, + "theoretical_loss": 3.4030405862546336, + "tokens_seen": 2203582464 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337979797979798, + "loss": 2.865, + "theoretical_loss": 3.403032307807403, + "tokens_seen": 2203648000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033795959595959594, + "loss": 2.795, + "theoretical_loss": 3.4030240296753016, + "tokens_seen": 2203713536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793939393939395, + "loss": 2.7568, + "theoretical_loss": 3.4030157518583075, + "tokens_seen": 2203779072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379191919191919, + "loss": 2.8166, + "theoretical_loss": 3.4030074743563996, + "tokens_seen": 2203844608 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1248832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7011544704437256, + "objective/train/theoretical_loss": 3.4029991971695566, + "objective/train/tokens_used": 562769376, + "theoretical_loss": 3.4029991971695566, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378989898989899, + "loss": 2.4495, + "theoretical_loss": 3.4029991971695566, + "tokens_seen": 2203910144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378787878787879, + "loss": 2.5301, + "theoretical_loss": 3.402990920297757, + "tokens_seen": 2203975680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378585858585859, + "loss": 2.5706, + "theoretical_loss": 3.40298264374098, + "tokens_seen": 2204041216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033783838383838385, + "loss": 2.539, + "theoretical_loss": 3.402974367499203, + "tokens_seen": 2204106752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378181818181818, + "loss": 2.6999, + "theoretical_loss": 3.402966091572406, + "tokens_seen": 2204172288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377979797979798, + "loss": 2.6983, + "theoretical_loss": 3.4029578159605665, + "tokens_seen": 2204237824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777777777777777, + "loss": 2.5168, + "theoretical_loss": 3.4029495406636645, + "tokens_seen": 2204303360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377575757575758, + "loss": 2.5664, + "theoretical_loss": 3.4029412656816773, + "tokens_seen": 2204368896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773737373737374, + "loss": 2.5627, + "theoretical_loss": 3.4029329910145845, + "tokens_seen": 2204434432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377171717171717, + "loss": 2.6933, + "theoretical_loss": 3.4029247166623637, + "tokens_seen": 2204499968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376969696969697, + "loss": 2.6931, + "theoretical_loss": 3.402916442624995, + "tokens_seen": 2204565504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376767676767677, + "loss": 2.5254, + "theoretical_loss": 3.402908168902456, + "tokens_seen": 2204631040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376565656565657, + "loss": 2.5688, + "theoretical_loss": 3.4028998954947256, + "tokens_seen": 2204696576 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033763636363636363, + "loss": 2.5806, + "theoretical_loss": 3.4028916224017824, + "tokens_seen": 2204762112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376161616161616, + "loss": 2.8796, + "theoretical_loss": 3.4028833496236057, + "tokens_seen": 2204827648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375959595959596, + "loss": 2.5132, + "theoretical_loss": 3.4028750771601732, + "tokens_seen": 2204893184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375757575757576, + "loss": 2.7521, + "theoretical_loss": 3.402866805011464, + "tokens_seen": 2204958720 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755555555555557, + "loss": 2.4985, + "theoretical_loss": 3.402858533177457, + "tokens_seen": 2205024256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375353535353535, + "loss": 2.6412, + "theoretical_loss": 3.4028502616581306, + "tokens_seen": 2205089792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375151515151515, + "loss": 2.5687, + "theoretical_loss": 3.4028419904534637, + "tokens_seen": 2205155328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033749494949494955, + "loss": 2.6525, + "theoretical_loss": 3.402833719563435, + "tokens_seen": 2205220864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374747474747475, + "loss": 2.7223, + "theoretical_loss": 3.4028254489880227, + "tokens_seen": 2205286400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745454545454546, + "loss": 2.896, + "theoretical_loss": 3.4028171787272057, + "tokens_seen": 2205351936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374343434343434, + "loss": 2.5432, + "theoretical_loss": 3.402808908780963, + "tokens_seen": 2205417472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374141414141414, + "loss": 2.7064, + "theoretical_loss": 3.402800639149273, + "tokens_seen": 2205483008 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1249474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.777233362197876, + "objective/train/theoretical_loss": 3.4027923698321145, + "objective/train/tokens_used": 564407776, + "theoretical_loss": 3.4027923698321145, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739393939393944, + "loss": 2.7913, + "theoretical_loss": 3.4027923698321145, + "tokens_seen": 2205548544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373737373737374, + "loss": 2.6776, + "theoretical_loss": 3.402784100829466, + "tokens_seen": 2205614080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033735353535353535, + "loss": 2.7815, + "theoretical_loss": 3.402775832141307, + "tokens_seen": 2205679616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373333333333333, + "loss": 2.8421, + "theoretical_loss": 3.4027675637676147, + "tokens_seen": 2205745152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731313131313127, + "loss": 2.807, + "theoretical_loss": 3.402759295708369, + "tokens_seen": 2205810688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033729292929292933, + "loss": 2.8392, + "theoretical_loss": 3.4027510279635482, + "tokens_seen": 2205876224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372727272727273, + "loss": 2.6815, + "theoretical_loss": 3.402742760533131, + "tokens_seen": 2205941760 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033725252525252525, + "loss": 2.7141, + "theoretical_loss": 3.402734493417096, + "tokens_seen": 2206007296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372323232323232, + "loss": 2.6045, + "theoretical_loss": 3.402726226615423, + "tokens_seen": 2206072832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372121212121212, + "loss": 2.7734, + "theoretical_loss": 3.4027179601280886, + "tokens_seen": 2206138368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371919191919192, + "loss": 2.7846, + "theoretical_loss": 3.402709693955073, + "tokens_seen": 2206203904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371717171717172, + "loss": 2.7178, + "theoretical_loss": 3.402701428096355, + "tokens_seen": 2206269440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715151515151514, + "loss": 2.805, + "theoretical_loss": 3.402693162551912, + "tokens_seen": 2206334976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313131313131, + "loss": 2.7554, + "theoretical_loss": 3.4026848973217243, + "tokens_seen": 2206400512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371111111111111, + "loss": 2.477, + "theoretical_loss": 3.40267663240577, + "tokens_seen": 2206466048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370909090909091, + "loss": 2.7073, + "theoretical_loss": 3.4026683678040275, + "tokens_seen": 2206531584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370707070707071, + "loss": 2.8755, + "theoretical_loss": 3.402660103516476, + "tokens_seen": 2206597120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705050505050503, + "loss": 2.87, + "theoretical_loss": 3.402651839543094, + "tokens_seen": 2206662656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703030303030304, + "loss": 2.4863, + "theoretical_loss": 3.4026435758838596, + "tokens_seen": 2206728192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701010101010105, + "loss": 2.8357, + "theoretical_loss": 3.4026353125387527, + "tokens_seen": 2206793728 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336989898989899, + "loss": 2.7257, + "theoretical_loss": 3.4026270495077515, + "tokens_seen": 2206859264 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033696969696969697, + "loss": 2.4505, + "theoretical_loss": 3.4026187867908346, + "tokens_seen": 2206924800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369494949494949, + "loss": 2.6195, + "theoretical_loss": 3.4026105243879807, + "tokens_seen": 2206990336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033692929292929294, + "loss": 2.9169, + "theoretical_loss": 3.402602262299169, + "tokens_seen": 2207055872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033690909090909095, + "loss": 2.6992, + "theoretical_loss": 3.4025940005243775, + "tokens_seen": 2207121408 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1250567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6240193843841553, + "objective/train/theoretical_loss": 3.4025857390635856, + "objective/train/tokens_used": 566046176, + "theoretical_loss": 3.4025857390635856, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368888888888889, + "loss": 2.6944, + "theoretical_loss": 3.4025857390635856, + "tokens_seen": 2207186944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033686868686868686, + "loss": 2.7155, + "theoretical_loss": 3.402577477916772, + "tokens_seen": 2207252480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033684848484848487, + "loss": 2.6411, + "theoretical_loss": 3.402569217083915, + "tokens_seen": 2207318016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033682828282828283, + "loss": 2.582, + "theoretical_loss": 3.4025609565649937, + "tokens_seen": 2207383552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033680808080808084, + "loss": 2.4495, + "theoretical_loss": 3.4025526963599866, + "tokens_seen": 2207449088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367878787878788, + "loss": 2.8278, + "theoretical_loss": 3.402544436468873, + "tokens_seen": 2207514624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033676767676767675, + "loss": 2.817, + "theoretical_loss": 3.402536176891631, + "tokens_seen": 2207580160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033674747474747476, + "loss": 2.8605, + "theoretical_loss": 3.4025279176282397, + "tokens_seen": 2207645696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367272727272727, + "loss": 2.6345, + "theoretical_loss": 3.4025196586786777, + "tokens_seen": 2207711232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670707070707073, + "loss": 2.57, + "theoretical_loss": 3.4025114000429237, + "tokens_seen": 2207776768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366868686868687, + "loss": 2.9448, + "theoretical_loss": 3.402503141720957, + "tokens_seen": 2207842304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366666666666667, + "loss": 2.8507, + "theoretical_loss": 3.4024948837127558, + "tokens_seen": 2207907840 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033664646464646466, + "loss": 2.7031, + "theoretical_loss": 3.4024866260182987, + "tokens_seen": 2207973376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366262626262626, + "loss": 2.7027, + "theoretical_loss": 3.4024783686375653, + "tokens_seen": 2208038912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366060606060606, + "loss": 2.6137, + "theoretical_loss": 3.4024701115705334, + "tokens_seen": 2208104448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365858585858586, + "loss": 2.5104, + "theoretical_loss": 3.4024618548171826, + "tokens_seen": 2208169984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365656565656566, + "loss": 2.4352, + "theoretical_loss": 3.402453598377491, + "tokens_seen": 2208235520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654545454545455, + "loss": 2.6834, + "theoretical_loss": 3.402445342251438, + "tokens_seen": 2208301056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365252525252525, + "loss": 2.7838, + "theoretical_loss": 3.4024370864390017, + "tokens_seen": 2208366592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365050505050505, + "loss": 2.8884, + "theoretical_loss": 3.4024288309401616, + "tokens_seen": 2208432128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648484848484853, + "loss": 2.8113, + "theoretical_loss": 3.402420575754896, + "tokens_seen": 2208497664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364646464646465, + "loss": 2.6378, + "theoretical_loss": 3.4024123208831836, + "tokens_seen": 2208563200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033644444444444444, + "loss": 2.4192, + "theoretical_loss": 3.402404066325004, + "tokens_seen": 2208628736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364242424242424, + "loss": 2.3226, + "theoretical_loss": 3.402395812080335, + "tokens_seen": 2208694272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364040404040404, + "loss": 2.5338, + "theoretical_loss": 3.4023875581491554, + "tokens_seen": 2208759808 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1251703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1004152297973633, + "objective/train/theoretical_loss": 3.4023793045314448, + "objective/train/tokens_used": 567684576, + "theoretical_loss": 3.4023793045314448, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363838383838384, + "loss": 2.9513, + "theoretical_loss": 3.4023793045314448, + "tokens_seen": 2208825344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363636363636364, + "loss": 2.505, + "theoretical_loss": 3.402371051227181, + "tokens_seen": 2208890880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033634343434343434, + "loss": 2.6219, + "theoretical_loss": 3.4023627982363442, + "tokens_seen": 2208956416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363232323232323, + "loss": 2.7752, + "theoretical_loss": 3.402354545558912, + "tokens_seen": 2209021952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033630303030303036, + "loss": 2.8439, + "theoretical_loss": 3.4023462931948636, + "tokens_seen": 2209087488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362828282828283, + "loss": 2.7192, + "theoretical_loss": 3.4023380411441777, + "tokens_seen": 2209153024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033626262626262627, + "loss": 2.5521, + "theoretical_loss": 3.402329789406833, + "tokens_seen": 2209218560 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624242424242423, + "loss": 2.5508, + "theoretical_loss": 3.402321537982809, + "tokens_seen": 2209284096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362222222222222, + "loss": 2.5948, + "theoretical_loss": 3.402313286872084, + "tokens_seen": 2209349632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620202020202025, + "loss": 2.7779, + "theoretical_loss": 3.4023050360746363, + "tokens_seen": 2209415168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361818181818182, + "loss": 2.6844, + "theoretical_loss": 3.4022967855904453, + "tokens_seen": 2209480704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616161616161616, + "loss": 2.8779, + "theoretical_loss": 3.4022885354194896, + "tokens_seen": 2209546240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361414141414141, + "loss": 2.6664, + "theoretical_loss": 3.4022802855617487, + "tokens_seen": 2209611776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033612121212121213, + "loss": 2.6672, + "theoretical_loss": 3.4022720360172007, + "tokens_seen": 2209677312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033610101010101014, + "loss": 2.4632, + "theoretical_loss": 3.402263786785824, + "tokens_seen": 2209742848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360808080808081, + "loss": 2.6818, + "theoretical_loss": 3.4022555378675987, + "tokens_seen": 2209808384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033606060606060606, + "loss": 2.539, + "theoretical_loss": 3.402247289262503, + "tokens_seen": 2209873920 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336040404040404, + "loss": 2.4487, + "theoretical_loss": 3.402239040970515, + "tokens_seen": 2209939456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336020202020202, + "loss": 2.5745, + "theoretical_loss": 3.4022307929916145, + "tokens_seen": 2210004992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600000000000004, + "loss": 2.605, + "theoretical_loss": 3.40222254532578, + "tokens_seen": 2210070528 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335979797979798, + "loss": 2.8467, + "theoretical_loss": 3.4022142979729906, + "tokens_seen": 2210136064 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033595959595959595, + "loss": 2.7019, + "theoretical_loss": 3.402206050933225, + "tokens_seen": 2210201600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033593939393939396, + "loss": 2.4772, + "theoretical_loss": 3.4021978042064616, + "tokens_seen": 2210267136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359191919191919, + "loss": 2.5822, + "theoretical_loss": 3.4021895577926795, + "tokens_seen": 2210332672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033589898989898993, + "loss": 2.8028, + "theoretical_loss": 3.402181311691858, + "tokens_seen": 2210398208 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 1252332, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4775390625, + "objective/train/theoretical_loss": 3.402173065903975, + "objective/train/tokens_used": 569322976, + "theoretical_loss": 3.402173065903975, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358787878787879, + "loss": 2.65, + "theoretical_loss": 3.402173065903975, + "tokens_seen": 2210463744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033585858585858584, + "loss": 2.6793, + "theoretical_loss": 3.4021648204290105, + "tokens_seen": 2210529280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033583838383838385, + "loss": 2.7478, + "theoretical_loss": 3.4021565752669423, + "tokens_seen": 2210594816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358181818181818, + "loss": 2.5129, + "theoretical_loss": 3.40214833041775, + "tokens_seen": 2210660352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357979797979798, + "loss": 2.553, + "theoretical_loss": 3.4021400858814124, + "tokens_seen": 2210725888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357777777777778, + "loss": 2.4985, + "theoretical_loss": 3.4021318416579076, + "tokens_seen": 2210791424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033575757575757573, + "loss": 2.475, + "theoretical_loss": 3.4021235977472153, + "tokens_seen": 2210856960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033573737373737375, + "loss": 2.6945, + "theoretical_loss": 3.4021153541493137, + "tokens_seen": 2210922496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357171717171717, + "loss": 2.7183, + "theoretical_loss": 3.4021071108641823, + "tokens_seen": 2210988032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356969696969697, + "loss": 2.3744, + "theoretical_loss": 3.4020988678918, + "tokens_seen": 2211053568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033567676767676767, + "loss": 2.6539, + "theoretical_loss": 3.4020906252321446, + "tokens_seen": 2211119104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356565656565657, + "loss": 2.4496, + "theoretical_loss": 3.4020823828851956, + "tokens_seen": 2211184640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033563636363636364, + "loss": 2.7619, + "theoretical_loss": 3.4020741408509325, + "tokens_seen": 2211250176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356161616161616, + "loss": 2.8439, + "theoretical_loss": 3.4020658991293335, + "tokens_seen": 2211315712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355959595959596, + "loss": 2.8032, + "theoretical_loss": 3.402057657720378, + "tokens_seen": 2211381248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033557575757575756, + "loss": 2.6485, + "theoretical_loss": 3.4020494166240436, + "tokens_seen": 2211446784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355555555555556, + "loss": 2.7595, + "theoretical_loss": 3.402041175840311, + "tokens_seen": 2211512320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033553535353535353, + "loss": 2.5337, + "theoretical_loss": 3.4020329353691574, + "tokens_seen": 2211577856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355151515151515, + "loss": 2.6897, + "theoretical_loss": 3.402024695210563, + "tokens_seen": 2211643392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354949494949495, + "loss": 2.495, + "theoretical_loss": 3.4020164553645054, + "tokens_seen": 2211708928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354747474747475, + "loss": 2.6053, + "theoretical_loss": 3.4020082158309646, + "tokens_seen": 2211774464 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033545454545454547, + "loss": 2.4451, + "theoretical_loss": 3.4019999766099187, + "tokens_seen": 2211840000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354343434343434, + "loss": 2.7346, + "theoretical_loss": 3.4019917377013473, + "tokens_seen": 2211905536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354141414141414, + "loss": 2.6537, + "theoretical_loss": 3.401983499105229, + "tokens_seen": 2211971072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353939393939394, + "loss": 2.6245, + "theoretical_loss": 3.4019752608215423, + "tokens_seen": 2212036608 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1253043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.915820598602295, + "objective/train/theoretical_loss": 3.401967022850267, + "objective/train/tokens_used": 570961376, + "theoretical_loss": 3.401967022850267, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353737373737374, + "loss": 2.7949, + "theoretical_loss": 3.401967022850267, + "tokens_seen": 2212102144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033535353535353536, + "loss": 2.7331, + "theoretical_loss": 3.401958785191381, + "tokens_seen": 2212167680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353333333333333, + "loss": 2.5328, + "theoretical_loss": 3.401950547844864, + "tokens_seen": 2212233216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033531313131313133, + "loss": 2.5384, + "theoretical_loss": 3.4019423108106945, + "tokens_seen": 2212298752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033529292929292934, + "loss": 2.6535, + "theoretical_loss": 3.4019340740888513, + "tokens_seen": 2212364288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352727272727273, + "loss": 2.5137, + "theoretical_loss": 3.401925837679313, + "tokens_seen": 2212429824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033525252525252525, + "loss": 2.5757, + "theoretical_loss": 3.40191760158206, + "tokens_seen": 2212495360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352323232323232, + "loss": 2.6798, + "theoretical_loss": 3.401909365797069, + "tokens_seen": 2212560896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352121212121212, + "loss": 2.6133, + "theoretical_loss": 3.4019011303243207, + "tokens_seen": 2212626432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033519191919191923, + "loss": 2.6383, + "theoretical_loss": 3.4018928951637935, + "tokens_seen": 2212691968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351717171717172, + "loss": 2.4688, + "theoretical_loss": 3.401884660315466, + "tokens_seen": 2212757504 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033515151515151515, + "loss": 2.6192, + "theoretical_loss": 3.4018764257793173, + "tokens_seen": 2212823040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351313131313131, + "loss": 2.6387, + "theoretical_loss": 3.4018681915553266, + "tokens_seen": 2212888576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033511111111111117, + "loss": 2.7996, + "theoretical_loss": 3.4018599576434725, + "tokens_seen": 2212954112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350909090909091, + "loss": 2.5716, + "theoretical_loss": 3.4018517240437336, + "tokens_seen": 2213019648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350707070707071, + "loss": 2.5923, + "theoretical_loss": 3.4018434907560895, + "tokens_seen": 2213085184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033505050505050504, + "loss": 2.6287, + "theoretical_loss": 3.401835257780519, + "tokens_seen": 2213150720 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335030303030303, + "loss": 2.6159, + "theoretical_loss": 3.401827025117001, + "tokens_seen": 2213216256 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033501010101010106, + "loss": 2.7077, + "theoretical_loss": 3.401818792765514, + "tokens_seen": 2213281792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334989898989899, + "loss": 2.665, + "theoretical_loss": 3.4018105607260374, + "tokens_seen": 2213347328 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334969696969697, + "loss": 2.7393, + "theoretical_loss": 3.40180232899855, + "tokens_seen": 2213412864 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494949494949493, + "loss": 2.761, + "theoretical_loss": 3.4017940975830308, + "tokens_seen": 2213478400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033492929292929294, + "loss": 2.6637, + "theoretical_loss": 3.4017858664794582, + "tokens_seen": 2213543936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033490909090909095, + "loss": 2.5763, + "theoretical_loss": 3.401777635687812, + "tokens_seen": 2213609472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348888888888889, + "loss": 2.861, + "theoretical_loss": 3.4017694052080705, + "tokens_seen": 2213675008 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1254214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.186537027359009, + "objective/train/theoretical_loss": 3.4017611750402135, + "objective/train/tokens_used": 572599776, + "theoretical_loss": 3.4017611750402135, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486868686868687, + "loss": 2.5184, + "theoretical_loss": 3.4017611750402135, + "tokens_seen": 2213740544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348484848484848, + "loss": 2.6243, + "theoretical_loss": 3.401752945184219, + "tokens_seen": 2213806080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033482828282828283, + "loss": 2.6046, + "theoretical_loss": 3.401744715640066, + "tokens_seen": 2213871616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480808080808085, + "loss": 2.6436, + "theoretical_loss": 3.401736486407734, + "tokens_seen": 2213937152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347878787878788, + "loss": 2.6505, + "theoretical_loss": 3.4017282574872016, + "tokens_seen": 2214002688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476767676767676, + "loss": 2.4952, + "theoretical_loss": 3.401720028878448, + "tokens_seen": 2214068224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033474747474747477, + "loss": 2.8316, + "theoretical_loss": 3.4017118005814524, + "tokens_seen": 2214133760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347272727272727, + "loss": 2.6188, + "theoretical_loss": 3.4017035725961926, + "tokens_seen": 2214199296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033470707070707074, + "loss": 2.8762, + "theoretical_loss": 3.401695344922649, + "tokens_seen": 2214264832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346868686868687, + "loss": 2.5057, + "theoretical_loss": 3.4016871175607997, + "tokens_seen": 2214330368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033466666666666665, + "loss": 2.7337, + "theoretical_loss": 3.401678890510624, + "tokens_seen": 2214395904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464646464646466, + "loss": 2.4621, + "theoretical_loss": 3.4016706637721006, + "tokens_seen": 2214461440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346262626262626, + "loss": 2.758, + "theoretical_loss": 3.4016624373452085, + "tokens_seen": 2214526976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033460606060606063, + "loss": 2.5283, + "theoretical_loss": 3.401654211229927, + "tokens_seen": 2214592512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345858585858586, + "loss": 2.7303, + "theoretical_loss": 3.4016459854262346, + "tokens_seen": 2214658048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345656565656566, + "loss": 2.8137, + "theoretical_loss": 3.4016377599341108, + "tokens_seen": 2214723584 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033454545454545456, + "loss": 2.8683, + "theoretical_loss": 3.4016295347535346, + "tokens_seen": 2214789120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345252525252525, + "loss": 2.5288, + "theoretical_loss": 3.4016213098844843, + "tokens_seen": 2214854656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345050505050505, + "loss": 2.525, + "theoretical_loss": 3.4016130853269395, + "tokens_seen": 2214920192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344848484848485, + "loss": 2.3212, + "theoretical_loss": 3.4016048610808785, + "tokens_seen": 2214985728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344646464646465, + "loss": 2.5096, + "theoretical_loss": 3.401596637146281, + "tokens_seen": 2215051264 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033444444444444445, + "loss": 2.7059, + "theoretical_loss": 3.4015884135231262, + "tokens_seen": 2215116800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344242424242424, + "loss": 2.6614, + "theoretical_loss": 3.4015801902113925, + "tokens_seen": 2215182336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344040404040404, + "loss": 2.4486, + "theoretical_loss": 3.401571967211059, + "tokens_seen": 2215247872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438383838383843, + "loss": 2.4743, + "theoretical_loss": 3.4015637445221043, + "tokens_seen": 2215313408 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1254739, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.672135829925537, + "objective/train/theoretical_loss": 3.4015555221445086, + "objective/train/tokens_used": 574238176, + "theoretical_loss": 3.4015555221445086, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343636363636364, + "loss": 2.4682, + "theoretical_loss": 3.4015555221445086, + "tokens_seen": 2215378944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434343434343434, + "loss": 2.8005, + "theoretical_loss": 3.4015473000782497, + "tokens_seen": 2215444480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343232323232323, + "loss": 2.4226, + "theoretical_loss": 3.401539078323307, + "tokens_seen": 2215510016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343030303030303, + "loss": 2.6452, + "theoretical_loss": 3.40153085687966, + "tokens_seen": 2215575552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828282828283, + "loss": 2.5656, + "theoretical_loss": 3.4015226357472867, + "tokens_seen": 2215641088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342626262626263, + "loss": 2.3352, + "theoretical_loss": 3.401514414926167, + "tokens_seen": 2215706624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033424242424242423, + "loss": 2.4581, + "theoretical_loss": 3.401506194416279, + "tokens_seen": 2215772160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342222222222222, + "loss": 2.7919, + "theoretical_loss": 3.4014979742176026, + "tokens_seen": 2215837696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033420202020202026, + "loss": 2.8719, + "theoretical_loss": 3.401489754330117, + "tokens_seen": 2215903232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341818181818182, + "loss": 2.651, + "theoretical_loss": 3.4014815347538, + "tokens_seen": 2215968768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033416161616161617, + "loss": 2.625, + "theoretical_loss": 3.401473315488632, + "tokens_seen": 2216034304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341414141414141, + "loss": 2.8716, + "theoretical_loss": 3.4014650965345905, + "tokens_seen": 2216099840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341212121212121, + "loss": 2.6403, + "theoretical_loss": 3.401456877891656, + "tokens_seen": 2216165376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410101010101015, + "loss": 2.9009, + "theoretical_loss": 3.401448659559807, + "tokens_seen": 2216230912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340808080808081, + "loss": 2.6835, + "theoretical_loss": 3.401440441539022, + "tokens_seen": 2216296448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033406060606060606, + "loss": 2.703, + "theoretical_loss": 3.4014322238292807, + "tokens_seen": 2216361984 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334040404040404, + "loss": 2.6409, + "theoretical_loss": 3.401424006430562, + "tokens_seen": 2216427520 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334020202020202, + "loss": 2.4752, + "theoretical_loss": 3.4014157893428445, + "tokens_seen": 2216493056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033400000000000004, + "loss": 2.518, + "theoretical_loss": 3.4014075725661077, + "tokens_seen": 2216558592 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333979797979798, + "loss": 2.6102, + "theoretical_loss": 3.40139935610033, + "tokens_seen": 2216624128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033395959595959595, + "loss": 2.6201, + "theoretical_loss": 3.401391139945492, + "tokens_seen": 2216689664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339393939393939, + "loss": 2.5032, + "theoretical_loss": 3.401382924101571, + "tokens_seen": 2216755200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339191919191919, + "loss": 2.8221, + "theoretical_loss": 3.4013747085685466, + "tokens_seen": 2216820736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033389898989898993, + "loss": 2.6619, + "theoretical_loss": 3.4013664933463983, + "tokens_seen": 2216886272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338787878787879, + "loss": 2.7588, + "theoretical_loss": 3.4013582784351044, + "tokens_seen": 2216951808 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1256282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.823676109313965, + "objective/train/theoretical_loss": 3.401350063834645, + "objective/train/tokens_used": 575876576, + "theoretical_loss": 3.401350063834645, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033385858585858585, + "loss": 2.5961, + "theoretical_loss": 3.401350063834645, + "tokens_seen": 2217017344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338383838383838, + "loss": 2.59, + "theoretical_loss": 3.401341849544998, + "tokens_seen": 2217082880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338181818181818, + "loss": 2.6327, + "theoretical_loss": 3.401333635566143, + "tokens_seen": 2217148416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337979797979798, + "loss": 2.7019, + "theoretical_loss": 3.401325421898059, + "tokens_seen": 2217213952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337777777777778, + "loss": 2.6227, + "theoretical_loss": 3.4013172085407253, + "tokens_seen": 2217279488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033375757575757574, + "loss": 2.5799, + "theoretical_loss": 3.4013089954941202, + "tokens_seen": 2217345024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373737373737375, + "loss": 2.5448, + "theoretical_loss": 3.401300782758224, + "tokens_seen": 2217410560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337171717171717, + "loss": 2.682, + "theoretical_loss": 3.4012925703330144, + "tokens_seen": 2217476096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336969696969697, + "loss": 2.5357, + "theoretical_loss": 3.4012843582184717, + "tokens_seen": 2217541632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336767676767677, + "loss": 2.7892, + "theoretical_loss": 3.401276146414574, + "tokens_seen": 2217607168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033365656565656563, + "loss": 2.6843, + "theoretical_loss": 3.401267934921301, + "tokens_seen": 2217672704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033363636363636364, + "loss": 2.6447, + "theoretical_loss": 3.4012597237386313, + "tokens_seen": 2217738240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033361616161616166, + "loss": 2.8162, + "theoretical_loss": 3.401251512866544, + "tokens_seen": 2217803776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335959595959596, + "loss": 2.5531, + "theoretical_loss": 3.4012433023050193, + "tokens_seen": 2217869312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033357575757575757, + "loss": 2.6685, + "theoretical_loss": 3.4012350920540344, + "tokens_seen": 2217934848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335555555555556, + "loss": 2.6622, + "theoretical_loss": 3.4012268821135696, + "tokens_seen": 2218000384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353535353535354, + "loss": 2.3917, + "theoretical_loss": 3.4012186724836035, + "tokens_seen": 2218065920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033351515151515155, + "loss": 2.543, + "theoretical_loss": 3.401210463164116, + "tokens_seen": 2218131456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334949494949495, + "loss": 2.5133, + "theoretical_loss": 3.401202254155085, + "tokens_seen": 2218196992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033347474747474746, + "loss": 2.7828, + "theoretical_loss": 3.4011940454564904, + "tokens_seen": 2218262528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345454545454547, + "loss": 2.4516, + "theoretical_loss": 3.401185837068311, + "tokens_seen": 2218328064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033343434343434343, + "loss": 2.553, + "theoretical_loss": 3.401177628990526, + "tokens_seen": 2218393600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033341414141414144, + "loss": 2.7467, + "theoretical_loss": 3.4011694212231145, + "tokens_seen": 2218459136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333939393939394, + "loss": 2.6067, + "theoretical_loss": 3.401161213766055, + "tokens_seen": 2218524672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333737373737374, + "loss": 2.6676, + "theoretical_loss": 3.401153006619328, + "tokens_seen": 2218590208 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1256744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.782334089279175, + "objective/train/theoretical_loss": 3.4011447997829114, + "objective/train/tokens_used": 577514976, + "theoretical_loss": 3.4011447997829114, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033335353535353537, + "loss": 2.608, + "theoretical_loss": 3.4011447997829114, + "tokens_seen": 2218655744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333333333333333, + "loss": 2.4797, + "theoretical_loss": 3.4011365932567843, + "tokens_seen": 2218721280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331313131313133, + "loss": 2.4629, + "theoretical_loss": 3.4011283870409263, + "tokens_seen": 2218786816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332929292929293, + "loss": 2.6982, + "theoretical_loss": 3.4011201811353167, + "tokens_seen": 2218852352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332727272727273, + "loss": 2.5358, + "theoretical_loss": 3.4011119755399335, + "tokens_seen": 2218917888 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033325252525252526, + "loss": 2.8221, + "theoretical_loss": 3.4011037702547573, + "tokens_seen": 2218983424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332323232323232, + "loss": 2.7867, + "theoretical_loss": 3.4010955652797663, + "tokens_seen": 2219048960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332121212121212, + "loss": 2.5916, + "theoretical_loss": 3.4010873606149397, + "tokens_seen": 2219114496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319191919191924, + "loss": 2.6418, + "theoretical_loss": 3.4010791562602565, + "tokens_seen": 2219180032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331717171717172, + "loss": 2.7306, + "theoretical_loss": 3.4010709522156963, + "tokens_seen": 2219245568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315151515151515, + "loss": 2.617, + "theoretical_loss": 3.401062748481238, + "tokens_seen": 2219311104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331313131313131, + "loss": 2.4629, + "theoretical_loss": 3.4010545450568603, + "tokens_seen": 2219376640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331111111111111, + "loss": 2.6789, + "theoretical_loss": 3.4010463419425427, + "tokens_seen": 2219442176 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033309090909090913, + "loss": 2.7128, + "theoretical_loss": 3.4010381391382647, + "tokens_seen": 2219507712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330707070707071, + "loss": 2.6473, + "theoretical_loss": 3.401029936644005, + "tokens_seen": 2219573248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033305050505050504, + "loss": 2.449, + "theoretical_loss": 3.4010217344597424, + "tokens_seen": 2219638784 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333030303030303, + "loss": 2.5431, + "theoretical_loss": 3.4010135325854565, + "tokens_seen": 2219704320 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301010101010107, + "loss": 2.5827, + "theoretical_loss": 3.4010053310211266, + "tokens_seen": 2219769856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332989898989899, + "loss": 2.7213, + "theoretical_loss": 3.4009971297667314, + "tokens_seen": 2219835392 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332969696969697, + "loss": 2.7437, + "theoretical_loss": 3.40098892882225, + "tokens_seen": 2219900928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033294949494949494, + "loss": 2.6257, + "theoretical_loss": 3.4009807281876623, + "tokens_seen": 2219966464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329292929292929, + "loss": 2.705, + "theoretical_loss": 3.4009725278629466, + "tokens_seen": 2220032000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033290909090909096, + "loss": 2.4609, + "theoretical_loss": 3.400964327848082, + "tokens_seen": 2220097536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328888888888889, + "loss": 2.6532, + "theoretical_loss": 3.4009561281430485, + "tokens_seen": 2220163072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033286868686868687, + "loss": 2.2724, + "theoretical_loss": 3.4009479287478244, + "tokens_seen": 2220228608 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1258010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4495670795440674, + "objective/train/theoretical_loss": 3.4009397296623893, + "objective/train/tokens_used": 579153376, + "theoretical_loss": 3.4009397296623893, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033284848484848483, + "loss": 2.5152, + "theoretical_loss": 3.4009397296623893, + "tokens_seen": 2220294144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033282828282828284, + "loss": 2.5416, + "theoretical_loss": 3.400931530886722, + "tokens_seen": 2220359680 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033280808080808085, + "loss": 2.8524, + "theoretical_loss": 3.4009233324208026, + "tokens_seen": 2220425216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327878787878788, + "loss": 2.5791, + "theoretical_loss": 3.4009151342646087, + "tokens_seen": 2220490752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033276767676767676, + "loss": 2.5846, + "theoretical_loss": 3.400906936418121, + "tokens_seen": 2220556288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327474747474747, + "loss": 2.478, + "theoretical_loss": 3.400898738881317, + "tokens_seen": 2220621824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033272727272727273, + "loss": 2.7062, + "theoretical_loss": 3.400890541654178, + "tokens_seen": 2220687360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033270707070707074, + "loss": 2.7309, + "theoretical_loss": 3.400882344736681, + "tokens_seen": 2220752896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326868686868687, + "loss": 2.46, + "theoretical_loss": 3.400874148128807, + "tokens_seen": 2220818432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033266666666666666, + "loss": 2.7546, + "theoretical_loss": 3.4008659518305335, + "tokens_seen": 2220883968 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033264646464646467, + "loss": 2.4755, + "theoretical_loss": 3.400857755841841, + "tokens_seen": 2220949504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326262626262626, + "loss": 2.7279, + "theoretical_loss": 3.4008495601627082, + "tokens_seen": 2221015040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033260606060606064, + "loss": 2.4773, + "theoretical_loss": 3.4008413647931137, + "tokens_seen": 2221080576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325858585858586, + "loss": 2.8909, + "theoretical_loss": 3.400833169733038, + "tokens_seen": 2221146112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033256565656565655, + "loss": 2.9056, + "theoretical_loss": 3.400824974982459, + "tokens_seen": 2221211648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033254545454545456, + "loss": 2.9133, + "theoretical_loss": 3.4008167805413563, + "tokens_seen": 2221277184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325252525252525, + "loss": 2.6736, + "theoretical_loss": 3.4008085864097093, + "tokens_seen": 2221342720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033250505050505053, + "loss": 2.5253, + "theoretical_loss": 3.400800392587497, + "tokens_seen": 2221408256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324848484848485, + "loss": 2.789, + "theoretical_loss": 3.4007921990746985, + "tokens_seen": 2221473792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033246464646464644, + "loss": 2.5376, + "theoretical_loss": 3.4007840058712935, + "tokens_seen": 2221539328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033244444444444445, + "loss": 2.7073, + "theoretical_loss": 3.4007758129772605, + "tokens_seen": 2221604864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324242424242424, + "loss": 2.7318, + "theoretical_loss": 3.400767620392579, + "tokens_seen": 2221670400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324040404040404, + "loss": 2.5606, + "theoretical_loss": 3.400759428117228, + "tokens_seen": 2221735936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323838383838384, + "loss": 2.66, + "theoretical_loss": 3.4007512361511876, + "tokens_seen": 2221801472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323636363636364, + "loss": 2.6721, + "theoretical_loss": 3.400743044494436, + "tokens_seen": 2221867008 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1258716, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.233450174331665, + "objective/train/theoretical_loss": 3.400734853146952, + "objective/train/tokens_used": 580791776, + "theoretical_loss": 3.400734853146952, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033234343434343435, + "loss": 2.6685, + "theoretical_loss": 3.400734853146952, + "tokens_seen": 2221932544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323232323232323, + "loss": 2.6244, + "theoretical_loss": 3.400726662108716, + "tokens_seen": 2221998080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323030303030303, + "loss": 2.9015, + "theoretical_loss": 3.4007184713797067, + "tokens_seen": 2222063616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033228282828282827, + "loss": 2.6943, + "theoretical_loss": 3.4007102809599035, + "tokens_seen": 2222129152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322626262626263, + "loss": 2.7914, + "theoretical_loss": 3.4007020908492853, + "tokens_seen": 2222194688 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033224242424242424, + "loss": 2.7541, + "theoretical_loss": 3.4006939010478314, + "tokens_seen": 2222260224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322222222222222, + "loss": 2.6447, + "theoretical_loss": 3.400685711555521, + "tokens_seen": 2222325760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322020202020202, + "loss": 2.7967, + "theoretical_loss": 3.4006775223723333, + "tokens_seen": 2222391296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321818181818182, + "loss": 2.5322, + "theoretical_loss": 3.4006693334982474, + "tokens_seen": 2222456832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321616161616162, + "loss": 2.7112, + "theoretical_loss": 3.4006611449332427, + "tokens_seen": 2222522368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033214141414141413, + "loss": 2.568, + "theoretical_loss": 3.4006529566772987, + "tokens_seen": 2222587904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321212121212121, + "loss": 2.7354, + "theoretical_loss": 3.4006447687303942, + "tokens_seen": 2222653440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321010101010101, + "loss": 2.6841, + "theoretical_loss": 3.4006365810925088, + "tokens_seen": 2222718976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320808080808081, + "loss": 2.5798, + "theoretical_loss": 3.400628393763621, + "tokens_seen": 2222784512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033206060606060607, + "loss": 2.7183, + "theoretical_loss": 3.4006202067437106, + "tokens_seen": 2222850048 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332040404040404, + "loss": 2.5212, + "theoretical_loss": 3.400612020032757, + "tokens_seen": 2222915584 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332020202020202, + "loss": 2.6422, + "theoretical_loss": 3.400603833630739, + "tokens_seen": 2222981120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033200000000000005, + "loss": 2.692, + "theoretical_loss": 3.400595647537636, + "tokens_seen": 2223046656 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331979797979798, + "loss": 2.699, + "theoretical_loss": 3.4005874617534273, + "tokens_seen": 2223112192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195959595959596, + "loss": 2.7642, + "theoretical_loss": 3.400579276278092, + "tokens_seen": 2223177728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319393939393939, + "loss": 2.7477, + "theoretical_loss": 3.400571091111609, + "tokens_seen": 2223243264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319191919191919, + "loss": 2.7297, + "theoretical_loss": 3.400562906253959, + "tokens_seen": 2223308800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033189898989898994, + "loss": 2.9961, + "theoretical_loss": 3.4005547217051193, + "tokens_seen": 2223374336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318787878787879, + "loss": 2.6401, + "theoretical_loss": 3.4005465374650705, + "tokens_seen": 2223439872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033185858585858585, + "loss": 2.789, + "theoretical_loss": 3.400538353533791, + "tokens_seen": 2223505408 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1259970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.019413709640503, + "objective/train/theoretical_loss": 3.400530169911261, + "objective/train/tokens_used": 582430176, + "theoretical_loss": 3.400530169911261, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318383838383838, + "loss": 2.9013, + "theoretical_loss": 3.400530169911261, + "tokens_seen": 2223570944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318181818181819, + "loss": 2.6298, + "theoretical_loss": 3.4005219865974587, + "tokens_seen": 2223636480 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179797979797983, + "loss": 2.8161, + "theoretical_loss": 3.4005138035923643, + "tokens_seen": 2223702016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317777777777778, + "loss": 2.638, + "theoretical_loss": 3.4005056208959563, + "tokens_seen": 2223767552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175757575757575, + "loss": 2.6743, + "theoretical_loss": 3.4004974385082143, + "tokens_seen": 2223833088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317373737373737, + "loss": 2.6845, + "theoretical_loss": 3.400489256429118, + "tokens_seen": 2223898624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171717171717177, + "loss": 2.605, + "theoretical_loss": 3.4004810746586456, + "tokens_seen": 2223964160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316969696969697, + "loss": 2.7814, + "theoretical_loss": 3.400472893196777, + "tokens_seen": 2224029696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316767676767677, + "loss": 2.4626, + "theoretical_loss": 3.4004647120434917, + "tokens_seen": 2224095232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165656565656564, + "loss": 2.6126, + "theoretical_loss": 3.4004565311987687, + "tokens_seen": 2224160768 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163636363636365, + "loss": 2.6932, + "theoretical_loss": 3.400448350662587, + "tokens_seen": 2224226304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033161616161616166, + "loss": 2.5647, + "theoretical_loss": 3.4004401704349263, + "tokens_seen": 2224291840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315959595959596, + "loss": 2.2725, + "theoretical_loss": 3.400431990515766, + "tokens_seen": 2224357376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315757575757576, + "loss": 2.7128, + "theoretical_loss": 3.400423810905085, + "tokens_seen": 2224422912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155555555555553, + "loss": 2.6523, + "theoretical_loss": 3.400415631602862, + "tokens_seen": 2224488448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033153535353535354, + "loss": 2.5534, + "theoretical_loss": 3.4004074526090777, + "tokens_seen": 2224553984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033151515151515155, + "loss": 2.6877, + "theoretical_loss": 3.4003992739237106, + "tokens_seen": 2224619520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314949494949495, + "loss": 2.648, + "theoretical_loss": 3.4003910955467394, + "tokens_seen": 2224685056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033147474747474747, + "loss": 2.5846, + "theoretical_loss": 3.4003829174781446, + "tokens_seen": 2224750592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314545454545455, + "loss": 2.5761, + "theoretical_loss": 3.400374739717905, + "tokens_seen": 2224816128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033143434343434344, + "loss": 2.7559, + "theoretical_loss": 3.400366562265999, + "tokens_seen": 2224881664 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141414141414145, + "loss": 2.5709, + "theoretical_loss": 3.4003583851224075, + "tokens_seen": 2224947200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313939393939394, + "loss": 2.5195, + "theoretical_loss": 3.4003502082871084, + "tokens_seen": 2225012736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033137373737373736, + "loss": 2.5565, + "theoretical_loss": 3.400342031760082, + "tokens_seen": 2225078272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033135353535353537, + "loss": 2.6756, + "theoretical_loss": 3.400333855541307, + "tokens_seen": 2225143808 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1260594, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.626549482345581, + "objective/train/theoretical_loss": 3.400325679630763, + "objective/train/tokens_used": 584068576, + "theoretical_loss": 3.400325679630763, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133333333333333, + "loss": 2.8385, + "theoretical_loss": 3.400325679630763, + "tokens_seen": 2225209344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033131313131313134, + "loss": 2.6329, + "theoretical_loss": 3.400317504028429, + "tokens_seen": 2225274880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312929292929293, + "loss": 2.7757, + "theoretical_loss": 3.4003093287342843, + "tokens_seen": 2225340416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312727272727273, + "loss": 2.4993, + "theoretical_loss": 3.4003011537483085, + "tokens_seen": 2225405952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125252525252526, + "loss": 2.4161, + "theoretical_loss": 3.400292979070481, + "tokens_seen": 2225471488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312323232323232, + "loss": 2.5188, + "theoretical_loss": 3.4002848047007808, + "tokens_seen": 2225537024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121212121212123, + "loss": 2.599, + "theoretical_loss": 3.4002766306391874, + "tokens_seen": 2225602560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311919191919192, + "loss": 2.6916, + "theoretical_loss": 3.40026845688568, + "tokens_seen": 2225668096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311717171717172, + "loss": 2.7231, + "theoretical_loss": 3.400260283440238, + "tokens_seen": 2225733632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033115151515151516, + "loss": 2.5663, + "theoretical_loss": 3.4002521103028407, + "tokens_seen": 2225799168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311313131313131, + "loss": 2.5838, + "theoretical_loss": 3.400243937473467, + "tokens_seen": 2225864704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311111111111111, + "loss": 2.5847, + "theoretical_loss": 3.4002357649520967, + "tokens_seen": 2225930240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109090909090914, + "loss": 2.5194, + "theoretical_loss": 3.4002275927387093, + "tokens_seen": 2225995776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310707070707071, + "loss": 2.7003, + "theoretical_loss": 3.4002194208332837, + "tokens_seen": 2226061312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105050505050505, + "loss": 2.6966, + "theoretical_loss": 3.4002112492357996, + "tokens_seen": 2226126848 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331030303030303, + "loss": 2.7116, + "theoretical_loss": 3.400203077946236, + "tokens_seen": 2226192384 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331010101010101, + "loss": 2.6512, + "theoretical_loss": 3.4001949069645723, + "tokens_seen": 2226257920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033098989898989903, + "loss": 2.8617, + "theoretical_loss": 3.400186736290788, + "tokens_seen": 2226323456 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330969696969697, + "loss": 2.7989, + "theoretical_loss": 3.400178565924862, + "tokens_seen": 2226388992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033094949494949494, + "loss": 2.6806, + "theoretical_loss": 3.400170395866774, + "tokens_seen": 2226454528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309292929292929, + "loss": 2.5705, + "theoretical_loss": 3.400162226116504, + "tokens_seen": 2226520064 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033090909090909096, + "loss": 2.3398, + "theoretical_loss": 3.40015405667403, + "tokens_seen": 2226585600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308888888888889, + "loss": 2.4576, + "theoretical_loss": 3.4001458875393324, + "tokens_seen": 2226651136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308686868686869, + "loss": 2.8021, + "theoretical_loss": 3.40013771871239, + "tokens_seen": 2226716672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033084848484848483, + "loss": 2.6832, + "theoretical_loss": 3.4001295501931823, + "tokens_seen": 2226782208 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 1261463, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.56870436668396, + "objective/train/theoretical_loss": 3.4001213819816885, + "objective/train/tokens_used": 585706976, + "theoretical_loss": 3.4001213819816885, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308282828282828, + "loss": 2.5754, + "theoretical_loss": 3.4001213819816885, + "tokens_seen": 2226847744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033080808080808086, + "loss": 2.5879, + "theoretical_loss": 3.400113214077888, + "tokens_seen": 2226913280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307878787878788, + "loss": 2.6998, + "theoretical_loss": 3.40010504648176, + "tokens_seen": 2226978816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033076767676767677, + "loss": 2.672, + "theoretical_loss": 3.400096879193285, + "tokens_seen": 2227044352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033074747474747473, + "loss": 2.7426, + "theoretical_loss": 3.400088712212441, + "tokens_seen": 2227109888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307272727272727, + "loss": 2.5351, + "theoretical_loss": 3.4000805455392076, + "tokens_seen": 2227175424 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033070707070707075, + "loss": 2.5906, + "theoretical_loss": 3.4000723791735643, + "tokens_seen": 2227240960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306868686868687, + "loss": 2.7407, + "theoretical_loss": 3.400064213115491, + "tokens_seen": 2227306496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033066666666666666, + "loss": 2.5634, + "theoretical_loss": 3.4000560473649664, + "tokens_seen": 2227372032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306464646464646, + "loss": 2.4096, + "theoretical_loss": 3.40004788192197, + "tokens_seen": 2227437568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033062626262626263, + "loss": 2.75, + "theoretical_loss": 3.4000397167864813, + "tokens_seen": 2227503104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033060606060606064, + "loss": 2.6621, + "theoretical_loss": 3.40003155195848, + "tokens_seen": 2227568640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305858585858586, + "loss": 2.6591, + "theoretical_loss": 3.400023387437945, + "tokens_seen": 2227634176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033056565656565656, + "loss": 2.7133, + "theoretical_loss": 3.4000152232248553, + "tokens_seen": 2227699712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305454545454545, + "loss": 2.6764, + "theoretical_loss": 3.400007059319191, + "tokens_seen": 2227765248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305252525252525, + "loss": 2.7313, + "theoretical_loss": 3.3999988957209313, + "tokens_seen": 2227830784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033050505050505053, + "loss": 2.9252, + "theoretical_loss": 3.3999907324300556, + "tokens_seen": 2227896320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304848484848485, + "loss": 2.681, + "theoretical_loss": 3.399982569446543, + "tokens_seen": 2227961856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033046464646464645, + "loss": 2.5034, + "theoretical_loss": 3.399974406770373, + "tokens_seen": 2228027392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033044444444444446, + "loss": 2.5193, + "theoretical_loss": 3.3999662444015257, + "tokens_seen": 2228092928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304242424242424, + "loss": 2.8616, + "theoretical_loss": 3.399958082339979, + "tokens_seen": 2228158464 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033040404040404043, + "loss": 2.8335, + "theoretical_loss": 3.3999499205857138, + "tokens_seen": 2228224000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303838383838384, + "loss": 2.6102, + "theoretical_loss": 3.399941759138709, + "tokens_seen": 2228289536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033036363636363634, + "loss": 2.6157, + "theoretical_loss": 3.399933597998943, + "tokens_seen": 2228355072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033034343434343435, + "loss": 2.6534, + "theoretical_loss": 3.399925437166397, + "tokens_seen": 2228420608 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1262743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.575108289718628, + "objective/train/theoretical_loss": 3.399917276641049, + "objective/train/tokens_used": 587345376, + "theoretical_loss": 3.399917276641049, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303232323232323, + "loss": 2.6803, + "theoretical_loss": 3.399917276641049, + "tokens_seen": 2228486144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303030303030303, + "loss": 2.7423, + "theoretical_loss": 3.399909116422879, + "tokens_seen": 2228551680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302828282828283, + "loss": 2.4674, + "theoretical_loss": 3.399900956511866, + "tokens_seen": 2228617216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302626262626263, + "loss": 2.916, + "theoretical_loss": 3.3998927969079897, + "tokens_seen": 2228682752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024242424242424, + "loss": 2.8921, + "theoretical_loss": 3.3998846376112297, + "tokens_seen": 2228748288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302222222222222, + "loss": 2.613, + "theoretical_loss": 3.3998764786215654, + "tokens_seen": 2228813824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302020202020202, + "loss": 2.6942, + "theoretical_loss": 3.3998683199389754, + "tokens_seen": 2228879360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033018181818181817, + "loss": 2.577, + "theoretical_loss": 3.3998601615634403, + "tokens_seen": 2228944896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301616161616162, + "loss": 2.592, + "theoretical_loss": 3.3998520034949387, + "tokens_seen": 2229010432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033014141414141414, + "loss": 2.5101, + "theoretical_loss": 3.39984384573345, + "tokens_seen": 2229075968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033012121212121215, + "loss": 2.801, + "theoretical_loss": 3.3998356882789538, + "tokens_seen": 2229141504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301010101010101, + "loss": 2.7757, + "theoretical_loss": 3.39982753113143, + "tokens_seen": 2229207040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300808080808081, + "loss": 2.428, + "theoretical_loss": 3.3998193742908573, + "tokens_seen": 2229272576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300606060606061, + "loss": 2.5617, + "theoretical_loss": 3.3998112177572155, + "tokens_seen": 2229338112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033004040404040403, + "loss": 2.6023, + "theoretical_loss": 3.399803061530484, + "tokens_seen": 2229403648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033002020202020204, + "loss": 2.6558, + "theoretical_loss": 3.3997949056106425, + "tokens_seen": 2229469184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033, + "loss": 2.5469, + "theoretical_loss": 3.39978674999767, + "tokens_seen": 2229534720 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329979797979798, + "loss": 2.4859, + "theoretical_loss": 3.399778594691546, + "tokens_seen": 2229600256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032995959595959597, + "loss": 2.4937, + "theoretical_loss": 3.39977043969225, + "tokens_seen": 2229665792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299393939393939, + "loss": 2.5443, + "theoretical_loss": 3.3997622849997615, + "tokens_seen": 2229731328 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032991919191919193, + "loss": 2.6399, + "theoretical_loss": 3.3997541306140597, + "tokens_seen": 2229796864 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032989898989898995, + "loss": 2.2843, + "theoretical_loss": 3.399745976535124, + "tokens_seen": 2229862400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298787878787879, + "loss": 2.4625, + "theoretical_loss": 3.3997378227629347, + "tokens_seen": 2229927936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032985858585858586, + "loss": 2.6892, + "theoretical_loss": 3.39972966929747, + "tokens_seen": 2229993472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298383838383838, + "loss": 2.6601, + "theoretical_loss": 3.3997215161387104, + "tokens_seen": 2230059008 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1263995, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898665428161621, + "objective/train/theoretical_loss": 3.3997133632866348, + "objective/train/tokens_used": 588983776, + "theoretical_loss": 3.3997133632866348, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298181818181818, + "loss": 2.8505, + "theoretical_loss": 3.3997133632866348, + "tokens_seen": 2230124544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032979797979797984, + "loss": 2.564, + "theoretical_loss": 3.3997052107412227, + "tokens_seen": 2230190080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297777777777778, + "loss": 2.5633, + "theoretical_loss": 3.3996970585024537, + "tokens_seen": 2230255616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032975757575757575, + "loss": 2.7512, + "theoretical_loss": 3.399688906570307, + "tokens_seen": 2230321152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297373737373737, + "loss": 2.5605, + "theoretical_loss": 3.3996807549447627, + "tokens_seen": 2230386688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297171717171718, + "loss": 2.4088, + "theoretical_loss": 3.3996726036257994, + "tokens_seen": 2230452224 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032969696969696973, + "loss": 2.8117, + "theoretical_loss": 3.399664452613397, + "tokens_seen": 2230517760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296767676767677, + "loss": 2.7242, + "theoretical_loss": 3.3996563019075348, + "tokens_seen": 2230583296 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032965656565656564, + "loss": 2.8799, + "theoretical_loss": 3.3996481515081927, + "tokens_seen": 2230648832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296363636363636, + "loss": 2.7234, + "theoretical_loss": 3.3996400014153494, + "tokens_seen": 2230714368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032961616161616167, + "loss": 2.6987, + "theoretical_loss": 3.399631851628985, + "tokens_seen": 2230779904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295959595959596, + "loss": 2.6425, + "theoretical_loss": 3.399623702149079, + "tokens_seen": 2230845440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295757575757576, + "loss": 2.6497, + "theoretical_loss": 3.3996155529756105, + "tokens_seen": 2230910976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032955555555555554, + "loss": 2.6831, + "theoretical_loss": 3.399607404108559, + "tokens_seen": 2230976512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032953535353535355, + "loss": 2.5465, + "theoretical_loss": 3.3995992555479044, + "tokens_seen": 2231042048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032951515151515156, + "loss": 2.8919, + "theoretical_loss": 3.3995911072936256, + "tokens_seen": 2231107584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294949494949495, + "loss": 2.6038, + "theoretical_loss": 3.399582959345703, + "tokens_seen": 2231173120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032947474747474747, + "loss": 2.6294, + "theoretical_loss": 3.399574811704115, + "tokens_seen": 2231238656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032945454545454543, + "loss": 2.849, + "theoretical_loss": 3.3995666643688414, + "tokens_seen": 2231304192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032943434343434344, + "loss": 2.8412, + "theoretical_loss": 3.399558517339862, + "tokens_seen": 2231369728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032941414141414145, + "loss": 2.6091, + "theoretical_loss": 3.399550370617156, + "tokens_seen": 2231435264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293939393939394, + "loss": 2.6846, + "theoretical_loss": 3.3995422242007036, + "tokens_seen": 2231500800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032937373737373737, + "loss": 2.6389, + "theoretical_loss": 3.399534078090483, + "tokens_seen": 2231566336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293535353535354, + "loss": 2.7995, + "theoretical_loss": 3.399525932286475, + "tokens_seen": 2231631872 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032933333333333333, + "loss": 2.6587, + "theoretical_loss": 3.3995177867886577, + "tokens_seen": 2231697408 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1264337, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1047985553741455, + "objective/train/theoretical_loss": 3.399509641597012, + "objective/train/tokens_used": 590622176, + "theoretical_loss": 3.399509641597012, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032931313131313134, + "loss": 2.7655, + "theoretical_loss": 3.399509641597012, + "tokens_seen": 2231762944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292929292929293, + "loss": 2.6674, + "theoretical_loss": 3.399501496711517, + "tokens_seen": 2231828480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032927272727272726, + "loss": 2.8853, + "theoretical_loss": 3.3994933521321515, + "tokens_seen": 2231894016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032925252525252527, + "loss": 2.8517, + "theoretical_loss": 3.3994852078588957, + "tokens_seen": 2231959552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292323232323232, + "loss": 2.7071, + "theoretical_loss": 3.3994770638917293, + "tokens_seen": 2232025088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032921212121212124, + "loss": 2.6889, + "theoretical_loss": 3.399468920230631, + "tokens_seen": 2232090624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291919191919192, + "loss": 2.7389, + "theoretical_loss": 3.3994607768755807, + "tokens_seen": 2232156160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032917171717171715, + "loss": 2.5154, + "theoretical_loss": 3.399452633826558, + "tokens_seen": 2232221696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032915151515151516, + "loss": 2.4387, + "theoretical_loss": 3.3994444910835426, + "tokens_seen": 2232287232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291313131313131, + "loss": 2.8881, + "theoretical_loss": 3.3994363486465136, + "tokens_seen": 2232352768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032911111111111113, + "loss": 2.7844, + "theoretical_loss": 3.399428206515451, + "tokens_seen": 2232418304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290909090909091, + "loss": 2.7049, + "theoretical_loss": 3.3994200646903336, + "tokens_seen": 2232483840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290707070707071, + "loss": 2.1456, + "theoretical_loss": 3.3994119231711415, + "tokens_seen": 2232549376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032905050505050505, + "loss": 2.5169, + "theoretical_loss": 3.3994037819578544, + "tokens_seen": 2232614912 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329030303030303, + "loss": 2.5953, + "theoretical_loss": 3.399395641050451, + "tokens_seen": 2232680448 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329010101010101, + "loss": 2.549, + "theoretical_loss": 3.3993875004489116, + "tokens_seen": 2232745984 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328989898989899, + "loss": 2.8874, + "theoretical_loss": 3.3993793601532154, + "tokens_seen": 2232811520 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328969696969697, + "loss": 2.6855, + "theoretical_loss": 3.399371220163342, + "tokens_seen": 2232877056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032894949494949495, + "loss": 2.4392, + "theoretical_loss": 3.399363080479271, + "tokens_seen": 2232942592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289292929292929, + "loss": 2.6024, + "theoretical_loss": 3.399354941100982, + "tokens_seen": 2233008128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289090909090909, + "loss": 2.6799, + "theoretical_loss": 3.399346802028454, + "tokens_seen": 2233073664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288888888888889, + "loss": 2.6031, + "theoretical_loss": 3.3993386632616676, + "tokens_seen": 2233139200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288686868686869, + "loss": 2.854, + "theoretical_loss": 3.399330524800601, + "tokens_seen": 2233204736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032884848484848484, + "loss": 2.7105, + "theoretical_loss": 3.3993223866452347, + "tokens_seen": 2233270272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288282828282828, + "loss": 2.6874, + "theoretical_loss": 3.399314248795548, + "tokens_seen": 2233335808 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1265500, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5811409950256348, + "objective/train/theoretical_loss": 3.3993061112515206, + "objective/train/tokens_used": 592260576, + "theoretical_loss": 3.3993061112515206, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288080808080808, + "loss": 2.53, + "theoretical_loss": 3.3993061112515206, + "tokens_seen": 2233401344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287878787878788, + "loss": 2.4328, + "theoretical_loss": 3.3992979740131317, + "tokens_seen": 2233466880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287676767676768, + "loss": 2.5813, + "theoretical_loss": 3.399289837080361, + "tokens_seen": 2233532416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874747474747473, + "loss": 2.5448, + "theoretical_loss": 3.399281700453188, + "tokens_seen": 2233597952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287272727272727, + "loss": 2.6082, + "theoretical_loss": 3.399273564131593, + "tokens_seen": 2233663488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032870707070707075, + "loss": 2.6965, + "theoretical_loss": 3.3992654281155543, + "tokens_seen": 2233729024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286868686868687, + "loss": 2.5803, + "theoretical_loss": 3.399257292405052, + "tokens_seen": 2233794560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866666666666667, + "loss": 2.5372, + "theoretical_loss": 3.399249157000066, + "tokens_seen": 2233860096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286464646464646, + "loss": 2.7925, + "theoretical_loss": 3.3992410219005755, + "tokens_seen": 2233925632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286262626262626, + "loss": 2.835, + "theoretical_loss": 3.3992328871065602, + "tokens_seen": 2233991168 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032860606060606065, + "loss": 2.6365, + "theoretical_loss": 3.3992247526179997, + "tokens_seen": 2234056704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285858585858586, + "loss": 2.6333, + "theoretical_loss": 3.3992166184348735, + "tokens_seen": 2234122240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032856565656565656, + "loss": 2.703, + "theoretical_loss": 3.3992084845571613, + "tokens_seen": 2234187776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285454545454545, + "loss": 2.8745, + "theoretical_loss": 3.399200350984842, + "tokens_seen": 2234253312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852525252525253, + "loss": 2.4651, + "theoretical_loss": 3.3991922177178964, + "tokens_seen": 2234318848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850505050505054, + "loss": 2.3866, + "theoretical_loss": 3.399184084756303, + "tokens_seen": 2234384384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284848484848485, + "loss": 2.8675, + "theoretical_loss": 3.3991759521000415, + "tokens_seen": 2234449920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032846464646464645, + "loss": 2.7956, + "theoretical_loss": 3.399167819749092, + "tokens_seen": 2234515456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284444444444444, + "loss": 2.655, + "theoretical_loss": 3.3991596877034342, + "tokens_seen": 2234580992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284242424242425, + "loss": 2.5362, + "theoretical_loss": 3.399151555963047, + "tokens_seen": 2234646528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032840404040404043, + "loss": 2.7459, + "theoretical_loss": 3.3991434245279106, + "tokens_seen": 2234712064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283838383838384, + "loss": 2.4476, + "theoretical_loss": 3.399135293398004, + "tokens_seen": 2234777600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836363636363635, + "loss": 2.8124, + "theoretical_loss": 3.3991271625733073, + "tokens_seen": 2234843136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032834343434343436, + "loss": 2.6475, + "theoretical_loss": 3.3991190320538, + "tokens_seen": 2234908672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832323232323237, + "loss": 2.6627, + "theoretical_loss": 3.399110901839461, + "tokens_seen": 2234974208 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1266280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.79852557182312, + "objective/train/theoretical_loss": 3.3991027719302713, + "objective/train/tokens_used": 593898976, + "theoretical_loss": 3.3991027719302713, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283030303030303, + "loss": 2.6414, + "theoretical_loss": 3.3991027719302713, + "tokens_seen": 2235039744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282828282828283, + "loss": 2.8331, + "theoretical_loss": 3.399094642326209, + "tokens_seen": 2235105280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032826262626262624, + "loss": 2.8807, + "theoretical_loss": 3.399086513027255, + "tokens_seen": 2235170816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824242424242425, + "loss": 2.7771, + "theoretical_loss": 3.399078384033388, + "tokens_seen": 2235236352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032822222222222226, + "loss": 2.6771, + "theoretical_loss": 3.3990702553445877, + "tokens_seen": 2235301888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282020202020202, + "loss": 2.5668, + "theoretical_loss": 3.399062126960834, + "tokens_seen": 2235367424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281818181818182, + "loss": 2.6998, + "theoretical_loss": 3.3990539988821067, + "tokens_seen": 2235432960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281616161616162, + "loss": 2.6145, + "theoretical_loss": 3.399045871108385, + "tokens_seen": 2235498496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032814141414141414, + "loss": 2.6813, + "theoretical_loss": 3.399037743639649, + "tokens_seen": 2235564032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812121212121215, + "loss": 2.6523, + "theoretical_loss": 3.399029616475877, + "tokens_seen": 2235629568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281010101010101, + "loss": 2.6534, + "theoretical_loss": 3.3990214896170503, + "tokens_seen": 2235695104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032808080808080807, + "loss": 2.7032, + "theoretical_loss": 3.3990133630631476, + "tokens_seen": 2235760640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280606060606061, + "loss": 2.6784, + "theoretical_loss": 3.399005236814149, + "tokens_seen": 2235826176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032804040404040404, + "loss": 2.6735, + "theoretical_loss": 3.3989971108700336, + "tokens_seen": 2235891712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802020202020205, + "loss": 2.6527, + "theoretical_loss": 3.3989889852307815, + "tokens_seen": 2235957248 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328, + "loss": 2.8622, + "theoretical_loss": 3.3989808598963718, + "tokens_seen": 2236022784 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327979797979798, + "loss": 2.5165, + "theoretical_loss": 3.3989727348667844, + "tokens_seen": 2236088320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032795959595959597, + "loss": 2.7449, + "theoretical_loss": 3.398964610141999, + "tokens_seen": 2236153856 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032793939393939393, + "loss": 2.8434, + "theoretical_loss": 3.3989564857219956, + "tokens_seen": 2236219392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032791919191919194, + "loss": 2.7358, + "theoretical_loss": 3.398948361606753, + "tokens_seen": 2236284928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278989898989899, + "loss": 2.8235, + "theoretical_loss": 3.398940237796251, + "tokens_seen": 2236350464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278787878787879, + "loss": 2.8465, + "theoretical_loss": 3.3989321142904703, + "tokens_seen": 2236416000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032785858585858586, + "loss": 2.6328, + "theoretical_loss": 3.398923991089389, + "tokens_seen": 2236481536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278383838383838, + "loss": 2.6346, + "theoretical_loss": 3.3989158681929883, + "tokens_seen": 2236547072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032781818181818183, + "loss": 2.9283, + "theoretical_loss": 3.3989077456012464, + "tokens_seen": 2236612608 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1267649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7353851795196533, + "objective/train/theoretical_loss": 3.3988996233141435, + "objective/train/tokens_used": 595537376, + "theoretical_loss": 3.3988996233141435, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032779797979797984, + "loss": 2.7969, + "theoretical_loss": 3.3988996233141435, + "tokens_seen": 2236678144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277777777777778, + "loss": 2.8321, + "theoretical_loss": 3.3988915013316596, + "tokens_seen": 2236743680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775757575757576, + "loss": 2.6709, + "theoretical_loss": 3.398883379653774, + "tokens_seen": 2236809216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277373737373737, + "loss": 2.7122, + "theoretical_loss": 3.3988752582804667, + "tokens_seen": 2236874752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277171717171717, + "loss": 2.5904, + "theoretical_loss": 3.3988671372117167, + "tokens_seen": 2236940288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032769696969696974, + "loss": 2.8814, + "theoretical_loss": 3.398859016447504, + "tokens_seen": 2237005824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276767676767677, + "loss": 2.4426, + "theoretical_loss": 3.3988508959878088, + "tokens_seen": 2237071360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032765656565656565, + "loss": 2.8617, + "theoretical_loss": 3.39884277583261, + "tokens_seen": 2237136896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276363636363636, + "loss": 2.9015, + "theoretical_loss": 3.3988346559818874, + "tokens_seen": 2237202432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032761616161616167, + "loss": 2.7819, + "theoretical_loss": 3.3988265364356205, + "tokens_seen": 2237267968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032759595959595963, + "loss": 2.7566, + "theoretical_loss": 3.3988184171937896, + "tokens_seen": 2237333504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275757575757576, + "loss": 2.8899, + "theoretical_loss": 3.3988102982563744, + "tokens_seen": 2237399040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032755555555555554, + "loss": 2.7122, + "theoretical_loss": 3.3988021796233534, + "tokens_seen": 2237464576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275353535353535, + "loss": 2.8097, + "theoretical_loss": 3.3987940612947076, + "tokens_seen": 2237530112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032751515151515156, + "loss": 2.5946, + "theoretical_loss": 3.398785943270416, + "tokens_seen": 2237595648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274949494949495, + "loss": 2.68, + "theoretical_loss": 3.3987778255504586, + "tokens_seen": 2237661184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274747474747475, + "loss": 2.7646, + "theoretical_loss": 3.3987697081348145, + "tokens_seen": 2237726720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032745454545454544, + "loss": 2.6223, + "theoretical_loss": 3.3987615910234643, + "tokens_seen": 2237792256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274343434343434, + "loss": 2.6616, + "theoretical_loss": 3.3987534742163867, + "tokens_seen": 2237857792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032741414141414146, + "loss": 2.7193, + "theoretical_loss": 3.398745357713562, + "tokens_seen": 2237923328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273939393939394, + "loss": 2.4738, + "theoretical_loss": 3.39873724151497, + "tokens_seen": 2237988864 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032737373737373737, + "loss": 2.591, + "theoretical_loss": 3.39872912562059, + "tokens_seen": 2238054400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032735353535353533, + "loss": 2.6333, + "theoretical_loss": 3.3987210100304015, + "tokens_seen": 2238119936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032733333333333334, + "loss": 2.6459, + "theoretical_loss": 3.3987128947443845, + "tokens_seen": 2238185472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731313131313135, + "loss": 2.4616, + "theoretical_loss": 3.398704779762519, + "tokens_seen": 2238251008 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1268187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8852992057800293, + "objective/train/theoretical_loss": 3.398696665084784, + "objective/train/tokens_used": 597175776, + "theoretical_loss": 3.398696665084784, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272929292929293, + "loss": 2.6175, + "theoretical_loss": 3.398696665084784, + "tokens_seen": 2238316544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032727272727272726, + "loss": 2.5367, + "theoretical_loss": 3.39868855071116, + "tokens_seen": 2238382080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272525252525252, + "loss": 2.7875, + "theoretical_loss": 3.398680436641626, + "tokens_seen": 2238447616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032723232323232323, + "loss": 2.752, + "theoretical_loss": 3.398672322876162, + "tokens_seen": 2238513152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032721212121212124, + "loss": 2.6511, + "theoretical_loss": 3.398664209414748, + "tokens_seen": 2238578688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271919191919192, + "loss": 2.7713, + "theoretical_loss": 3.398656096257363, + "tokens_seen": 2238644224 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032717171717171716, + "loss": 2.5673, + "theoretical_loss": 3.398647983403987, + "tokens_seen": 2238709760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032715151515151517, + "loss": 2.6595, + "theoretical_loss": 3.3986398708546, + "tokens_seen": 2238775296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271313131313131, + "loss": 2.3959, + "theoretical_loss": 3.3986317586091817, + "tokens_seen": 2238840832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032711111111111114, + "loss": 2.426, + "theoretical_loss": 3.398623646667712, + "tokens_seen": 2238906368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270909090909091, + "loss": 2.5436, + "theoretical_loss": 3.3986155350301694, + "tokens_seen": 2238971904 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032707070707070705, + "loss": 2.8047, + "theoretical_loss": 3.398607423696535, + "tokens_seen": 2239037440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032705050505050506, + "loss": 2.788, + "theoretical_loss": 3.3985993126667875, + "tokens_seen": 2239102976 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327030303030303, + "loss": 2.5166, + "theoretical_loss": 3.3985912019409077, + "tokens_seen": 2239168512 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032701010101010103, + "loss": 2.6686, + "theoretical_loss": 3.3985830915188746, + "tokens_seen": 2239234048 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326989898989899, + "loss": 2.7931, + "theoretical_loss": 3.398574981400668, + "tokens_seen": 2239299584 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326969696969697, + "loss": 2.5386, + "theoretical_loss": 3.3985668715862674, + "tokens_seen": 2239365120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032694949494949495, + "loss": 2.7573, + "theoretical_loss": 3.398558762075653, + "tokens_seen": 2239430656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269292929292929, + "loss": 2.6225, + "theoretical_loss": 3.3985506528688045, + "tokens_seen": 2239496192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269090909090909, + "loss": 2.6789, + "theoretical_loss": 3.3985425439657013, + "tokens_seen": 2239561728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268888888888889, + "loss": 2.6696, + "theoretical_loss": 3.398534435366323, + "tokens_seen": 2239627264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268686868686869, + "loss": 2.5787, + "theoretical_loss": 3.3985263270706505, + "tokens_seen": 2239692800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684848484848485, + "loss": 2.5279, + "theoretical_loss": 3.398518219078662, + "tokens_seen": 2239758336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268282828282828, + "loss": 2.6894, + "theoretical_loss": 3.398510111390338, + "tokens_seen": 2239823872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268080808080808, + "loss": 2.5157, + "theoretical_loss": 3.3985020040056586, + "tokens_seen": 2239889408 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1268791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9491946697235107, + "objective/train/theoretical_loss": 3.3984938969246024, + "objective/train/tokens_used": 598814176, + "theoretical_loss": 3.3984938969246024, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267878787878788, + "loss": 2.8172, + "theoretical_loss": 3.3984938969246024, + "tokens_seen": 2239954944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267676767676768, + "loss": 2.6561, + "theoretical_loss": 3.3984857901471504, + "tokens_seen": 2240020480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032674747474747474, + "loss": 2.7424, + "theoretical_loss": 3.3984776836732817, + "tokens_seen": 2240086016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267272727272727, + "loss": 2.595, + "theoretical_loss": 3.398469577502976, + "tokens_seen": 2240151552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267070707070707, + "loss": 2.4964, + "theoretical_loss": 3.3984614716362134, + "tokens_seen": 2240217088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266868686868687, + "loss": 2.6752, + "theoretical_loss": 3.398453366072973, + "tokens_seen": 2240282624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266666666666667, + "loss": 2.8368, + "theoretical_loss": 3.3984452608132356, + "tokens_seen": 2240348160 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032664646464646463, + "loss": 2.6559, + "theoretical_loss": 3.39843715585698, + "tokens_seen": 2240413696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032662626262626264, + "loss": 2.6891, + "theoretical_loss": 3.398429051204187, + "tokens_seen": 2240479232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032660606060606065, + "loss": 2.6084, + "theoretical_loss": 3.398420946854835, + "tokens_seen": 2240544768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265858585858586, + "loss": 2.555, + "theoretical_loss": 3.398412842808905, + "tokens_seen": 2240610304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032656565656565657, + "loss": 2.526, + "theoretical_loss": 3.3984047390663754, + "tokens_seen": 2240675840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265454545454545, + "loss": 2.6734, + "theoretical_loss": 3.3983966356272277, + "tokens_seen": 2240741376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032652525252525253, + "loss": 2.5602, + "theoretical_loss": 3.39838853249144, + "tokens_seen": 2240806912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032650505050505055, + "loss": 2.5568, + "theoretical_loss": 3.3983804296589932, + "tokens_seen": 2240872448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264848484848485, + "loss": 2.6828, + "theoretical_loss": 3.398372327129867, + "tokens_seen": 2240937984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032646464646464646, + "loss": 2.6308, + "theoretical_loss": 3.3983642249040407, + "tokens_seen": 2241003520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264444444444444, + "loss": 2.5996, + "theoretical_loss": 3.398356122981494, + "tokens_seen": 2241069056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264242424242425, + "loss": 2.7291, + "theoretical_loss": 3.398348021362207, + "tokens_seen": 2241134592 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032640404040404044, + "loss": 2.9172, + "theoretical_loss": 3.3983399200461597, + "tokens_seen": 2241200128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263838383838384, + "loss": 2.732, + "theoretical_loss": 3.3983318190333316, + "tokens_seen": 2241265664 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032636363636363635, + "loss": 2.4339, + "theoretical_loss": 3.3983237183237023, + "tokens_seen": 2241331200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263434343434343, + "loss": 2.8154, + "theoretical_loss": 3.398315617917252, + "tokens_seen": 2241396736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263232323232324, + "loss": 2.8695, + "theoretical_loss": 3.3983075178139597, + "tokens_seen": 2241462272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032630303030303033, + "loss": 2.5841, + "theoretical_loss": 3.3982994180138064, + "tokens_seen": 2241527808 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1270246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4890239238739014, + "objective/train/theoretical_loss": 3.3982913185167707, + "objective/train/tokens_used": 600452576, + "theoretical_loss": 3.3982913185167707, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262828282828283, + "loss": 2.4799, + "theoretical_loss": 3.3982913185167707, + "tokens_seen": 2241593344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032626262626262624, + "loss": 2.816, + "theoretical_loss": 3.3982832193228334, + "tokens_seen": 2241658880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032624242424242426, + "loss": 2.6981, + "theoretical_loss": 3.3982751204319737, + "tokens_seen": 2241724416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032622222222222227, + "loss": 2.7706, + "theoretical_loss": 3.3982670218441715, + "tokens_seen": 2241789952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262020202020202, + "loss": 2.7131, + "theoretical_loss": 3.398258923559407, + "tokens_seen": 2241855488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261818181818182, + "loss": 2.6132, + "theoretical_loss": 3.398250825577659, + "tokens_seen": 2241921024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032616161616161614, + "loss": 2.6414, + "theoretical_loss": 3.398242727898908, + "tokens_seen": 2241986560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032614141414141415, + "loss": 2.57, + "theoretical_loss": 3.3982346305231346, + "tokens_seen": 2242052096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032612121212121216, + "loss": 2.6417, + "theoretical_loss": 3.398226533450317, + "tokens_seen": 2242117632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261010101010101, + "loss": 2.6045, + "theoretical_loss": 3.3982184366804358, + "tokens_seen": 2242183168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260808080808081, + "loss": 2.5008, + "theoretical_loss": 3.3982103402134713, + "tokens_seen": 2242248704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260606060606061, + "loss": 2.5463, + "theoretical_loss": 3.3982022440494024, + "tokens_seen": 2242314240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032604040404040404, + "loss": 2.4412, + "theoretical_loss": 3.3981941481882094, + "tokens_seen": 2242379776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032602020202020205, + "loss": 2.5397, + "theoretical_loss": 3.398186052629872, + "tokens_seen": 2242445312 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326, + "loss": 2.5515, + "theoretical_loss": 3.39817795737437, + "tokens_seen": 2242510848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597979797979797, + "loss": 2.677, + "theoretical_loss": 3.3981698624216836, + "tokens_seen": 2242576384 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325959595959596, + "loss": 2.5205, + "theoretical_loss": 3.398161767771792, + "tokens_seen": 2242641920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593939393939393, + "loss": 2.4664, + "theoretical_loss": 3.3981536734246753, + "tokens_seen": 2242707456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591919191919195, + "loss": 2.784, + "theoretical_loss": 3.3981455793803135, + "tokens_seen": 2242772992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258989898989899, + "loss": 2.5334, + "theoretical_loss": 3.3981374856386863, + "tokens_seen": 2242838528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032587878787878786, + "loss": 2.4578, + "theoretical_loss": 3.3981293921997735, + "tokens_seen": 2242904064 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585858585858587, + "loss": 2.8318, + "theoretical_loss": 3.398121299063555, + "tokens_seen": 2242969600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258383838383838, + "loss": 2.6173, + "theoretical_loss": 3.3981132062300103, + "tokens_seen": 2243035136 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581818181818184, + "loss": 2.4224, + "theoretical_loss": 3.3981051136991196, + "tokens_seen": 2243100672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257979797979798, + "loss": 2.5038, + "theoretical_loss": 3.398097021470863, + "tokens_seen": 2243166208 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 1270875, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.950016498565674, + "objective/train/theoretical_loss": 3.39808892954522, + "objective/train/tokens_used": 602090976, + "theoretical_loss": 3.39808892954522, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003257777777777778, + "loss": 2.5943, + "theoretical_loss": 3.39808892954522, + "tokens_seen": 2243231744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032575757575757576, + "loss": 2.5864, + "theoretical_loss": 3.39808083792217, + "tokens_seen": 2243297280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257373737373737, + "loss": 2.8612, + "theoretical_loss": 3.3980727466016933, + "tokens_seen": 2243362816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032571717171717173, + "loss": 2.4327, + "theoretical_loss": 3.39806465558377, + "tokens_seen": 2243428352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256969696969697, + "loss": 2.4105, + "theoretical_loss": 3.39805656486838, + "tokens_seen": 2243493888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256767676767677, + "loss": 2.7575, + "theoretical_loss": 3.3980484744555026, + "tokens_seen": 2243559424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565656565656566, + "loss": 2.5702, + "theoretical_loss": 3.3980403843451175, + "tokens_seen": 2243624960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256363636363636, + "loss": 2.6734, + "theoretical_loss": 3.3980322945372055, + "tokens_seen": 2243690496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256161616161616, + "loss": 2.6277, + "theoretical_loss": 3.3980242050317457, + "tokens_seen": 2243756032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559595959595963, + "loss": 2.5169, + "theoretical_loss": 3.3980161158287183, + "tokens_seen": 2243821568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255757575757576, + "loss": 2.7245, + "theoretical_loss": 3.3980080269281028, + "tokens_seen": 2243887104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032555555555555555, + "loss": 2.6094, + "theoretical_loss": 3.397999938329879, + "tokens_seen": 2243952640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255353535353535, + "loss": 2.7583, + "theoretical_loss": 3.3979918500340274, + "tokens_seen": 2244018176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255151515151515, + "loss": 2.6528, + "theoretical_loss": 3.3979837620405275, + "tokens_seen": 2244083712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032549494949494953, + "loss": 2.7155, + "theoretical_loss": 3.397975674349359, + "tokens_seen": 2244149248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254747474747475, + "loss": 2.7584, + "theoretical_loss": 3.3979675869605024, + "tokens_seen": 2244214784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032545454545454544, + "loss": 2.5625, + "theoretical_loss": 3.397959499873937, + "tokens_seen": 2244280320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254343434343434, + "loss": 2.572, + "theoretical_loss": 3.3979514130896424, + "tokens_seen": 2244345856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032541414141414146, + "loss": 2.5019, + "theoretical_loss": 3.3979433266075993, + "tokens_seen": 2244411392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253939393939394, + "loss": 2.7502, + "theoretical_loss": 3.3979352404277874, + "tokens_seen": 2244476928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253737373737374, + "loss": 2.6844, + "theoretical_loss": 3.3979271545501857, + "tokens_seen": 2244542464 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535353535353533, + "loss": 2.6034, + "theoretical_loss": 3.397919068974775, + "tokens_seen": 2244608000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253333333333333, + "loss": 2.5087, + "theoretical_loss": 3.397910983701535, + "tokens_seen": 2244673536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531313131313136, + "loss": 2.6938, + "theoretical_loss": 3.3979028987304454, + "tokens_seen": 2244739072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252929292929293, + "loss": 2.5101, + "theoretical_loss": 3.397894814061486, + "tokens_seen": 2244804608 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1272103, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5674431324005127, + "objective/train/theoretical_loss": 3.397886729694637, + "objective/train/tokens_used": 603729376, + "theoretical_loss": 3.397886729694637, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527272727272727, + "loss": 2.5625, + "theoretical_loss": 3.397886729694637, + "tokens_seen": 2244870144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252525252525252, + "loss": 2.3459, + "theoretical_loss": 3.397878645629878, + "tokens_seen": 2244935680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523232323232324, + "loss": 2.5674, + "theoretical_loss": 3.3978705618671894, + "tokens_seen": 2245001216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521212121212125, + "loss": 2.6104, + "theoretical_loss": 3.3978624784065508, + "tokens_seen": 2245066752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251919191919192, + "loss": 2.6443, + "theoretical_loss": 3.3978543952479416, + "tokens_seen": 2245132288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032517171717171716, + "loss": 2.6945, + "theoretical_loss": 3.3978463123913425, + "tokens_seen": 2245197824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251515151515151, + "loss": 2.6332, + "theoretical_loss": 3.397838229836733, + "tokens_seen": 2245263360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513131313131313, + "loss": 2.4937, + "theoretical_loss": 3.397830147584093, + "tokens_seen": 2245328896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511111111111114, + "loss": 2.8085, + "theoretical_loss": 3.3978220656334024, + "tokens_seen": 2245394432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250909090909091, + "loss": 2.6212, + "theoretical_loss": 3.3978139839846406, + "tokens_seen": 2245459968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507070707070705, + "loss": 2.7134, + "theoretical_loss": 3.397805902637789, + "tokens_seen": 2245525504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032505050505050507, + "loss": 2.3959, + "theoretical_loss": 3.397797821592826, + "tokens_seen": 2245591040 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325030303030303, + "loss": 2.4853, + "theoretical_loss": 3.397789740849732, + "tokens_seen": 2245656576 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501010101010103, + "loss": 2.553, + "theoretical_loss": 3.3977816604084876, + "tokens_seen": 2245722112 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324989898989899, + "loss": 2.586, + "theoretical_loss": 3.397773580269072, + "tokens_seen": 2245787648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032496969696969695, + "loss": 2.861, + "theoretical_loss": 3.397765500431465, + "tokens_seen": 2245853184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494949494949496, + "loss": 2.77, + "theoretical_loss": 3.3977574208956467, + "tokens_seen": 2245918720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032492929292929297, + "loss": 2.6707, + "theoretical_loss": 3.3977493416615974, + "tokens_seen": 2245984256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249090909090909, + "loss": 2.7473, + "theoretical_loss": 3.3977412627292964, + "tokens_seen": 2246049792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248888888888889, + "loss": 2.7291, + "theoretical_loss": 3.397733184098724, + "tokens_seen": 2246115328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248686868686869, + "loss": 2.7064, + "theoretical_loss": 3.39772510576986, + "tokens_seen": 2246180864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032484848484848485, + "loss": 2.6414, + "theoretical_loss": 3.3977170277426847, + "tokens_seen": 2246246400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032482828282828286, + "loss": 2.6425, + "theoretical_loss": 3.3977089500171775, + "tokens_seen": 2246311936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248080808080808, + "loss": 2.4297, + "theoretical_loss": 3.3977008725933184, + "tokens_seen": 2246377472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247878787878788, + "loss": 2.5046, + "theoretical_loss": 3.3976927954710874, + "tokens_seen": 2246443008 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1272482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.021787643432617, + "objective/train/theoretical_loss": 3.397684718650465, + "objective/train/tokens_used": 605367776, + "theoretical_loss": 3.397684718650465, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247676767676768, + "loss": 2.5795, + "theoretical_loss": 3.397684718650465, + "tokens_seen": 2246508544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474747474747474, + "loss": 2.767, + "theoretical_loss": 3.3976766421314304, + "tokens_seen": 2246574080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032472727272727276, + "loss": 2.8035, + "theoretical_loss": 3.397668565913964, + "tokens_seen": 2246639616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247070707070707, + "loss": 2.4585, + "theoretical_loss": 3.3976604899980454, + "tokens_seen": 2246705152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246868686868687, + "loss": 2.6376, + "theoretical_loss": 3.3976524143836544, + "tokens_seen": 2246770688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246666666666667, + "loss": 2.7913, + "theoretical_loss": 3.3976443390707716, + "tokens_seen": 2246836224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032464646464646464, + "loss": 2.487, + "theoretical_loss": 3.3976362640593765, + "tokens_seen": 2246901760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032462626262626265, + "loss": 2.47, + "theoretical_loss": 3.397628189349449, + "tokens_seen": 2246967296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246060606060606, + "loss": 2.8459, + "theoretical_loss": 3.3976201149409695, + "tokens_seen": 2247032832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245858585858586, + "loss": 2.89, + "theoretical_loss": 3.3976120408339177, + "tokens_seen": 2247098368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032456565656565657, + "loss": 2.5356, + "theoretical_loss": 3.3976039670282736, + "tokens_seen": 2247163904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032454545454545453, + "loss": 2.6939, + "theoretical_loss": 3.3975958935240165, + "tokens_seen": 2247229440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032452525252525254, + "loss": 2.4779, + "theoretical_loss": 3.397587820321127, + "tokens_seen": 2247294976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032450505050505055, + "loss": 2.8676, + "theoretical_loss": 3.3975797474195857, + "tokens_seen": 2247360512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244848484848485, + "loss": 2.6743, + "theoretical_loss": 3.397571674819371, + "tokens_seen": 2247426048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032446464646464647, + "loss": 2.6035, + "theoretical_loss": 3.3975636025204645, + "tokens_seen": 2247491584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244444444444444, + "loss": 2.9726, + "theoretical_loss": 3.397555530522845, + "tokens_seen": 2247557120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032442424242424243, + "loss": 2.682, + "theoretical_loss": 3.3975474588264927, + "tokens_seen": 2247622656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032440404040404044, + "loss": 2.6291, + "theoretical_loss": 3.397539387431388, + "tokens_seen": 2247688192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243838383838384, + "loss": 2.5478, + "theoretical_loss": 3.3975313163375107, + "tokens_seen": 2247753728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032436363636363636, + "loss": 2.7784, + "theoretical_loss": 3.39752324554484, + "tokens_seen": 2247819264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243434343434343, + "loss": 2.6609, + "theoretical_loss": 3.3975151750533574, + "tokens_seen": 2247884800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243232323232324, + "loss": 2.4443, + "theoretical_loss": 3.397507104863042, + "tokens_seen": 2247950336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032430303030303034, + "loss": 2.6177, + "theoretical_loss": 3.397499034973873, + "tokens_seen": 2248015872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242828282828283, + "loss": 2.6543, + "theoretical_loss": 3.397490965385832, + "tokens_seen": 2248081408 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1273740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4786767959594727, + "objective/train/theoretical_loss": 3.397482896098898, + "objective/train/tokens_used": 607006176, + "theoretical_loss": 3.397482896098898, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426262626262625, + "loss": 2.7811, + "theoretical_loss": 3.397482896098898, + "tokens_seen": 2248146944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242424242424242, + "loss": 2.569, + "theoretical_loss": 3.397474827113051, + "tokens_seen": 2248212480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032422222222222227, + "loss": 2.552, + "theoretical_loss": 3.397466758428271, + "tokens_seen": 2248278016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032420202020202023, + "loss": 2.7027, + "theoretical_loss": 3.3974586900445383, + "tokens_seen": 2248343552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241818181818182, + "loss": 2.6871, + "theoretical_loss": 3.3974506219618332, + "tokens_seen": 2248409088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032416161616161614, + "loss": 2.6429, + "theoretical_loss": 3.3974425541801345, + "tokens_seen": 2248474624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241414141414141, + "loss": 2.7274, + "theoretical_loss": 3.3974344866994235, + "tokens_seen": 2248540160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032412121212121217, + "loss": 2.5753, + "theoretical_loss": 3.3974264195196797, + "tokens_seen": 2248605696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241010101010101, + "loss": 2.6743, + "theoretical_loss": 3.3974183526408828, + "tokens_seen": 2248671232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240808080808081, + "loss": 2.7561, + "theoretical_loss": 3.397410286063013, + "tokens_seen": 2248736768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032406060606060604, + "loss": 2.9231, + "theoretical_loss": 3.3974022197860503, + "tokens_seen": 2248802304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032404040404040405, + "loss": 2.7856, + "theoretical_loss": 3.3973941538099752, + "tokens_seen": 2248867840 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032402020202020206, + "loss": 2.5254, + "theoretical_loss": 3.397386088134767, + "tokens_seen": 2248933376 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324, + "loss": 2.5281, + "theoretical_loss": 3.397378022760406, + "tokens_seen": 2248998912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032397979797979797, + "loss": 2.6786, + "theoretical_loss": 3.397369957686872, + "tokens_seen": 2249064448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032395959595959593, + "loss": 2.718, + "theoretical_loss": 3.3973618929141454, + "tokens_seen": 2249129984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032393939393939394, + "loss": 2.752, + "theoretical_loss": 3.397353828442206, + "tokens_seen": 2249195520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391919191919195, + "loss": 2.7551, + "theoretical_loss": 3.3973457642710336, + "tokens_seen": 2249261056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238989898989899, + "loss": 2.8657, + "theoretical_loss": 3.397337700400609, + "tokens_seen": 2249326592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387878787878786, + "loss": 2.8072, + "theoretical_loss": 3.397329636830911, + "tokens_seen": 2249392128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238585858585859, + "loss": 2.8258, + "theoretical_loss": 3.397321573561921, + "tokens_seen": 2249457664 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383838383838383, + "loss": 2.5268, + "theoretical_loss": 3.3973135105936176, + "tokens_seen": 2249523200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032381818181818184, + "loss": 2.5821, + "theoretical_loss": 3.397305447925982, + "tokens_seen": 2249588736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237979797979798, + "loss": 2.4664, + "theoretical_loss": 3.3972973855589936, + "tokens_seen": 2249654272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032377777777777776, + "loss": 2.5834, + "theoretical_loss": 3.397289323492633, + "tokens_seen": 2249719808 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1274460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.961244583129883, + "objective/train/theoretical_loss": 3.3972812617268797, + "objective/train/tokens_used": 608644576, + "theoretical_loss": 3.3972812617268797, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375757575757577, + "loss": 2.4873, + "theoretical_loss": 3.3972812617268797, + "tokens_seen": 2249785344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237373737373737, + "loss": 2.6115, + "theoretical_loss": 3.397273200261713, + "tokens_seen": 2249850880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371717171717174, + "loss": 2.641, + "theoretical_loss": 3.397265139097115, + "tokens_seen": 2249916416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236969696969697, + "loss": 2.6753, + "theoretical_loss": 3.397257078233064, + "tokens_seen": 2249981952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236767676767677, + "loss": 2.4864, + "theoretical_loss": 3.397249017669541, + "tokens_seen": 2250047488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032365656565656566, + "loss": 2.5734, + "theoretical_loss": 3.397240957406525, + "tokens_seen": 2250113024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236363636363636, + "loss": 2.7756, + "theoretical_loss": 3.397232897443997, + "tokens_seen": 2250178560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361616161616163, + "loss": 2.7017, + "theoretical_loss": 3.397224837781937, + "tokens_seen": 2250244096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235959595959596, + "loss": 2.8285, + "theoretical_loss": 3.397216778420325, + "tokens_seen": 2250309632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235757575757576, + "loss": 2.7862, + "theoretical_loss": 3.3972087193591403, + "tokens_seen": 2250375168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355555555555555, + "loss": 2.8327, + "theoretical_loss": 3.3972006605983633, + "tokens_seen": 2250440704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235353535353535, + "loss": 2.716, + "theoretical_loss": 3.397192602137975, + "tokens_seen": 2250506240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235151515151515, + "loss": 2.7223, + "theoretical_loss": 3.397184543977954, + "tokens_seen": 2250571776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032349494949494953, + "loss": 2.8049, + "theoretical_loss": 3.3971764861182816, + "tokens_seen": 2250637312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234747474747475, + "loss": 2.6489, + "theoretical_loss": 3.397168428558937, + "tokens_seen": 2250702848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032345454545454545, + "loss": 2.5951, + "theoretical_loss": 3.3971603712999006, + "tokens_seen": 2250768384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234343434343434, + "loss": 2.5462, + "theoretical_loss": 3.3971523143411524, + "tokens_seen": 2250833920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234141414141414, + "loss": 2.7418, + "theoretical_loss": 3.3971442576826725, + "tokens_seen": 2250899456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233939393939394, + "loss": 2.8896, + "theoretical_loss": 3.3971362013244413, + "tokens_seen": 2250964992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233737373737374, + "loss": 2.9272, + "theoretical_loss": 3.3971281452664384, + "tokens_seen": 2251030528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335353535353534, + "loss": 2.5054, + "theoretical_loss": 3.3971200895086437, + "tokens_seen": 2251096064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233333333333333, + "loss": 2.6965, + "theoretical_loss": 3.3971120340510375, + "tokens_seen": 2251161600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032331313131313136, + "loss": 2.9238, + "theoretical_loss": 3.3971039788936004, + "tokens_seen": 2251227136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232929292929293, + "loss": 2.7428, + "theoretical_loss": 3.3970959240363117, + "tokens_seen": 2251292672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232727272727273, + "loss": 2.6916, + "theoretical_loss": 3.397087869479152, + "tokens_seen": 2251358208 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1275535, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3812789916992188, + "objective/train/theoretical_loss": 3.3970798152221016, + "objective/train/tokens_used": 610282976, + "theoretical_loss": 3.3970798152221016, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032325252525252523, + "loss": 2.5431, + "theoretical_loss": 3.3970798152221016, + "tokens_seen": 2251423744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032323232323232324, + "loss": 2.8023, + "theoretical_loss": 3.3970717612651398, + "tokens_seen": 2251489280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032321212121212125, + "loss": 2.6487, + "theoretical_loss": 3.397063707608247, + "tokens_seen": 2251554816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231919191919192, + "loss": 2.5717, + "theoretical_loss": 3.397055654251403, + "tokens_seen": 2251620352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032317171717171717, + "loss": 2.2965, + "theoretical_loss": 3.397047601194589, + "tokens_seen": 2251685888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231515151515151, + "loss": 2.6939, + "theoretical_loss": 3.397039548437784, + "tokens_seen": 2251751424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231313131313132, + "loss": 2.4326, + "theoretical_loss": 3.3970314959809684, + "tokens_seen": 2251816960 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032311111111111115, + "loss": 2.5424, + "theoretical_loss": 3.3970234438241222, + "tokens_seen": 2251882496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230909090909091, + "loss": 2.6073, + "theoretical_loss": 3.397015391967226, + "tokens_seen": 2251948032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032307070707070706, + "loss": 2.8456, + "theoretical_loss": 3.3970073404102594, + "tokens_seen": 2252013568 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323050505050505, + "loss": 2.594, + "theoretical_loss": 3.3969992891532024, + "tokens_seen": 2252079104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230303030303031, + "loss": 2.4315, + "theoretical_loss": 3.3969912381960357, + "tokens_seen": 2252144640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032301010101010104, + "loss": 2.1016, + "theoretical_loss": 3.396983187538739, + "tokens_seen": 2252210176 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322989898989899, + "loss": 2.6748, + "theoretical_loss": 3.3969751371812915, + "tokens_seen": 2252275712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032296969696969695, + "loss": 2.6811, + "theoretical_loss": 3.396967087123675, + "tokens_seen": 2252341248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032294949494949496, + "loss": 2.4999, + "theoretical_loss": 3.396959037365869, + "tokens_seen": 2252406784 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322929292929293, + "loss": 2.6347, + "theoretical_loss": 3.3969509879078528, + "tokens_seen": 2252472320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032290909090909093, + "loss": 2.68, + "theoretical_loss": 3.396942938749608, + "tokens_seen": 2252537856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228888888888889, + "loss": 2.7145, + "theoretical_loss": 3.396934889891113, + "tokens_seen": 2252603392 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032286868686868685, + "loss": 2.8149, + "theoretical_loss": 3.3969268413323497, + "tokens_seen": 2252668928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284848484848486, + "loss": 2.5409, + "theoretical_loss": 3.3969187930732967, + "tokens_seen": 2252734464 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032282828282828287, + "loss": 2.5977, + "theoretical_loss": 3.396910745113935, + "tokens_seen": 2252800000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228080808080808, + "loss": 2.5085, + "theoretical_loss": 3.3969026974542444, + "tokens_seen": 2252865536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227878787878788, + "loss": 2.8243, + "theoretical_loss": 3.3968946500942048, + "tokens_seen": 2252931072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276767676767674, + "loss": 2.6032, + "theoretical_loss": 3.396886603033797, + "tokens_seen": 2252996608 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1276002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.562368392944336, + "objective/train/theoretical_loss": 3.3968785562730006, + "objective/train/tokens_used": 611921376, + "theoretical_loss": 3.3968785562730006, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032274747474747475, + "loss": 2.5382, + "theoretical_loss": 3.3968785562730006, + "tokens_seen": 2253062144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272727272727276, + "loss": 2.6067, + "theoretical_loss": 3.396870509811796, + "tokens_seen": 2253127680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227070707070707, + "loss": 2.5649, + "theoretical_loss": 3.396862463650163, + "tokens_seen": 2253193216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226868686868687, + "loss": 2.7801, + "theoretical_loss": 3.3968544177880817, + "tokens_seen": 2253258752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226666666666667, + "loss": 2.6994, + "theoretical_loss": 3.396846372225533, + "tokens_seen": 2253324288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032264646464646464, + "loss": 2.5297, + "theoretical_loss": 3.3968383269624964, + "tokens_seen": 2253389824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262626262626265, + "loss": 2.6383, + "theoretical_loss": 3.396830281998952, + "tokens_seen": 2253455360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226060606060606, + "loss": 2.6169, + "theoretical_loss": 3.3968222373348804, + "tokens_seen": 2253520896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032258585858585857, + "loss": 2.9142, + "theoretical_loss": 3.396814192970261, + "tokens_seen": 2253586432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225656565656566, + "loss": 2.6498, + "theoretical_loss": 3.3968061489050747, + "tokens_seen": 2253651968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254545454545453, + "loss": 2.8907, + "theoretical_loss": 3.3967981051393012, + "tokens_seen": 2253717504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032252525252525255, + "loss": 2.6787, + "theoretical_loss": 3.3967900616729207, + "tokens_seen": 2253783040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225050505050505, + "loss": 2.6014, + "theoretical_loss": 3.3967820185059137, + "tokens_seen": 2253848576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224848484848485, + "loss": 2.6126, + "theoretical_loss": 3.3967739756382596, + "tokens_seen": 2253914112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246464646464647, + "loss": 2.7829, + "theoretical_loss": 3.3967659330699393, + "tokens_seen": 2253979648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032244444444444443, + "loss": 2.6887, + "theoretical_loss": 3.396757890800933, + "tokens_seen": 2254045184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032242424242424244, + "loss": 2.8417, + "theoretical_loss": 3.39674984883122, + "tokens_seen": 2254110720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224040404040404, + "loss": 2.7011, + "theoretical_loss": 3.3967418071607813, + "tokens_seen": 2254176256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223838383838384, + "loss": 2.5723, + "theoretical_loss": 3.396733765789597, + "tokens_seen": 2254241792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032236363636363636, + "loss": 2.702, + "theoretical_loss": 3.3967257247176468, + "tokens_seen": 2254307328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223434343434343, + "loss": 2.6153, + "theoretical_loss": 3.396717683944911, + "tokens_seen": 2254372864 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032232323232323233, + "loss": 2.6556, + "theoretical_loss": 3.39670964347137, + "tokens_seen": 2254438400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230303030303034, + "loss": 2.9557, + "theoretical_loss": 3.3967016032970037, + "tokens_seen": 2254503936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222828282828283, + "loss": 2.5851, + "theoretical_loss": 3.3966935634217927, + "tokens_seen": 2254569472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032226262626262626, + "loss": 2.7897, + "theoretical_loss": 3.3966855238457168, + "tokens_seen": 2254635008 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1277060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.81943941116333, + "objective/train/theoretical_loss": 3.396677484568756, + "objective/train/tokens_used": 613559776, + "theoretical_loss": 3.396677484568756, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222424242424242, + "loss": 2.9179, + "theoretical_loss": 3.396677484568756, + "tokens_seen": 2254700544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222222222222222, + "loss": 2.6304, + "theoretical_loss": 3.396669445590891, + "tokens_seen": 2254766080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032220202020202024, + "loss": 2.5574, + "theoretical_loss": 3.3966614069121017, + "tokens_seen": 2254831616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221818181818182, + "loss": 2.7258, + "theoretical_loss": 3.3966533685323683, + "tokens_seen": 2254897152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032216161616161615, + "loss": 2.8606, + "theoretical_loss": 3.3966453304516713, + "tokens_seen": 2254962688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221414141414141, + "loss": 2.7512, + "theoretical_loss": 3.39663729266999, + "tokens_seen": 2255028224 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032212121212121217, + "loss": 2.8463, + "theoretical_loss": 3.3966292551873054, + "tokens_seen": 2255093760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032210101010101013, + "loss": 2.9031, + "theoretical_loss": 3.3966212180035975, + "tokens_seen": 2255159296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220808080808081, + "loss": 2.5414, + "theoretical_loss": 3.3966131811188465, + "tokens_seen": 2255224832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032206060606060604, + "loss": 2.8381, + "theoretical_loss": 3.3966051445330323, + "tokens_seen": 2255290368 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322040404040404, + "loss": 2.8611, + "theoretical_loss": 3.3965971082461355, + "tokens_seen": 2255355904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202020202020206, + "loss": 2.6295, + "theoretical_loss": 3.396589072258136, + "tokens_seen": 2255421440 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322, + "loss": 2.5768, + "theoretical_loss": 3.3965810365690143, + "tokens_seen": 2255486976 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321979797979798, + "loss": 2.5923, + "theoretical_loss": 3.3965730011787505, + "tokens_seen": 2255552512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032195959595959593, + "loss": 2.6404, + "theoretical_loss": 3.3965649660873245, + "tokens_seen": 2255618048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032193939393939395, + "loss": 2.5027, + "theoretical_loss": 3.3965569312947164, + "tokens_seen": 2255683584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032191919191919196, + "loss": 2.93, + "theoretical_loss": 3.3965488968009074, + "tokens_seen": 2255749120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218989898989899, + "loss": 2.4466, + "theoretical_loss": 3.3965408626058764, + "tokens_seen": 2255814656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032187878787878787, + "loss": 2.7849, + "theoretical_loss": 3.3965328287096046, + "tokens_seen": 2255880192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218585858585858, + "loss": 2.6707, + "theoretical_loss": 3.396524795112072, + "tokens_seen": 2255945728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032183838383838384, + "loss": 2.7288, + "theoretical_loss": 3.396516761813259, + "tokens_seen": 2256011264 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032181818181818185, + "loss": 2.5117, + "theoretical_loss": 3.396508728813145, + "tokens_seen": 2256076800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217979797979798, + "loss": 2.6672, + "theoretical_loss": 3.3965006961117106, + "tokens_seen": 2256142336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032177777777777776, + "loss": 2.7585, + "theoretical_loss": 3.396492663708936, + "tokens_seen": 2256207872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217575757575758, + "loss": 2.7323, + "theoretical_loss": 3.396484631604802, + "tokens_seen": 2256273408 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1277665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.990206480026245, + "objective/train/theoretical_loss": 3.3964765997992883, + "objective/train/tokens_used": 615198176, + "theoretical_loss": 3.3964765997992883, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032173737373737373, + "loss": 2.946, + "theoretical_loss": 3.3964765997992883, + "tokens_seen": 2256338944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032171717171717174, + "loss": 2.7839, + "theoretical_loss": 3.396468568292375, + "tokens_seen": 2256404480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216969696969697, + "loss": 2.878, + "theoretical_loss": 3.3964605370840424, + "tokens_seen": 2256470016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032167676767676766, + "loss": 2.8086, + "theoretical_loss": 3.3964525061742714, + "tokens_seen": 2256535552 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032165656565656567, + "loss": 2.7081, + "theoretical_loss": 3.396444475563041, + "tokens_seen": 2256601088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216363636363636, + "loss": 2.6946, + "theoretical_loss": 3.3964364452503326, + "tokens_seen": 2256666624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032161616161616163, + "loss": 2.6659, + "theoretical_loss": 3.3964284152361257, + "tokens_seen": 2256732160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215959595959596, + "loss": 2.5013, + "theoretical_loss": 3.396420385520401, + "tokens_seen": 2256797696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215757575757576, + "loss": 2.6207, + "theoretical_loss": 3.3964123561031383, + "tokens_seen": 2256863232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032155555555555556, + "loss": 2.5314, + "theoretical_loss": 3.3964043269843183, + "tokens_seen": 2256928768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032153535353535357, + "loss": 2.4925, + "theoretical_loss": 3.396396298163921, + "tokens_seen": 2256994304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032151515151515153, + "loss": 2.8995, + "theoretical_loss": 3.3963882696419265, + "tokens_seen": 2257059840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214949494949495, + "loss": 2.646, + "theoretical_loss": 3.3963802414183153, + "tokens_seen": 2257125376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214747474747475, + "loss": 2.6798, + "theoretical_loss": 3.396372213493067, + "tokens_seen": 2257190912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032145454545454545, + "loss": 2.6747, + "theoretical_loss": 3.396364185866163, + "tokens_seen": 2257256448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032143434343434346, + "loss": 3.0013, + "theoretical_loss": 3.396356158537583, + "tokens_seen": 2257321984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214141414141414, + "loss": 2.6588, + "theoretical_loss": 3.396348131507307, + "tokens_seen": 2257387520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032139393939393943, + "loss": 2.5462, + "theoretical_loss": 3.3963401047753154, + "tokens_seen": 2257453056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213737373737374, + "loss": 2.4291, + "theoretical_loss": 3.3963320783415885, + "tokens_seen": 2257518592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032135353535353534, + "loss": 2.821, + "theoretical_loss": 3.3963240522061064, + "tokens_seen": 2257584128 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032133333333333336, + "loss": 2.5902, + "theoretical_loss": 3.39631602636885, + "tokens_seen": 2257649664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213131313131313, + "loss": 2.7217, + "theoretical_loss": 3.396308000829799, + "tokens_seen": 2257715200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212929292929293, + "loss": 2.6576, + "theoretical_loss": 3.396299975588933, + "tokens_seen": 2257780736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212727272727273, + "loss": 2.8719, + "theoretical_loss": 3.396291950646234, + "tokens_seen": 2257846272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032125252525252524, + "loss": 2.7192, + "theoretical_loss": 3.396283926001681, + "tokens_seen": 2257911808 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1279074, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.95670223236084, + "objective/train/theoretical_loss": 3.396275901655254, + "objective/train/tokens_used": 616836576, + "theoretical_loss": 3.396275901655254, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032123232323232325, + "loss": 2.7832, + "theoretical_loss": 3.396275901655254, + "tokens_seen": 2257977344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121212121212126, + "loss": 2.5081, + "theoretical_loss": 3.3962678776069346, + "tokens_seen": 2258042880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211919191919192, + "loss": 2.6619, + "theoretical_loss": 3.396259853856702, + "tokens_seen": 2258108416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211717171717172, + "loss": 2.4934, + "theoretical_loss": 3.3962518304045366, + "tokens_seen": 2258173952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032115151515151513, + "loss": 2.5633, + "theoretical_loss": 3.396243807250419, + "tokens_seen": 2258239488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113131313131314, + "loss": 2.8629, + "theoretical_loss": 3.3962357843943294, + "tokens_seen": 2258305024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032111111111111115, + "loss": 2.4839, + "theoretical_loss": 3.396227761836248, + "tokens_seen": 2258370560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210909090909091, + "loss": 2.4714, + "theoretical_loss": 3.396219739576155, + "tokens_seen": 2258436096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032107070707070707, + "loss": 2.6169, + "theoretical_loss": 3.3962117176140305, + "tokens_seen": 2258501632 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321050505050505, + "loss": 2.5835, + "theoretical_loss": 3.396203695949856, + "tokens_seen": 2258567168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210303030303031, + "loss": 2.5036, + "theoretical_loss": 3.39619567458361, + "tokens_seen": 2258632704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032101010101010104, + "loss": 2.5372, + "theoretical_loss": 3.396187653515274, + "tokens_seen": 2258698240 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320989898989899, + "loss": 2.5557, + "theoretical_loss": 3.3961796327448277, + "tokens_seen": 2258763776 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032096969696969696, + "loss": 2.5578, + "theoretical_loss": 3.396171612272252, + "tokens_seen": 2258829312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209494949494949, + "loss": 2.8161, + "theoretical_loss": 3.3961635920975266, + "tokens_seen": 2258894848 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320929292929293, + "loss": 2.5571, + "theoretical_loss": 3.396155572220632, + "tokens_seen": 2258960384 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090909090909094, + "loss": 2.7973, + "theoretical_loss": 3.3961475526415486, + "tokens_seen": 2259025920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208888888888889, + "loss": 2.7616, + "theoretical_loss": 3.396139533360256, + "tokens_seen": 2259091456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086868686868685, + "loss": 2.6441, + "theoretical_loss": 3.3961315143767363, + "tokens_seen": 2259156992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208484848484848, + "loss": 2.661, + "theoretical_loss": 3.396123495690968, + "tokens_seen": 2259222528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208282828282829, + "loss": 2.4985, + "theoretical_loss": 3.3961154773029323, + "tokens_seen": 2259288064 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032080808080808083, + "loss": 2.6996, + "theoretical_loss": 3.396107459212609, + "tokens_seen": 2259353600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207878787878788, + "loss": 2.4891, + "theoretical_loss": 3.3960994414199783, + "tokens_seen": 2259419136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032076767676767674, + "loss": 2.5969, + "theoretical_loss": 3.3960914239250215, + "tokens_seen": 2259484672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032074747474747476, + "loss": 2.7007, + "theoretical_loss": 3.396083406727718, + "tokens_seen": 2259550208 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 1279794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7679245471954346, + "objective/train/theoretical_loss": 3.3960753898280487, + "objective/train/tokens_used": 618474976, + "theoretical_loss": 3.3960753898280487, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032072727272727277, + "loss": 2.6482, + "theoretical_loss": 3.3960753898280487, + "tokens_seen": 2259615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207070707070707, + "loss": 2.6122, + "theoretical_loss": 3.3960673732259936, + "tokens_seen": 2259681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206868686868687, + "loss": 2.6586, + "theoretical_loss": 3.396059356921533, + "tokens_seen": 2259746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032066666666666664, + "loss": 2.551, + "theoretical_loss": 3.396051340914647, + "tokens_seen": 2259812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032064646464646465, + "loss": 2.6618, + "theoretical_loss": 3.3960433252053166, + "tokens_seen": 2259877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062626262626266, + "loss": 2.6736, + "theoretical_loss": 3.3960353097935214, + "tokens_seen": 2259943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206060606060606, + "loss": 2.5871, + "theoretical_loss": 3.396027294679242, + "tokens_seen": 2260008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032058585858585857, + "loss": 2.4798, + "theoretical_loss": 3.3960192798624593, + "tokens_seen": 2260074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205656565656566, + "loss": 2.8271, + "theoretical_loss": 3.3960112653431525, + "tokens_seen": 2260140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032054545454545454, + "loss": 2.5646, + "theoretical_loss": 3.396003251121303, + "tokens_seen": 2260205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032052525252525255, + "loss": 2.4123, + "theoretical_loss": 3.3959952371968907, + "tokens_seen": 2260271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205050505050505, + "loss": 2.4885, + "theoretical_loss": 3.3959872235698954, + "tokens_seen": 2260336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032048484848484847, + "loss": 2.6754, + "theoretical_loss": 3.3959792102402986, + "tokens_seen": 2260402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204646464646465, + "loss": 2.6006, + "theoretical_loss": 3.3959711972080795, + "tokens_seen": 2260467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032044444444444443, + "loss": 2.6704, + "theoretical_loss": 3.395963184473219, + "tokens_seen": 2260533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042424242424244, + "loss": 2.5213, + "theoretical_loss": 3.3959551720356975, + "tokens_seen": 2260598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204040404040404, + "loss": 2.6812, + "theoretical_loss": 3.3959471598954956, + "tokens_seen": 2260664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203838383838384, + "loss": 2.8489, + "theoretical_loss": 3.3959391480525927, + "tokens_seen": 2260729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032036363636363637, + "loss": 2.6838, + "theoretical_loss": 3.39593113650697, + "tokens_seen": 2260795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203434343434343, + "loss": 2.6011, + "theoretical_loss": 3.3959231252586077, + "tokens_seen": 2260860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032032323232323234, + "loss": 2.5322, + "theoretical_loss": 3.3959151143074857, + "tokens_seen": 2260926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203030303030303, + "loss": 2.63, + "theoretical_loss": 3.395907103653585, + "tokens_seen": 2260992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202828282828283, + "loss": 2.7358, + "theoretical_loss": 3.3958990932968858, + "tokens_seen": 2261057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026262626262626, + "loss": 2.7877, + "theoretical_loss": 3.395891083237368, + "tokens_seen": 2261123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202424242424242, + "loss": 2.7427, + "theoretical_loss": 3.3958830734750123, + "tokens_seen": 2261188608 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1281070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.551194190979004, + "objective/train/theoretical_loss": 3.3958750640097994, + "objective/train/tokens_used": 620113376, + "theoretical_loss": 3.3958750640097994, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032022222222222223, + "loss": 2.8395, + "theoretical_loss": 3.3958750640097994, + "tokens_seen": 2261254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032020202020202024, + "loss": 2.7282, + "theoretical_loss": 3.395867054841709, + "tokens_seen": 2261319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201818181818182, + "loss": 2.4626, + "theoretical_loss": 3.3958590459707216, + "tokens_seen": 2261385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032016161616161615, + "loss": 2.3936, + "theoretical_loss": 3.3958510373968176, + "tokens_seen": 2261450752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201414141414141, + "loss": 2.68, + "theoretical_loss": 3.395843029119978, + "tokens_seen": 2261516288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201212121212121, + "loss": 2.7477, + "theoretical_loss": 3.395835021140183, + "tokens_seen": 2261581824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010101010101013, + "loss": 2.6192, + "theoretical_loss": 3.395827013457412, + "tokens_seen": 2261647360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200808080808081, + "loss": 2.7385, + "theoretical_loss": 3.395819006071646, + "tokens_seen": 2261712896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032006060606060605, + "loss": 2.6902, + "theoretical_loss": 3.3958109989828658, + "tokens_seen": 2261778432 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320040404040404, + "loss": 2.4685, + "theoretical_loss": 3.3958029921910513, + "tokens_seen": 2261843968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002020202020207, + "loss": 2.3433, + "theoretical_loss": 3.395794985696183, + "tokens_seen": 2261909504 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 2.698, + "theoretical_loss": 3.395786979498241, + "tokens_seen": 2261975040 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319979797979798, + "loss": 2.5136, + "theoretical_loss": 3.3957789735972064, + "tokens_seen": 2262040576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995959595959594, + "loss": 2.465, + "theoretical_loss": 3.3957709679930588, + "tokens_seen": 2262106112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199393939393939, + "loss": 2.7494, + "theoretical_loss": 3.395762962685779, + "tokens_seen": 2262171648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991919191919196, + "loss": 2.6632, + "theoretical_loss": 3.395754957675347, + "tokens_seen": 2262237184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198989898989899, + "loss": 2.7212, + "theoretical_loss": 3.395746952961744, + "tokens_seen": 2262302720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198787878787879, + "loss": 2.5752, + "theoretical_loss": 3.39573894854495, + "tokens_seen": 2262368256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031985858585858583, + "loss": 2.773, + "theoretical_loss": 3.395730944424945, + "tokens_seen": 2262433792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983838383838384, + "loss": 2.9093, + "theoretical_loss": 3.3957229406017095, + "tokens_seen": 2262499328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031981818181818185, + "loss": 2.5593, + "theoretical_loss": 3.3957149370752244, + "tokens_seen": 2262564864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197979797979798, + "loss": 2.393, + "theoretical_loss": 3.39570693384547, + "tokens_seen": 2262630400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031977777777777777, + "loss": 2.6542, + "theoretical_loss": 3.3956989309124257, + "tokens_seen": 2262695936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197575757575757, + "loss": 2.6978, + "theoretical_loss": 3.3956909282760734, + "tokens_seen": 2262761472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197373737373738, + "loss": 2.5784, + "theoretical_loss": 3.3956829259363928, + "tokens_seen": 2262827008 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1281765, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.712146043777466, + "objective/train/theoretical_loss": 3.395674923893364, + "objective/train/tokens_used": 621751776, + "theoretical_loss": 3.395674923893364, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971717171717175, + "loss": 2.6164, + "theoretical_loss": 3.395674923893364, + "tokens_seen": 2262892544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196969696969697, + "loss": 2.5272, + "theoretical_loss": 3.395666922146968, + "tokens_seen": 2262958080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031967676767676766, + "loss": 2.7256, + "theoretical_loss": 3.3956589206971852, + "tokens_seen": 2263023616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031965656565656567, + "loss": 2.4905, + "theoretical_loss": 3.3956509195439954, + "tokens_seen": 2263089152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196363636363637, + "loss": 2.5538, + "theoretical_loss": 3.3956429186873796, + "tokens_seen": 2263154688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031961616161616164, + "loss": 2.5558, + "theoretical_loss": 3.395634918127318, + "tokens_seen": 2263220224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195959595959596, + "loss": 2.7045, + "theoretical_loss": 3.3956269178637912, + "tokens_seen": 2263285760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957575757575755, + "loss": 2.8127, + "theoretical_loss": 3.395618917896779, + "tokens_seen": 2263351296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955555555555556, + "loss": 2.722, + "theoretical_loss": 3.3956109182262626, + "tokens_seen": 2263416832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195353535353536, + "loss": 2.7746, + "theoretical_loss": 3.395602918852222, + "tokens_seen": 2263482368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031951515151515153, + "loss": 2.8089, + "theoretical_loss": 3.395594919774638, + "tokens_seen": 2263547904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194949494949495, + "loss": 2.5962, + "theoretical_loss": 3.39558692099349, + "tokens_seen": 2263613440 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947474747474745, + "loss": 2.6376, + "theoretical_loss": 3.39557892250876, + "tokens_seen": 2263678976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945454545454546, + "loss": 2.7549, + "theoretical_loss": 3.3955709243204275, + "tokens_seen": 2263744512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031943434343434347, + "loss": 2.5541, + "theoretical_loss": 3.3955629264284726, + "tokens_seen": 2263810048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194141414141414, + "loss": 2.5184, + "theoretical_loss": 3.3955549288328766, + "tokens_seen": 2263875584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193939393939394, + "loss": 2.6872, + "theoretical_loss": 3.3955469315336195, + "tokens_seen": 2263941120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193737373737374, + "loss": 2.6968, + "theoretical_loss": 3.395538934530682, + "tokens_seen": 2264006656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031935353535353535, + "loss": 2.8954, + "theoretical_loss": 3.395530937824044, + "tokens_seen": 2264072192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933333333333336, + "loss": 2.5245, + "theoretical_loss": 3.3955229414136863, + "tokens_seen": 2264137728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193131313131313, + "loss": 2.7546, + "theoretical_loss": 3.3955149452995896, + "tokens_seen": 2264203264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192929292929293, + "loss": 2.6081, + "theoretical_loss": 3.395506949481734, + "tokens_seen": 2264268800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192727272727273, + "loss": 2.9112, + "theoretical_loss": 3.3954989539601, + "tokens_seen": 2264334336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925252525252524, + "loss": 2.4577, + "theoretical_loss": 3.395490958734668, + "tokens_seen": 2264399872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031923232323232325, + "loss": 2.6115, + "theoretical_loss": 3.3954829638054185, + "tokens_seen": 2264465408 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1283066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.296011209487915, + "objective/train/theoretical_loss": 3.395474969172332, + "objective/train/tokens_used": 623390176, + "theoretical_loss": 3.395474969172332, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192121212121212, + "loss": 2.54, + "theoretical_loss": 3.395474969172332, + "tokens_seen": 2264530944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191919191919192, + "loss": 2.5181, + "theoretical_loss": 3.395466974835389, + "tokens_seen": 2264596480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191717171717172, + "loss": 2.6597, + "theoretical_loss": 3.39545898079457, + "tokens_seen": 2264662016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915151515151514, + "loss": 2.303, + "theoretical_loss": 3.3954509870498555, + "tokens_seen": 2264727552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031913131313131315, + "loss": 2.7512, + "theoretical_loss": 3.3954429936012254, + "tokens_seen": 2264793088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191111111111111, + "loss": 2.5292, + "theoretical_loss": 3.3954350004486606, + "tokens_seen": 2264858624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190909090909091, + "loss": 2.7234, + "theoretical_loss": 3.3954270075921418, + "tokens_seen": 2264924160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907070707070707, + "loss": 2.7749, + "theoretical_loss": 3.3954190150316492, + "tokens_seen": 2264989696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905050505050503, + "loss": 2.564, + "theoretical_loss": 3.3954110227671634, + "tokens_seen": 2265055232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903030303030304, + "loss": 2.562, + "theoretical_loss": 3.395403030798665, + "tokens_seen": 2265120768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031901010101010105, + "loss": 2.6979, + "theoretical_loss": 3.395395039126134, + "tokens_seen": 2265186304 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318989898989899, + "loss": 2.6194, + "theoretical_loss": 3.3953870477495514, + "tokens_seen": 2265251840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896969696969696, + "loss": 2.491, + "theoretical_loss": 3.395379056668897, + "tokens_seen": 2265317376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189494949494949, + "loss": 2.7009, + "theoretical_loss": 3.395371065884152, + "tokens_seen": 2265382912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031892929292929293, + "loss": 2.7369, + "theoretical_loss": 3.395363075395296, + "tokens_seen": 2265448448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031890909090909094, + "loss": 2.7806, + "theoretical_loss": 3.395355085202311, + "tokens_seen": 2265513984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188888888888889, + "loss": 2.657, + "theoretical_loss": 3.395347095305176, + "tokens_seen": 2265579520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031886868686868686, + "loss": 2.4588, + "theoretical_loss": 3.3953391057038718, + "tokens_seen": 2265645056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188484848484848, + "loss": 2.8589, + "theoretical_loss": 3.3953311163983795, + "tokens_seen": 2265710592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188282828282829, + "loss": 2.6209, + "theoretical_loss": 3.395323127388679, + "tokens_seen": 2265776128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880808080808084, + "loss": 2.7148, + "theoretical_loss": 3.3953151386747513, + "tokens_seen": 2265841664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187878787878788, + "loss": 2.5451, + "theoretical_loss": 3.3953071502565764, + "tokens_seen": 2265907200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876767676767675, + "loss": 2.537, + "theoretical_loss": 3.395299162134135, + "tokens_seen": 2265972736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187474747474747, + "loss": 2.63, + "theoretical_loss": 3.395291174307408, + "tokens_seen": 2266038272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031872727272727277, + "loss": 2.4884, + "theoretical_loss": 3.395283186776375, + "tokens_seen": 2266103808 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1283537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.690002918243408, + "objective/train/theoretical_loss": 3.3952751995410173, + "objective/train/tokens_used": 625028576, + "theoretical_loss": 3.3952751995410173, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031870707070707073, + "loss": 2.7042, + "theoretical_loss": 3.3952751995410173, + "tokens_seen": 2266169344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186868686868687, + "loss": 2.6847, + "theoretical_loss": 3.395267212601315, + "tokens_seen": 2266234880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031866666666666664, + "loss": 2.5936, + "theoretical_loss": 3.3952592259572487, + "tokens_seen": 2266300416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031864646464646465, + "loss": 2.5025, + "theoretical_loss": 3.395251239608799, + "tokens_seen": 2266365952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031862626262626266, + "loss": 2.6091, + "theoretical_loss": 3.3952432535559463, + "tokens_seen": 2266431488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186060606060606, + "loss": 2.6902, + "theoretical_loss": 3.3952352677986712, + "tokens_seen": 2266497024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185858585858586, + "loss": 2.6268, + "theoretical_loss": 3.395227282336954, + "tokens_seen": 2266562560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031856565656565654, + "loss": 2.5877, + "theoretical_loss": 3.3952192971707755, + "tokens_seen": 2266628096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031854545454545455, + "loss": 2.7937, + "theoretical_loss": 3.395211312300116, + "tokens_seen": 2266693632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031852525252525256, + "loss": 2.6755, + "theoretical_loss": 3.3952033277249565, + "tokens_seen": 2266759168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185050505050505, + "loss": 2.3402, + "theoretical_loss": 3.395195343445277, + "tokens_seen": 2266824704 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031848484848484847, + "loss": 2.7971, + "theoretical_loss": 3.3951873594610578, + "tokens_seen": 2266890240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184646464646465, + "loss": 2.7485, + "theoretical_loss": 3.3951793757722797, + "tokens_seen": 2266955776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031844444444444444, + "loss": 2.5641, + "theoretical_loss": 3.395171392378924, + "tokens_seen": 2267021312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031842424242424245, + "loss": 2.6284, + "theoretical_loss": 3.39516340928097, + "tokens_seen": 2267086848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184040404040404, + "loss": 2.7499, + "theoretical_loss": 3.395155426478399, + "tokens_seen": 2267152384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031838383838383836, + "loss": 2.7842, + "theoretical_loss": 3.395147443971191, + "tokens_seen": 2267217920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183636363636364, + "loss": 2.3383, + "theoretical_loss": 3.395139461759327, + "tokens_seen": 2267283456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031834343434343433, + "loss": 2.7595, + "theoretical_loss": 3.395131479842788, + "tokens_seen": 2267348992 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031832323232323234, + "loss": 2.7914, + "theoretical_loss": 3.3951234982215532, + "tokens_seen": 2267414528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183030303030303, + "loss": 2.5715, + "theoretical_loss": 3.395115516895604, + "tokens_seen": 2267480064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182828282828283, + "loss": 2.6634, + "theoretical_loss": 3.3951075358649208, + "tokens_seen": 2267545600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031826262626262627, + "loss": 2.7236, + "theoretical_loss": 3.3950995551294842, + "tokens_seen": 2267611136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182424242424242, + "loss": 2.4207, + "theoretical_loss": 3.3950915746892747, + "tokens_seen": 2267676672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031822222222222224, + "loss": 2.5555, + "theoretical_loss": 3.395083594544273, + "tokens_seen": 2267742208 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1284224, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.767606258392334, + "objective/train/theoretical_loss": 3.395075614694459, + "objective/train/tokens_used": 626666976, + "theoretical_loss": 3.395075614694459, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182020202020202, + "loss": 2.6675, + "theoretical_loss": 3.395075614694459, + "tokens_seen": 2267807744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181818181818182, + "loss": 2.6148, + "theoretical_loss": 3.395067635139814, + "tokens_seen": 2267873280 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816161616161616, + "loss": 2.4883, + "theoretical_loss": 3.3950596558803183, + "tokens_seen": 2267938816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181414141414141, + "loss": 2.5675, + "theoretical_loss": 3.3950516769159527, + "tokens_seen": 2268004352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031812121212121213, + "loss": 2.6325, + "theoretical_loss": 3.395043698246697, + "tokens_seen": 2268069888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031810101010101014, + "loss": 2.731, + "theoretical_loss": 3.3950357198725327, + "tokens_seen": 2268135424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180808080808081, + "loss": 2.562, + "theoretical_loss": 3.39502774179344, + "tokens_seen": 2268200960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031806060606060605, + "loss": 2.8213, + "theoretical_loss": 3.395019764009399, + "tokens_seen": 2268266496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031804040404040406, + "loss": 2.6691, + "theoretical_loss": 3.3950117865203904, + "tokens_seen": 2268332032 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318020202020202, + "loss": 2.5632, + "theoretical_loss": 3.3950038093263952, + "tokens_seen": 2268397568 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031800000000000003, + "loss": 2.7816, + "theoretical_loss": 3.3949958324273943, + "tokens_seen": 2268463104 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317979797979798, + "loss": 2.5123, + "theoretical_loss": 3.3949878558233673, + "tokens_seen": 2268528640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031795959595959595, + "loss": 2.5393, + "theoretical_loss": 3.3949798795142954, + "tokens_seen": 2268594176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793939393939396, + "loss": 2.5995, + "theoretical_loss": 3.3949719035001586, + "tokens_seen": 2268659712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031791919191919197, + "loss": 2.6275, + "theoretical_loss": 3.394963927780938, + "tokens_seen": 2268725248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178989898989899, + "loss": 2.8318, + "theoretical_loss": 3.3949559523566144, + "tokens_seen": 2268790784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178787878787879, + "loss": 2.6674, + "theoretical_loss": 3.394947977227168, + "tokens_seen": 2268856320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785858585858584, + "loss": 2.7654, + "theoretical_loss": 3.3949400023925786, + "tokens_seen": 2268921856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031783838383838385, + "loss": 2.6946, + "theoretical_loss": 3.3949320278528283, + "tokens_seen": 2268987392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781818181818186, + "loss": 2.6252, + "theoretical_loss": 3.394924053607897, + "tokens_seen": 2269052928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177979797979798, + "loss": 2.7237, + "theoretical_loss": 3.394916079657765, + "tokens_seen": 2269118464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177777777777778, + "loss": 2.6838, + "theoretical_loss": 3.394908106002413, + "tokens_seen": 2269184000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031775757575757573, + "loss": 2.6968, + "theoretical_loss": 3.394900132641822, + "tokens_seen": 2269249536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177373737373738, + "loss": 2.7087, + "theoretical_loss": 3.3948921595759725, + "tokens_seen": 2269315072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771717171717175, + "loss": 2.3921, + "theoretical_loss": 3.3948841868048447, + "tokens_seen": 2269380608 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1285503, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9202969074249268, + "objective/train/theoretical_loss": 3.3948762143284195, + "objective/train/tokens_used": 628305376, + "theoretical_loss": 3.3948762143284195, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176969696969697, + "loss": 2.7717, + "theoretical_loss": 3.3948762143284195, + "tokens_seen": 2269446144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031767676767676767, + "loss": 2.8162, + "theoretical_loss": 3.394868242146677, + "tokens_seen": 2269511680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176565656565656, + "loss": 2.6963, + "theoretical_loss": 3.3948602702595987, + "tokens_seen": 2269577216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176363636363637, + "loss": 2.975, + "theoretical_loss": 3.3948522986671645, + "tokens_seen": 2269642752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031761616161616165, + "loss": 2.6897, + "theoretical_loss": 3.394844327369355, + "tokens_seen": 2269708288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175959595959596, + "loss": 2.6951, + "theoretical_loss": 3.3948363563661514, + "tokens_seen": 2269773824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031757575757575756, + "loss": 2.3717, + "theoretical_loss": 3.394828385657534, + "tokens_seen": 2269839360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175555555555555, + "loss": 2.7428, + "theoretical_loss": 3.394820415243483, + "tokens_seen": 2269904896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175353535353536, + "loss": 2.6378, + "theoretical_loss": 3.394812445123979, + "tokens_seen": 2269970432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751515151515154, + "loss": 2.5271, + "theoretical_loss": 3.3948044752990034, + "tokens_seen": 2270035968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174949494949495, + "loss": 2.7187, + "theoretical_loss": 3.3947965057685363, + "tokens_seen": 2270101504 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031747474747474745, + "loss": 2.6866, + "theoretical_loss": 3.3947885365325585, + "tokens_seen": 2270167040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745454545454546, + "loss": 2.6669, + "theoretical_loss": 3.3947805675910505, + "tokens_seen": 2270232576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174343434343435, + "loss": 2.6194, + "theoretical_loss": 3.3947725989439927, + "tokens_seen": 2270298112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741414141414143, + "loss": 2.464, + "theoretical_loss": 3.3947646305913657, + "tokens_seen": 2270363648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173939393939394, + "loss": 2.4835, + "theoretical_loss": 3.394756662533151, + "tokens_seen": 2270429184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737373737373734, + "loss": 2.7394, + "theoretical_loss": 3.394748694769328, + "tokens_seen": 2270494720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031735353535353536, + "loss": 2.6127, + "theoretical_loss": 3.394740727299878, + "tokens_seen": 2270560256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733333333333337, + "loss": 2.7019, + "theoretical_loss": 3.394732760124782, + "tokens_seen": 2270625792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173131313131313, + "loss": 2.64, + "theoretical_loss": 3.39472479324402, + "tokens_seen": 2270691328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172929292929293, + "loss": 2.7041, + "theoretical_loss": 3.3947168266575725, + "tokens_seen": 2270756864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172727272727273, + "loss": 2.6794, + "theoretical_loss": 3.3947088603654203, + "tokens_seen": 2270822400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031725252525252525, + "loss": 2.854, + "theoretical_loss": 3.3947008943675447, + "tokens_seen": 2270887936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031723232323232326, + "loss": 2.5688, + "theoretical_loss": 3.3946929286639254, + "tokens_seen": 2270953472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172121212121212, + "loss": 2.5858, + "theoretical_loss": 3.3946849632545435, + "tokens_seen": 2271019008 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1286076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.749222993850708, + "objective/train/theoretical_loss": 3.3946769981393796, + "objective/train/tokens_used": 629943776, + "theoretical_loss": 3.3946769981393796, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171919191919192, + "loss": 2.7007, + "theoretical_loss": 3.3946769981393796, + "tokens_seen": 2271084544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171717171717172, + "loss": 2.5237, + "theoretical_loss": 3.394669033318414, + "tokens_seen": 2271150080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031715151515151514, + "loss": 2.7694, + "theoretical_loss": 3.3946610687916285, + "tokens_seen": 2271215616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713131313131315, + "loss": 2.734, + "theoretical_loss": 3.3946531045590023, + "tokens_seen": 2271281152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171111111111111, + "loss": 2.7772, + "theoretical_loss": 3.3946451406205167, + "tokens_seen": 2271346688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170909090909091, + "loss": 2.7044, + "theoretical_loss": 3.3946371769761527, + "tokens_seen": 2271412224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170707070707071, + "loss": 2.6541, + "theoretical_loss": 3.3946292136258904, + "tokens_seen": 2271477760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031705050505050503, + "loss": 2.6546, + "theoretical_loss": 3.3946212505697106, + "tokens_seen": 2271543296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703030303030305, + "loss": 2.3366, + "theoretical_loss": 3.3946132878075934, + "tokens_seen": 2271608832 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317010101010101, + "loss": 2.5821, + "theoretical_loss": 3.394605325339521, + "tokens_seen": 2271674368 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316989898989899, + "loss": 2.5909, + "theoretical_loss": 3.3945973631654724, + "tokens_seen": 2271739904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031696969696969697, + "loss": 2.7556, + "theoretical_loss": 3.394589401285429, + "tokens_seen": 2271805440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169494949494949, + "loss": 2.615, + "theoretical_loss": 3.3945814396993716, + "tokens_seen": 2271870976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031692929292929294, + "loss": 2.4971, + "theoretical_loss": 3.394573478407281, + "tokens_seen": 2271936512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031690909090909095, + "loss": 2.6743, + "theoretical_loss": 3.394565517409137, + "tokens_seen": 2272002048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168888888888889, + "loss": 2.6609, + "theoretical_loss": 3.394557556704921, + "tokens_seen": 2272067584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031686868686868686, + "loss": 2.6302, + "theoretical_loss": 3.3945495962946137, + "tokens_seen": 2272133120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168484848484848, + "loss": 2.1904, + "theoretical_loss": 3.3945416361781953, + "tokens_seen": 2272198656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031682828282828283, + "loss": 2.5532, + "theoretical_loss": 3.394533676355647, + "tokens_seen": 2272264192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680808080808084, + "loss": 2.8318, + "theoretical_loss": 3.3945257168269487, + "tokens_seen": 2272329728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167878787878788, + "loss": 2.5108, + "theoretical_loss": 3.394517757592082, + "tokens_seen": 2272395264 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031676767676767676, + "loss": 2.5576, + "theoretical_loss": 3.3945097986510273, + "tokens_seen": 2272460800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167474747474747, + "loss": 2.7048, + "theoretical_loss": 3.3945018400037648, + "tokens_seen": 2272526336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167272727272728, + "loss": 2.5815, + "theoretical_loss": 3.3944938816502757, + "tokens_seen": 2272591872 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031670707070707073, + "loss": 2.5876, + "theoretical_loss": 3.3944859235905405, + "tokens_seen": 2272657408 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1287335, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.962573289871216, + "objective/train/theoretical_loss": 3.3944779658245396, + "objective/train/tokens_used": 631582176, + "theoretical_loss": 3.3944779658245396, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166868686868687, + "loss": 2.7256, + "theoretical_loss": 3.3944779658245396, + "tokens_seen": 2272722944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031666666666666665, + "loss": 2.6446, + "theoretical_loss": 3.394470008352254, + "tokens_seen": 2272788480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166464646464646, + "loss": 2.7113, + "theoretical_loss": 3.394462051173665, + "tokens_seen": 2272854016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031662626262626267, + "loss": 2.4929, + "theoretical_loss": 3.3944540942887524, + "tokens_seen": 2272919552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166060606060606, + "loss": 2.5837, + "theoretical_loss": 3.394446137697497, + "tokens_seen": 2272985088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165858585858586, + "loss": 2.6141, + "theoretical_loss": 3.3944381813998796, + "tokens_seen": 2273050624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656565656565654, + "loss": 2.3675, + "theoretical_loss": 3.3944302253958814, + "tokens_seen": 2273116160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031654545454545455, + "loss": 2.8376, + "theoretical_loss": 3.394422269685482, + "tokens_seen": 2273181696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031652525252525256, + "loss": 2.6253, + "theoretical_loss": 3.394414314268663, + "tokens_seen": 2273247232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165050505050505, + "loss": 2.6559, + "theoretical_loss": 3.394406359145405, + "tokens_seen": 2273312768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164848484848485, + "loss": 2.5129, + "theoretical_loss": 3.3943984043156887, + "tokens_seen": 2273378304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031646464646464643, + "loss": 2.7211, + "theoretical_loss": 3.3943904497794946, + "tokens_seen": 2273443840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644444444444444, + "loss": 2.5939, + "theoretical_loss": 3.3943824955368034, + "tokens_seen": 2273509376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031642424242424246, + "loss": 2.9084, + "theoretical_loss": 3.3943745415875957, + "tokens_seen": 2273574912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164040404040404, + "loss": 2.5273, + "theoretical_loss": 3.3943665879318528, + "tokens_seen": 2273640448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031638383838383837, + "loss": 2.6627, + "theoretical_loss": 3.3943586345695547, + "tokens_seen": 2273705984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163636363636364, + "loss": 2.5423, + "theoretical_loss": 3.3943506815006823, + "tokens_seen": 2273771520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163434343434344, + "loss": 2.728, + "theoretical_loss": 3.394342728725217, + "tokens_seen": 2273837056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632323232323235, + "loss": 2.6336, + "theoretical_loss": 3.3943347762431384, + "tokens_seen": 2273902592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163030303030303, + "loss": 2.6536, + "theoretical_loss": 3.394326824054428, + "tokens_seen": 2273968128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628282828282826, + "loss": 2.5242, + "theoretical_loss": 3.3943188721590665, + "tokens_seen": 2274033664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162626262626263, + "loss": 2.6659, + "theoretical_loss": 3.394310920557034, + "tokens_seen": 2274099200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162424242424243, + "loss": 2.5545, + "theoretical_loss": 3.3943029692483124, + "tokens_seen": 2274164736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031622222222222224, + "loss": 2.5029, + "theoretical_loss": 3.394295018232881, + "tokens_seen": 2274230272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162020202020202, + "loss": 2.7369, + "theoretical_loss": 3.3942870675107217, + "tokens_seen": 2274295808 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1288139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7594943046569824, + "objective/train/theoretical_loss": 3.3942791170818145, + "objective/train/tokens_used": 633220576, + "theoretical_loss": 3.3942791170818145, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618181818181815, + "loss": 2.8163, + "theoretical_loss": 3.3942791170818145, + "tokens_seen": 2274361344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616161616161617, + "loss": 2.6155, + "theoretical_loss": 3.3942711669461407, + "tokens_seen": 2274426880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161414141414142, + "loss": 2.6304, + "theoretical_loss": 3.3942632171036804, + "tokens_seen": 2274492416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031612121212121213, + "loss": 2.741, + "theoretical_loss": 3.394255267554415, + "tokens_seen": 2274557952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161010101010101, + "loss": 2.8347, + "theoretical_loss": 3.3942473182983246, + "tokens_seen": 2274623488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160808080808081, + "loss": 2.6541, + "theoretical_loss": 3.3942393693353905, + "tokens_seen": 2274689024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031606060606060606, + "loss": 2.7243, + "theoretical_loss": 3.3942314206655926, + "tokens_seen": 2274754560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604040404040407, + "loss": 2.6091, + "theoretical_loss": 3.3942234722889126, + "tokens_seen": 2274820096 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316020202020202, + "loss": 2.5127, + "theoretical_loss": 3.394215524205331, + "tokens_seen": 2274885632 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316, + "loss": 2.5109, + "theoretical_loss": 3.3942075764148285, + "tokens_seen": 2274951168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315979797979798, + "loss": 2.6052, + "theoretical_loss": 3.3941996289173857, + "tokens_seen": 2275016704 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031595959595959595, + "loss": 2.6418, + "theoretical_loss": 3.3941916817129836, + "tokens_seen": 2275082240 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031593939393939396, + "loss": 2.6157, + "theoretical_loss": 3.3941837348016026, + "tokens_seen": 2275147776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159191919191919, + "loss": 2.7339, + "theoretical_loss": 3.3941757881832233, + "tokens_seen": 2275213312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031589898989898993, + "loss": 2.9413, + "theoretical_loss": 3.3941678418578274, + "tokens_seen": 2275278848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158787878787879, + "loss": 2.5078, + "theoretical_loss": 3.394159895825395, + "tokens_seen": 2275344384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031585858585858584, + "loss": 2.5473, + "theoretical_loss": 3.3941519500859063, + "tokens_seen": 2275409920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031583838383838385, + "loss": 2.6503, + "theoretical_loss": 3.3941440046393434, + "tokens_seen": 2275475456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158181818181818, + "loss": 2.5156, + "theoretical_loss": 3.394136059485686, + "tokens_seen": 2275540992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157979797979798, + "loss": 2.4341, + "theoretical_loss": 3.394128114624915, + "tokens_seen": 2275606528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157777777777778, + "loss": 2.5858, + "theoretical_loss": 3.394120170057012, + "tokens_seen": 2275672064 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575757575757574, + "loss": 2.5978, + "theoretical_loss": 3.394112225781957, + "tokens_seen": 2275737600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031573737373737375, + "loss": 2.5942, + "theoretical_loss": 3.3941042817997307, + "tokens_seen": 2275803136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031571717171717176, + "loss": 2.835, + "theoretical_loss": 3.394096338110314, + "tokens_seen": 2275868672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003156969696969697, + "loss": 2.78, + "theoretical_loss": 3.394088394713688, + "tokens_seen": 2275934208 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 1289239, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.700709342956543, + "objective/train/theoretical_loss": 3.3940804516098333, + "objective/train/tokens_used": 634858976, + "theoretical_loss": 3.3940804516098333, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031567676767676767, + "loss": 2.5177, + "theoretical_loss": 3.3940804516098333, + "tokens_seen": 2275999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031565656565656563, + "loss": 2.5746, + "theoretical_loss": 3.3940725087987307, + "tokens_seen": 2276065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031563636363636364, + "loss": 2.6685, + "theoretical_loss": 3.3940645662803606, + "tokens_seen": 2276130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031561616161616165, + "loss": 2.7437, + "theoretical_loss": 3.3940566240547043, + "tokens_seen": 2276196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155959595959596, + "loss": 2.646, + "theoretical_loss": 3.3940486821217424, + "tokens_seen": 2276261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031557575757575757, + "loss": 2.5219, + "theoretical_loss": 3.3940407404814557, + "tokens_seen": 2276327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155555555555555, + "loss": 2.5794, + "theoretical_loss": 3.394032799133825, + "tokens_seen": 2276392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155353535353536, + "loss": 2.8988, + "theoretical_loss": 3.394024858078831, + "tokens_seen": 2276458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031551515151515154, + "loss": 2.7709, + "theoretical_loss": 3.3940169173164545, + "tokens_seen": 2276524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154949494949495, + "loss": 2.6822, + "theoretical_loss": 3.394008976846676, + "tokens_seen": 2276589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031547474747474746, + "loss": 2.7118, + "theoretical_loss": 3.394001036669477, + "tokens_seen": 2276655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154545454545454, + "loss": 2.7489, + "theoretical_loss": 3.393993096784838, + "tokens_seen": 2276720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154343434343435, + "loss": 2.6887, + "theoretical_loss": 3.3939851571927395, + "tokens_seen": 2276786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031541414141414144, + "loss": 2.4316, + "theoretical_loss": 3.393977217893163, + "tokens_seen": 2276851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153939393939394, + "loss": 2.7927, + "theoretical_loss": 3.3939692788860887, + "tokens_seen": 2276917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031537373737373735, + "loss": 2.4505, + "theoretical_loss": 3.393961340171497, + "tokens_seen": 2276982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031535353535353536, + "loss": 2.5004, + "theoretical_loss": 3.39395340174937, + "tokens_seen": 2277048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031533333333333337, + "loss": 2.7306, + "theoretical_loss": 3.393945463619687, + "tokens_seen": 2277113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031531313131313133, + "loss": 2.6, + "theoretical_loss": 3.3939375257824302, + "tokens_seen": 2277179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152929292929293, + "loss": 2.6726, + "theoretical_loss": 3.3939295882375795, + "tokens_seen": 2277244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527272727272724, + "loss": 2.6785, + "theoretical_loss": 3.3939216509851162, + "tokens_seen": 2277310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031525252525252525, + "loss": 2.7128, + "theoretical_loss": 3.393913714025021, + "tokens_seen": 2277376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523232323232327, + "loss": 2.661, + "theoretical_loss": 3.393905777357274, + "tokens_seen": 2277441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152121212121212, + "loss": 2.6237, + "theoretical_loss": 3.393897840981857, + "tokens_seen": 2277507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151919191919192, + "loss": 2.6208, + "theoretical_loss": 3.3938899048987508, + "tokens_seen": 2277572608 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1289790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6543092727661133, + "objective/train/theoretical_loss": 3.3938819691079356, + "objective/train/tokens_used": 636497376, + "theoretical_loss": 3.3938819691079356, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151717171717172, + "loss": 2.6048, + "theoretical_loss": 3.3938819691079356, + "tokens_seen": 2277638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031515151515151515, + "loss": 2.3697, + "theoretical_loss": 3.393874033609392, + "tokens_seen": 2277703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031513131313131316, + "loss": 2.5072, + "theoretical_loss": 3.393866098403102, + "tokens_seen": 2277769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151111111111111, + "loss": 2.4611, + "theoretical_loss": 3.3938581634890457, + "tokens_seen": 2277834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031509090909090907, + "loss": 2.6193, + "theoretical_loss": 3.393850228867204, + "tokens_seen": 2277900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150707070707071, + "loss": 2.6693, + "theoretical_loss": 3.3938422945375577, + "tokens_seen": 2277965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031505050505050504, + "loss": 2.5084, + "theoretical_loss": 3.3938343605000876, + "tokens_seen": 2278031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031503030303030305, + "loss": 2.7837, + "theoretical_loss": 3.3938264267547744, + "tokens_seen": 2278096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315010101010101, + "loss": 2.7756, + "theoretical_loss": 3.3938184933015996, + "tokens_seen": 2278162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314989898989899, + "loss": 2.7695, + "theoretical_loss": 3.3938105601405435, + "tokens_seen": 2278227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314969696969697, + "loss": 2.7398, + "theoretical_loss": 3.3938026272715867, + "tokens_seen": 2278293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031494949494949493, + "loss": 2.5966, + "theoretical_loss": 3.3937946946947104, + "tokens_seen": 2278359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031492929292929294, + "loss": 2.629, + "theoretical_loss": 3.393786762409896, + "tokens_seen": 2278424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149090909090909, + "loss": 2.8381, + "theoretical_loss": 3.3937788304171232, + "tokens_seen": 2278490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148888888888889, + "loss": 2.6338, + "theoretical_loss": 3.3937708987163733, + "tokens_seen": 2278555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031486868686868687, + "loss": 2.638, + "theoretical_loss": 3.393762967307628, + "tokens_seen": 2278621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148484848484848, + "loss": 2.6007, + "theoretical_loss": 3.393755036190867, + "tokens_seen": 2278686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031482828282828284, + "loss": 2.5686, + "theoretical_loss": 3.393747105366071, + "tokens_seen": 2278752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480808080808085, + "loss": 2.3691, + "theoretical_loss": 3.393739174833222, + "tokens_seen": 2278817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147878787878788, + "loss": 2.6757, + "theoretical_loss": 3.3937312445923, + "tokens_seen": 2278883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031476767676767676, + "loss": 2.4352, + "theoretical_loss": 3.393723314643286, + "tokens_seen": 2278948864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147474747474747, + "loss": 2.4303, + "theoretical_loss": 3.393715384986162, + "tokens_seen": 2279014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472727272727273, + "loss": 2.6556, + "theoretical_loss": 3.393707455620907, + "tokens_seen": 2279079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031470707070707074, + "loss": 2.7393, + "theoretical_loss": 3.3936995265475027, + "tokens_seen": 2279145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146868686868687, + "loss": 2.8891, + "theoretical_loss": 3.3936915977659305, + "tokens_seen": 2279211008 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1290983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.968740940093994, + "objective/train/theoretical_loss": 3.39368366927617, + "objective/train/tokens_used": 638135776, + "theoretical_loss": 3.39368366927617, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466666666666665, + "loss": 2.8834, + "theoretical_loss": 3.39368366927617, + "tokens_seen": 2279276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464646464646466, + "loss": 2.8011, + "theoretical_loss": 3.3936757410782032, + "tokens_seen": 2279342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146262626262627, + "loss": 2.6871, + "theoretical_loss": 3.393667813172011, + "tokens_seen": 2279407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031460606060606063, + "loss": 2.6425, + "theoretical_loss": 3.3936598855575735, + "tokens_seen": 2279473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145858585858586, + "loss": 2.6207, + "theoretical_loss": 3.393651958234872, + "tokens_seen": 2279538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031456565656565655, + "loss": 2.8368, + "theoretical_loss": 3.393644031203887, + "tokens_seen": 2279604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031454545454545456, + "loss": 2.7533, + "theoretical_loss": 3.3936361044646, + "tokens_seen": 2279669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031452525252525257, + "loss": 2.7263, + "theoretical_loss": 3.393628178016992, + "tokens_seen": 2279735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145050505050505, + "loss": 2.8641, + "theoretical_loss": 3.3936202518610425, + "tokens_seen": 2279800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144848484848485, + "loss": 2.7097, + "theoretical_loss": 3.393612325996734, + "tokens_seen": 2279866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031446464646464644, + "loss": 2.7172, + "theoretical_loss": 3.3936044004240467, + "tokens_seen": 2279931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144444444444445, + "loss": 2.7238, + "theoretical_loss": 3.3935964751429615, + "tokens_seen": 2279997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031442424242424246, + "loss": 2.7604, + "theoretical_loss": 3.393588550153459, + "tokens_seen": 2280062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003144040404040404, + "loss": 2.6306, + "theoretical_loss": 3.393580625455521, + "tokens_seen": 2280128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143838383838384, + "loss": 2.436, + "theoretical_loss": 3.3935727010491274, + "tokens_seen": 2280194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031436363636363633, + "loss": 2.5233, + "theoretical_loss": 3.3935647769342596, + "tokens_seen": 2280259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143434343434344, + "loss": 2.5811, + "theoretical_loss": 3.3935568531108986, + "tokens_seen": 2280325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031432323232323235, + "loss": 2.5796, + "theoretical_loss": 3.393548929579025, + "tokens_seen": 2280390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003143030303030303, + "loss": 2.6673, + "theoretical_loss": 3.3935410063386193, + "tokens_seen": 2280456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031428282828282827, + "loss": 2.7612, + "theoretical_loss": 3.3935330833896638, + "tokens_seen": 2280521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142626262626262, + "loss": 2.8212, + "theoretical_loss": 3.3935251607321377, + "tokens_seen": 2280587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142424242424243, + "loss": 2.663, + "theoretical_loss": 3.393517238366023, + "tokens_seen": 2280652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031422222222222225, + "loss": 2.5763, + "theoretical_loss": 3.3935093162913, + "tokens_seen": 2280718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003142020202020202, + "loss": 2.56, + "theoretical_loss": 3.3935013945079504, + "tokens_seen": 2280783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031418181818181816, + "loss": 2.9296, + "theoretical_loss": 3.3934934730159547, + "tokens_seen": 2280849408 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1291619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3147969245910645, + "objective/train/theoretical_loss": 3.3934855518152935, + "objective/train/tokens_used": 639774176, + "theoretical_loss": 3.3934855518152935, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031416161616161617, + "loss": 2.592, + "theoretical_loss": 3.3934855518152935, + "tokens_seen": 2280914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141414141414142, + "loss": 2.7454, + "theoretical_loss": 3.393477630905948, + "tokens_seen": 2280980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031412121212121214, + "loss": 2.7388, + "theoretical_loss": 3.393469710287899, + "tokens_seen": 2281046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003141010101010101, + "loss": 2.7386, + "theoretical_loss": 3.3934617899611275, + "tokens_seen": 2281111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031408080808080805, + "loss": 2.6474, + "theoretical_loss": 3.3934538699256143, + "tokens_seen": 2281177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031406060606060606, + "loss": 2.2727, + "theoretical_loss": 3.393445950181341, + "tokens_seen": 2281242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003140404040404041, + "loss": 2.8027, + "theoretical_loss": 3.393438030728287, + "tokens_seen": 2281308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031402020202020203, + "loss": 2.5566, + "theoretical_loss": 3.393430111566435, + "tokens_seen": 2281373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314, + "loss": 2.701, + "theoretical_loss": 3.3934221926957653, + "tokens_seen": 2281439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313979797979798, + "loss": 2.7578, + "theoretical_loss": 3.393414274116258, + "tokens_seen": 2281504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031395959595959596, + "loss": 2.9712, + "theoretical_loss": 3.393406355827895, + "tokens_seen": 2281570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031393939393939397, + "loss": 2.6719, + "theoretical_loss": 3.393398437830657, + "tokens_seen": 2281635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003139191919191919, + "loss": 2.6097, + "theoretical_loss": 3.3933905201245245, + "tokens_seen": 2281701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138989898989899, + "loss": 2.6856, + "theoretical_loss": 3.393382602709479, + "tokens_seen": 2281766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138787878787879, + "loss": 2.7168, + "theoretical_loss": 3.3933746855855014, + "tokens_seen": 2281832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031385858585858585, + "loss": 2.5393, + "theoretical_loss": 3.393366768752572, + "tokens_seen": 2281897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031383838383838386, + "loss": 2.6217, + "theoretical_loss": 3.393358852210673, + "tokens_seen": 2281963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003138181818181818, + "loss": 2.7786, + "theoretical_loss": 3.3933509359597838, + "tokens_seen": 2282029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031379797979797983, + "loss": 2.5535, + "theoretical_loss": 3.393343019999886, + "tokens_seen": 2282094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137777777777778, + "loss": 2.518, + "theoretical_loss": 3.393335104330961, + "tokens_seen": 2282160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031375757575757574, + "loss": 2.6657, + "theoretical_loss": 3.3933271889529895, + "tokens_seen": 2282225664 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031373737373737375, + "loss": 2.6135, + "theoretical_loss": 3.393319273865952, + "tokens_seen": 2282291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003137171717171717, + "loss": 2.662, + "theoretical_loss": 3.3933113590698305, + "tokens_seen": 2282356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136969696969697, + "loss": 2.8004, + "theoretical_loss": 3.3933034445646046, + "tokens_seen": 2282422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003136767676767677, + "loss": 2.5795, + "theoretical_loss": 3.393295530350256, + "tokens_seen": 2282487808 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1292123, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5015830993652344, + "objective/train/theoretical_loss": 3.3932876164267656, + "objective/train/tokens_used": 641412576, + "theoretical_loss": 3.3932876164267656, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031365656565656563, + "loss": 2.5286, + "theoretical_loss": 3.3932876164267656, + "tokens_seen": 2282553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031363636363636365, + "loss": 2.4843, + "theoretical_loss": 3.3932797027941146, + "tokens_seen": 2282618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031361616161616166, + "loss": 2.6528, + "theoretical_loss": 3.3932717894522835, + "tokens_seen": 2282684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003135959595959596, + "loss": 2.5263, + "theoretical_loss": 3.393263876401253, + "tokens_seen": 2282749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031357575757575757, + "loss": 2.6792, + "theoretical_loss": 3.393255963641005, + "tokens_seen": 2282815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031355555555555553, + "loss": 2.485, + "theoretical_loss": 3.39324805117152, + "tokens_seen": 2282881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031353535353535354, + "loss": 2.8307, + "theoretical_loss": 3.393240138992779, + "tokens_seen": 2282946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031351515151515155, + "loss": 2.5944, + "theoretical_loss": 3.393232227104763, + "tokens_seen": 2283012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134949494949495, + "loss": 2.6044, + "theoretical_loss": 3.393224315507452, + "tokens_seen": 2283077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031347474747474746, + "loss": 2.7923, + "theoretical_loss": 3.393216404200829, + "tokens_seen": 2283143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134545454545454, + "loss": 2.6021, + "theoretical_loss": 3.3932084931848734, + "tokens_seen": 2283208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003134343434343435, + "loss": 2.3416, + "theoretical_loss": 3.3932005824595666, + "tokens_seen": 2283274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031341414141414144, + "loss": 2.4041, + "theoretical_loss": 3.3931926720248895, + "tokens_seen": 2283339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133939393939394, + "loss": 2.7948, + "theoretical_loss": 3.3931847618808235, + "tokens_seen": 2283405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031337373737373736, + "loss": 2.5157, + "theoretical_loss": 3.393176852027349, + "tokens_seen": 2283470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133535353535353, + "loss": 2.868, + "theoretical_loss": 3.393168942464447, + "tokens_seen": 2283536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003133333333333334, + "loss": 2.7892, + "theoretical_loss": 3.3931610331920994, + "tokens_seen": 2283601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031331313131313134, + "loss": 2.5395, + "theoretical_loss": 3.3931531242102864, + "tokens_seen": 2283667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003132929292929293, + "loss": 2.56, + "theoretical_loss": 3.393145215518989, + "tokens_seen": 2283732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031327272727272725, + "loss": 2.6932, + "theoretical_loss": 3.393137307118188, + "tokens_seen": 2283798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031325252525252526, + "loss": 2.79, + "theoretical_loss": 3.393129399007865, + "tokens_seen": 2283864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031323232323232327, + "loss": 2.4567, + "theoretical_loss": 3.3931214911880008, + "tokens_seen": 2283929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031321212121212123, + "loss": 2.7182, + "theoretical_loss": 3.393113583658576, + "tokens_seen": 2283995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131919191919192, + "loss": 2.4051, + "theoretical_loss": 3.393105676419572, + "tokens_seen": 2284060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031317171717171714, + "loss": 2.7486, + "theoretical_loss": 3.3930977694709696, + "tokens_seen": 2284126208 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1293247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.631059169769287, + "objective/train/theoretical_loss": 3.3930898628127504, + "objective/train/tokens_used": 643050976, + "theoretical_loss": 3.3930898628127504, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031315151515151515, + "loss": 2.5297, + "theoretical_loss": 3.3930898628127504, + "tokens_seen": 2284191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031313131313131316, + "loss": 2.6521, + "theoretical_loss": 3.3930819564448944, + "tokens_seen": 2284257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003131111111111111, + "loss": 2.5779, + "theoretical_loss": 3.3930740503673835, + "tokens_seen": 2284322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130909090909091, + "loss": 2.5923, + "theoretical_loss": 3.3930661445801977, + "tokens_seen": 2284388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003130707070707071, + "loss": 2.7683, + "theoretical_loss": 3.3930582390833193, + "tokens_seen": 2284453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031305050505050505, + "loss": 2.6935, + "theoretical_loss": 3.3930503338767286, + "tokens_seen": 2284519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031303030303030306, + "loss": 2.8456, + "theoretical_loss": 3.393042428960406, + "tokens_seen": 2284584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000313010101010101, + "loss": 2.7877, + "theoretical_loss": 3.393034524334334, + "tokens_seen": 2284650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031298989898989897, + "loss": 2.7203, + "theoretical_loss": 3.3930266199984924, + "tokens_seen": 2284716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312969696969697, + "loss": 2.6002, + "theoretical_loss": 3.3930187159528624, + "tokens_seen": 2284781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031294949494949494, + "loss": 2.4277, + "theoretical_loss": 3.393010812197425, + "tokens_seen": 2284847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031292929292929295, + "loss": 2.4764, + "theoretical_loss": 3.3930029087321625, + "tokens_seen": 2284912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003129090909090909, + "loss": 2.7543, + "theoretical_loss": 3.3929950055570544, + "tokens_seen": 2284978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031288888888888886, + "loss": 2.6969, + "theoretical_loss": 3.3929871026720817, + "tokens_seen": 2285043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128686868686869, + "loss": 2.554, + "theoretical_loss": 3.3929792000772263, + "tokens_seen": 2285109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128484848484849, + "loss": 2.6198, + "theoretical_loss": 3.392971297772469, + "tokens_seen": 2285174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031282828282828284, + "loss": 2.5844, + "theoretical_loss": 3.3929633957577905, + "tokens_seen": 2285240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003128080808080808, + "loss": 2.8193, + "theoretical_loss": 3.3929554940331723, + "tokens_seen": 2285305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127878787878788, + "loss": 2.6017, + "theoretical_loss": 3.392947592598595, + "tokens_seen": 2285371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031276767676767677, + "loss": 2.5296, + "theoretical_loss": 3.3929396914540395, + "tokens_seen": 2285436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127474747474748, + "loss": 2.6244, + "theoretical_loss": 3.3929317905994876, + "tokens_seen": 2285502464 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031272727272727273, + "loss": 2.6866, + "theoretical_loss": 3.3929238900349197, + "tokens_seen": 2285568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003127070707070707, + "loss": 2.6911, + "theoretical_loss": 3.3929159897603167, + "tokens_seen": 2285633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126868686868687, + "loss": 2.39, + "theoretical_loss": 3.3929080897756605, + "tokens_seen": 2285699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031266666666666666, + "loss": 2.7141, + "theoretical_loss": 3.3929001900809315, + "tokens_seen": 2285764608 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1293760, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.768867254257202, + "objective/train/theoretical_loss": 3.3928922906761105, + "objective/train/tokens_used": 644689376, + "theoretical_loss": 3.3928922906761105, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031264646464646467, + "loss": 2.7658, + "theoretical_loss": 3.3928922906761105, + "tokens_seen": 2285830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003126262626262626, + "loss": 2.8177, + "theoretical_loss": 3.392884391561179, + "tokens_seen": 2285895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031260606060606064, + "loss": 2.6825, + "theoretical_loss": 3.3928764927361184, + "tokens_seen": 2285961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125858585858586, + "loss": 2.4876, + "theoretical_loss": 3.392868594200909, + "tokens_seen": 2286026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031256565656565655, + "loss": 2.8075, + "theoretical_loss": 3.392860695955532, + "tokens_seen": 2286092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031254545454545456, + "loss": 2.8573, + "theoretical_loss": 3.3928527979999688, + "tokens_seen": 2286157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003125252525252525, + "loss": 2.7313, + "theoretical_loss": 3.3928449003342003, + "tokens_seen": 2286223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031250505050505053, + "loss": 2.6927, + "theoretical_loss": 3.3928370029582076, + "tokens_seen": 2286288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124848484848485, + "loss": 2.7244, + "theoretical_loss": 3.3928291058719715, + "tokens_seen": 2286354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031246464646464644, + "loss": 2.5998, + "theoretical_loss": 3.3928212090754735, + "tokens_seen": 2286419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031244444444444446, + "loss": 2.7578, + "theoretical_loss": 3.3928133125686943, + "tokens_seen": 2286485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031242424242424247, + "loss": 2.6236, + "theoretical_loss": 3.3928054163516155, + "tokens_seen": 2286551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003124040404040404, + "loss": 2.7602, + "theoretical_loss": 3.392797520424217, + "tokens_seen": 2286616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123838383838384, + "loss": 2.709, + "theoretical_loss": 3.3927896247864813, + "tokens_seen": 2286682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031236363636363634, + "loss": 2.3687, + "theoretical_loss": 3.392781729438388, + "tokens_seen": 2286747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031234343434343435, + "loss": 2.6726, + "theoretical_loss": 3.39277383437992, + "tokens_seen": 2286813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031232323232323236, + "loss": 2.6965, + "theoretical_loss": 3.3927659396110568, + "tokens_seen": 2286878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003123030303030303, + "loss": 2.7106, + "theoretical_loss": 3.39275804513178, + "tokens_seen": 2286944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122828282828283, + "loss": 2.5242, + "theoretical_loss": 3.3927501509420708, + "tokens_seen": 2287009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031226262626262623, + "loss": 2.5724, + "theoretical_loss": 3.3927422570419106, + "tokens_seen": 2287075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122424242424243, + "loss": 2.518, + "theoretical_loss": 3.3927343634312797, + "tokens_seen": 2287140864 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031222222222222225, + "loss": 2.7226, + "theoretical_loss": 3.3927264701101594, + "tokens_seen": 2287206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003122020202020202, + "loss": 2.6079, + "theoretical_loss": 3.392718577078531, + "tokens_seen": 2287271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031218181818181817, + "loss": 2.5639, + "theoretical_loss": 3.392710684336376, + "tokens_seen": 2287337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121616161616161, + "loss": 2.7112, + "theoretical_loss": 3.3927027918836745, + "tokens_seen": 2287403008 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1294930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5454752445220947, + "objective/train/theoretical_loss": 3.3926948997204085, + "objective/train/tokens_used": 646327776, + "theoretical_loss": 3.3926948997204085, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121414141414142, + "loss": 2.5384, + "theoretical_loss": 3.3926948997204085, + "tokens_seen": 2287468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031212121212121214, + "loss": 2.3836, + "theoretical_loss": 3.3926870078465585, + "tokens_seen": 2287534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003121010101010101, + "loss": 2.8493, + "theoretical_loss": 3.392679116262106, + "tokens_seen": 2287599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031208080808080806, + "loss": 2.75, + "theoretical_loss": 3.3926712249670317, + "tokens_seen": 2287665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031206060606060607, + "loss": 2.9176, + "theoretical_loss": 3.392663333961317, + "tokens_seen": 2287730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003120404040404041, + "loss": 2.7044, + "theoretical_loss": 3.392655443244943, + "tokens_seen": 2287796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031202020202020204, + "loss": 2.5219, + "theoretical_loss": 3.3926475528178908, + "tokens_seen": 2287861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.000312, + "loss": 2.822, + "theoretical_loss": 3.392639662680141, + "tokens_seen": 2287927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031197979797979795, + "loss": 2.8442, + "theoretical_loss": 3.3926317728316757, + "tokens_seen": 2287992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031195959595959596, + "loss": 2.6715, + "theoretical_loss": 3.392623883272475, + "tokens_seen": 2288058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311939393939394, + "loss": 2.713, + "theoretical_loss": 3.3926159940025205, + "tokens_seen": 2288123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031191919191919193, + "loss": 2.6284, + "theoretical_loss": 3.3926081050217936, + "tokens_seen": 2288189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118989898989899, + "loss": 2.7606, + "theoretical_loss": 3.392600216330275, + "tokens_seen": 2288254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118787878787879, + "loss": 2.7515, + "theoretical_loss": 3.3925923279279453, + "tokens_seen": 2288320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031185858585858585, + "loss": 2.4116, + "theoretical_loss": 3.3925844398147866, + "tokens_seen": 2288386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031183838383838387, + "loss": 2.5794, + "theoretical_loss": 3.39257655199078, + "tokens_seen": 2288451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003118181818181818, + "loss": 2.5323, + "theoretical_loss": 3.3925686644559057, + "tokens_seen": 2288517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117979797979798, + "loss": 2.8981, + "theoretical_loss": 3.3925607772101456, + "tokens_seen": 2288582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117777777777778, + "loss": 2.456, + "theoretical_loss": 3.39255289025348, + "tokens_seen": 2288648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031175757575757575, + "loss": 2.7548, + "theoretical_loss": 3.3925450035858913, + "tokens_seen": 2288713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031173737373737376, + "loss": 2.695, + "theoretical_loss": 3.3925371172073597, + "tokens_seen": 2288779264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003117171717171717, + "loss": 2.707, + "theoretical_loss": 3.3925292311178668, + "tokens_seen": 2288844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116969696969697, + "loss": 2.5185, + "theoretical_loss": 3.3925213453173932, + "tokens_seen": 2288910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116767676767677, + "loss": 2.8847, + "theoretical_loss": 3.3925134598059206, + "tokens_seen": 2288975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031165656565656564, + "loss": 2.8396, + "theoretical_loss": 3.3925055745834296, + "tokens_seen": 2289041408 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1295606, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.113692045211792, + "objective/train/theoretical_loss": 3.392497689649902, + "objective/train/tokens_used": 647966176, + "theoretical_loss": 3.392497689649902, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031163636363636365, + "loss": 2.8057, + "theoretical_loss": 3.392497689649902, + "tokens_seen": 2289106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003116161616161616, + "loss": 2.8307, + "theoretical_loss": 3.3924898050053183, + "tokens_seen": 2289172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115959595959596, + "loss": 2.7058, + "theoretical_loss": 3.3924819206496597, + "tokens_seen": 2289238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003115757575757576, + "loss": 2.5537, + "theoretical_loss": 3.392474036582908, + "tokens_seen": 2289303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031155555555555553, + "loss": 2.5137, + "theoretical_loss": 3.3924661528050435, + "tokens_seen": 2289369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031153535353535354, + "loss": 2.573, + "theoretical_loss": 3.3924582693160477, + "tokens_seen": 2289434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031151515151515156, + "loss": 2.6735, + "theoretical_loss": 3.3924503861159017, + "tokens_seen": 2289500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114949494949495, + "loss": 2.6006, + "theoretical_loss": 3.392442503204587, + "tokens_seen": 2289565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031147474747474747, + "loss": 2.6069, + "theoretical_loss": 3.392434620582084, + "tokens_seen": 2289631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003114545454545454, + "loss": 2.726, + "theoretical_loss": 3.392426738248375, + "tokens_seen": 2289696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031143434343434344, + "loss": 2.5095, + "theoretical_loss": 3.39241885620344, + "tokens_seen": 2289762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031141414141414145, + "loss": 2.7781, + "theoretical_loss": 3.3924109744472606, + "tokens_seen": 2289827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113939393939394, + "loss": 2.6291, + "theoretical_loss": 3.3924030929798183, + "tokens_seen": 2289893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031137373737373736, + "loss": 2.5594, + "theoretical_loss": 3.3923952118010936, + "tokens_seen": 2289958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113535353535353, + "loss": 2.4947, + "theoretical_loss": 3.392387330911068, + "tokens_seen": 2290024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003113333333333334, + "loss": 2.8011, + "theoretical_loss": 3.392379450309723, + "tokens_seen": 2290089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031131313131313134, + "loss": 2.504, + "theoretical_loss": 3.392371569997039, + "tokens_seen": 2290155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112929292929293, + "loss": 2.6959, + "theoretical_loss": 3.392363689972998, + "tokens_seen": 2290221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031127272727272725, + "loss": 2.6852, + "theoretical_loss": 3.3923558102375804, + "tokens_seen": 2290286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112525252525252, + "loss": 2.834, + "theoretical_loss": 3.392347930790768, + "tokens_seen": 2290352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003112323232323233, + "loss": 2.7857, + "theoretical_loss": 3.3923400516325417, + "tokens_seen": 2290417664 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031121212121212123, + "loss": 2.6232, + "theoretical_loss": 3.392332172762883, + "tokens_seen": 2290483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111919191919192, + "loss": 2.7227, + "theoretical_loss": 3.3923242941817717, + "tokens_seen": 2290548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031117171717171715, + "loss": 2.4779, + "theoretical_loss": 3.3923164158891908, + "tokens_seen": 2290614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111515151515152, + "loss": 2.5849, + "theoretical_loss": 3.3923085378851208, + "tokens_seen": 2290679808 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3294901847839355, + "objective/train/theoretical_loss": 3.3923006601695427, + "objective/train/tokens_used": 649604576, + "theoretical_loss": 3.3923006601695427, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031113131313131317, + "loss": 2.7014, + "theoretical_loss": 3.3923006601695427, + "tokens_seen": 2290745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003111111111111111, + "loss": 2.8297, + "theoretical_loss": 3.3922927827424374, + "tokens_seen": 2290810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110909090909091, + "loss": 2.7365, + "theoretical_loss": 3.3922849056037867, + "tokens_seen": 2290876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031107070707070704, + "loss": 2.6501, + "theoretical_loss": 3.392277028753572, + "tokens_seen": 2290941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003110505050505051, + "loss": 2.5704, + "theoretical_loss": 3.3922691521917736, + "tokens_seen": 2291007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031103030303030306, + "loss": 2.7115, + "theoretical_loss": 3.392261275918373, + "tokens_seen": 2291073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.000311010101010101, + "loss": 2.6757, + "theoretical_loss": 3.3922533999333515, + "tokens_seen": 2291138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310989898989899, + "loss": 2.7173, + "theoretical_loss": 3.3922455242366905, + "tokens_seen": 2291204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031096969696969693, + "loss": 2.512, + "theoretical_loss": 3.392237648828371, + "tokens_seen": 2291269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.000310949494949495, + "loss": 2.7061, + "theoretical_loss": 3.392229773708374, + "tokens_seen": 2291335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031092929292929295, + "loss": 2.744, + "theoretical_loss": 3.3922218988766812, + "tokens_seen": 2291400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003109090909090909, + "loss": 2.6969, + "theoretical_loss": 3.3922140243332732, + "tokens_seen": 2291466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031088888888888887, + "loss": 2.7261, + "theoretical_loss": 3.3922061500781315, + "tokens_seen": 2291531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108686868686869, + "loss": 2.7356, + "theoretical_loss": 3.3921982761112375, + "tokens_seen": 2291597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108484848484849, + "loss": 2.7848, + "theoretical_loss": 3.392190402432572, + "tokens_seen": 2291662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031082828282828285, + "loss": 2.4382, + "theoretical_loss": 3.392182529042117, + "tokens_seen": 2291728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003108080808080808, + "loss": 2.5894, + "theoretical_loss": 3.3921746559398525, + "tokens_seen": 2291793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031078787878787876, + "loss": 2.6754, + "theoretical_loss": 3.3921667831257607, + "tokens_seen": 2291859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031076767676767677, + "loss": 2.7722, + "theoretical_loss": 3.392158910599822, + "tokens_seen": 2291924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107474747474748, + "loss": 2.6494, + "theoretical_loss": 3.3921510383620186, + "tokens_seen": 2291990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031072727272727274, + "loss": 2.6683, + "theoretical_loss": 3.3921431664123305, + "tokens_seen": 2292056064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003107070707070707, + "loss": 2.8466, + "theoretical_loss": 3.39213529475074, + "tokens_seen": 2292121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106868686868687, + "loss": 2.5207, + "theoretical_loss": 3.392127423377228, + "tokens_seen": 2292187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031066666666666666, + "loss": 2.5336, + "theoretical_loss": 3.3921195522917755, + "tokens_seen": 2292252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003106464646464647, + "loss": 2.5477, + "theoretical_loss": 3.392111681494364, + "tokens_seen": 2292318208 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.478738784790039, + "objective/train/theoretical_loss": 3.3921038109849744, + "objective/train/tokens_used": 651242976, + "theoretical_loss": 3.3921038109849744, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031062626262626263, + "loss": 2.7095, + "theoretical_loss": 3.3921038109849744, + "tokens_seen": 2292383744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003106060606060606, + "loss": 2.938, + "theoretical_loss": 3.3920959407635882, + "tokens_seen": 2292449280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105858585858586, + "loss": 2.8258, + "theoretical_loss": 3.3920880708301864, + "tokens_seen": 2292514816 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031056565656565656, + "loss": 2.7483, + "theoretical_loss": 3.3920802011847506, + "tokens_seen": 2292580352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031054545454545457, + "loss": 2.8156, + "theoretical_loss": 3.3920723318272614, + "tokens_seen": 2292645888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003105252525252525, + "loss": 2.9363, + "theoretical_loss": 3.392064462757701, + "tokens_seen": 2292711424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031050505050505054, + "loss": 2.6891, + "theoretical_loss": 3.39205659397605, + "tokens_seen": 2292776960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104848484848485, + "loss": 2.7168, + "theoretical_loss": 3.3920487254822893, + "tokens_seen": 2292842496 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031046464646464645, + "loss": 2.8379, + "theoretical_loss": 3.3920408572764007, + "tokens_seen": 2292908032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031044444444444446, + "loss": 2.8652, + "theoretical_loss": 3.392032989358365, + "tokens_seen": 2292973568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003104242424242424, + "loss": 2.7438, + "theoretical_loss": 3.392025121728164, + "tokens_seen": 2293039104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031040404040404043, + "loss": 2.6891, + "theoretical_loss": 3.3920172543857787, + "tokens_seen": 2293104640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103838383838384, + "loss": 2.763, + "theoretical_loss": 3.39200938733119, + "tokens_seen": 2293170176 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031036363636363634, + "loss": 2.8258, + "theoretical_loss": 3.39200152056438, + "tokens_seen": 2293235712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031034343434343435, + "loss": 2.682, + "theoretical_loss": 3.3919936540853293, + "tokens_seen": 2293301248 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031032323232323236, + "loss": 2.4021, + "theoretical_loss": 3.3919857878940194, + "tokens_seen": 2293366784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003103030303030303, + "loss": 2.6128, + "theoretical_loss": 3.391977921990431, + "tokens_seen": 2293432320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102828282828283, + "loss": 2.9165, + "theoretical_loss": 3.3919700563745456, + "tokens_seen": 2293497856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031026262626262624, + "loss": 2.936, + "theoretical_loss": 3.3919621910463453, + "tokens_seen": 2293563392 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031024242424242425, + "loss": 2.813, + "theoretical_loss": 3.39195432600581, + "tokens_seen": 2293628928 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031022222222222226, + "loss": 2.723, + "theoretical_loss": 3.391946461252922, + "tokens_seen": 2293694464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003102020202020202, + "loss": 2.9012, + "theoretical_loss": 3.3919385967876625, + "tokens_seen": 2293760000 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031018181818181817, + "loss": 2.5958, + "theoretical_loss": 3.391930732610012, + "tokens_seen": 2293825536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031016161616161613, + "loss": 2.9196, + "theoretical_loss": 3.3919228687199525, + "tokens_seen": 2293891072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101414141414142, + "loss": 2.6752, + "theoretical_loss": 3.3919150051174647, + "tokens_seen": 2293956608 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6804754734039307, + "objective/train/theoretical_loss": 3.3919071418025304, + "objective/train/tokens_used": 652881376, + "theoretical_loss": 3.3919071418025304, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031012121212121215, + "loss": 2.7294, + "theoretical_loss": 3.3919071418025304, + "tokens_seen": 2294022144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003101010101010101, + "loss": 2.7259, + "theoretical_loss": 3.3918992787751305, + "tokens_seen": 2294087680 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031008080808080806, + "loss": 2.7297, + "theoretical_loss": 3.391891416035247, + "tokens_seen": 2294153216 + }, + { + "epoch": 0.39, + "learning_rate": 0.000310060606060606, + "loss": 2.7629, + "theoretical_loss": 3.39188355358286, + "tokens_seen": 2294218752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003100404040404041, + "loss": 2.8312, + "theoretical_loss": 3.3918756914179515, + "tokens_seen": 2294284288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031002020202020204, + "loss": 2.7235, + "theoretical_loss": 3.3918678295405025, + "tokens_seen": 2294349824 + }, + { + "epoch": 0.39, + "learning_rate": 0.00031, + "loss": 2.9055, + "theoretical_loss": 3.3918599679504946, + "tokens_seen": 2294415360 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030997979797979796, + "loss": 2.7504, + "theoretical_loss": 3.391852106647909, + "tokens_seen": 2294480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030995959595959597, + "loss": 2.7924, + "theoretical_loss": 3.3918442456327265, + "tokens_seen": 2294546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309939393939394, + "loss": 2.7638, + "theoretical_loss": 3.391836384904929, + "tokens_seen": 2294611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030991919191919194, + "loss": 2.9706, + "theoretical_loss": 3.3918285244644975, + "tokens_seen": 2294677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003098989898989899, + "loss": 2.9993, + "theoretical_loss": 3.3918206643114135, + "tokens_seen": 2294743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030987878787878785, + "loss": 2.7997, + "theoretical_loss": 3.391812804445658, + "tokens_seen": 2294808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030985858585858586, + "loss": 2.7279, + "theoretical_loss": 3.3918049448672125, + "tokens_seen": 2294874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030983838383838387, + "loss": 2.9885, + "theoretical_loss": 3.391797085576058, + "tokens_seen": 2294939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030981818181818183, + "loss": 3.0366, + "theoretical_loss": 3.3917892265721763, + "tokens_seen": 2295005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097979797979798, + "loss": 2.8506, + "theoretical_loss": 3.391781367855548, + "tokens_seen": 2295070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097777777777778, + "loss": 2.7856, + "theoretical_loss": 3.391773509426155, + "tokens_seen": 2295136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030975757575757575, + "loss": 2.8729, + "theoretical_loss": 3.3917656512839787, + "tokens_seen": 2295201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030973737373737376, + "loss": 2.6805, + "theoretical_loss": 3.391757793429, + "tokens_seen": 2295267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003097171717171717, + "loss": 3.0021, + "theoretical_loss": 3.3917499358612, + "tokens_seen": 2295332864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096969696969697, + "loss": 2.8128, + "theoretical_loss": 3.391742078580561, + "tokens_seen": 2295398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096767676767677, + "loss": 2.7601, + "theoretical_loss": 3.391734221587063, + "tokens_seen": 2295463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030965656565656565, + "loss": 2.8974, + "theoretical_loss": 3.391726364880688, + "tokens_seen": 2295529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030963636363636366, + "loss": 2.6921, + "theoretical_loss": 3.3917185084614174, + "tokens_seen": 2295595008 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6300604343414307, + "objective/train/theoretical_loss": 3.391710652329232, + "objective/train/tokens_used": 654519776, + "theoretical_loss": 3.391710652329232, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003096161616161616, + "loss": 2.9572, + "theoretical_loss": 3.391710652329232, + "tokens_seen": 2295660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030959595959595957, + "loss": 2.8242, + "theoretical_loss": 3.391702796484114, + "tokens_seen": 2295726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095757575757576, + "loss": 2.841, + "theoretical_loss": 3.391694940926044, + "tokens_seen": 2295791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030955555555555554, + "loss": 2.8566, + "theoretical_loss": 3.3916870856550037, + "tokens_seen": 2295857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030953535353535355, + "loss": 2.7818, + "theoretical_loss": 3.3916792306709738, + "tokens_seen": 2295922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003095151515151515, + "loss": 2.8407, + "theoretical_loss": 3.391671375973936, + "tokens_seen": 2295988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094949494949495, + "loss": 2.9361, + "theoretical_loss": 3.3916635215638724, + "tokens_seen": 2296053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094747474747475, + "loss": 2.9141, + "theoretical_loss": 3.391655667440763, + "tokens_seen": 2296119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094545454545455, + "loss": 2.6661, + "theoretical_loss": 3.39164781360459, + "tokens_seen": 2296184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030943434343434344, + "loss": 2.9636, + "theoretical_loss": 3.391639960055334, + "tokens_seen": 2296250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003094141414141414, + "loss": 2.6961, + "theoretical_loss": 3.391632106792977, + "tokens_seen": 2296315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093939393939394, + "loss": 2.8512, + "theoretical_loss": 3.3916242538175005, + "tokens_seen": 2296381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030937373737373737, + "loss": 2.8879, + "theoretical_loss": 3.3916164011288847, + "tokens_seen": 2296446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003093535353535354, + "loss": 2.8129, + "theoretical_loss": 3.3916085487271124, + "tokens_seen": 2296512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030933333333333334, + "loss": 2.8111, + "theoretical_loss": 3.391600696612164, + "tokens_seen": 2296578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030931313131313135, + "loss": 3.0447, + "theoretical_loss": 3.3915928447840207, + "tokens_seen": 2296643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003092929292929293, + "loss": 2.7961, + "theoretical_loss": 3.3915849932426645, + "tokens_seen": 2296709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030927272727272726, + "loss": 3.0225, + "theoretical_loss": 3.3915771419880762, + "tokens_seen": 2296774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030925252525252527, + "loss": 3.0279, + "theoretical_loss": 3.3915692910202377, + "tokens_seen": 2296840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030923232323232323, + "loss": 2.6538, + "theoretical_loss": 3.39156144033913, + "tokens_seen": 2296905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030921212121212124, + "loss": 2.6334, + "theoretical_loss": 3.3915535899447344, + "tokens_seen": 2296971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091919191919192, + "loss": 2.8902, + "theoretical_loss": 3.3915457398370323, + "tokens_seen": 2297036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030917171717171715, + "loss": 2.844, + "theoretical_loss": 3.391537890016005, + "tokens_seen": 2297102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030915151515151516, + "loss": 2.957, + "theoretical_loss": 3.3915300404816344, + "tokens_seen": 2297167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003091313131313132, + "loss": 2.8537, + "theoretical_loss": 3.391522191233901, + "tokens_seen": 2297233408 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9326298236846924, + "objective/train/theoretical_loss": 3.391514342272786, + "objective/train/tokens_used": 656158176, + "theoretical_loss": 3.391514342272786, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030911111111111113, + "loss": 2.9218, + "theoretical_loss": 3.391514342272786, + "tokens_seen": 2297298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003090909090909091, + "loss": 2.9966, + "theoretical_loss": 3.391506493598272, + "tokens_seen": 2297364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030907070707070705, + "loss": 2.8629, + "theoretical_loss": 3.3914986452103397, + "tokens_seen": 2297430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030905050505050506, + "loss": 2.8437, + "theoretical_loss": 3.39149079710897, + "tokens_seen": 2297495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030903030303030307, + "loss": 2.6681, + "theoretical_loss": 3.391482949294145, + "tokens_seen": 2297561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.000309010101010101, + "loss": 2.9994, + "theoretical_loss": 3.3914751017658458, + "tokens_seen": 2297626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308989898989899, + "loss": 2.8402, + "theoretical_loss": 3.3914672545240534, + "tokens_seen": 2297692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030896969696969694, + "loss": 2.8483, + "theoretical_loss": 3.3914594075687496, + "tokens_seen": 2297757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308949494949495, + "loss": 2.8081, + "theoretical_loss": 3.391451560899916, + "tokens_seen": 2297823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030892929292929296, + "loss": 2.9542, + "theoretical_loss": 3.391443714517533, + "tokens_seen": 2297888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003089090909090909, + "loss": 2.5247, + "theoretical_loss": 3.391435868421583, + "tokens_seen": 2297954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088888888888889, + "loss": 2.9018, + "theoretical_loss": 3.391428022612047, + "tokens_seen": 2298019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030886868686868683, + "loss": 2.9597, + "theoretical_loss": 3.391420177088906, + "tokens_seen": 2298085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088484848484849, + "loss": 3.0003, + "theoretical_loss": 3.391412331852142, + "tokens_seen": 2298150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030882828282828285, + "loss": 2.9256, + "theoretical_loss": 3.391404486901736, + "tokens_seen": 2298216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003088080808080808, + "loss": 2.7586, + "theoretical_loss": 3.3913966422376696, + "tokens_seen": 2298281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030878787878787877, + "loss": 2.7901, + "theoretical_loss": 3.391388797859924, + "tokens_seen": 2298347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087676767676768, + "loss": 2.8721, + "theoretical_loss": 3.3913809537684805, + "tokens_seen": 2298413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087474747474748, + "loss": 2.8997, + "theoretical_loss": 3.391373109963321, + "tokens_seen": 2298478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030872727272727275, + "loss": 2.8015, + "theoretical_loss": 3.391365266444426, + "tokens_seen": 2298544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003087070707070707, + "loss": 3.0476, + "theoretical_loss": 3.3913574232117782, + "tokens_seen": 2298609664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030868686868686866, + "loss": 2.8377, + "theoretical_loss": 3.3913495802653575, + "tokens_seen": 2298675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030866666666666667, + "loss": 2.9171, + "theoretical_loss": 3.3913417376051465, + "tokens_seen": 2298740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086464646464647, + "loss": 2.7859, + "theoretical_loss": 3.391333895231126, + "tokens_seen": 2298806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030862626262626264, + "loss": 2.9737, + "theoretical_loss": 3.3913260531432776, + "tokens_seen": 2298871808 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1296233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.943025588989258, + "objective/train/theoretical_loss": 3.3913182113415825, + "objective/train/tokens_used": 657796576, + "theoretical_loss": 3.3913182113415825, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003086060606060606, + "loss": 2.9505, + "theoretical_loss": 3.3913182113415825, + "tokens_seen": 2298937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085858585858586, + "loss": 2.7605, + "theoretical_loss": 3.391310369826022, + "tokens_seen": 2299002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030856565656565656, + "loss": 3.2008, + "theoretical_loss": 3.3913025285965777, + "tokens_seen": 2299068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085454545454546, + "loss": 2.9413, + "theoretical_loss": 3.3912946876532315, + "tokens_seen": 2299133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030852525252525253, + "loss": 2.9043, + "theoretical_loss": 3.391286846995964, + "tokens_seen": 2299199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003085050505050505, + "loss": 2.7444, + "theoretical_loss": 3.3912790066247567, + "tokens_seen": 2299265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084848484848485, + "loss": 2.7879, + "theoretical_loss": 3.3912711665395916, + "tokens_seen": 2299330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030846464646464646, + "loss": 2.9863, + "theoretical_loss": 3.3912633267404497, + "tokens_seen": 2299396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030844444444444447, + "loss": 2.8726, + "theoretical_loss": 3.391255487227313, + "tokens_seen": 2299461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003084242424242424, + "loss": 2.9346, + "theoretical_loss": 3.3912476480001614, + "tokens_seen": 2299527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030840404040404043, + "loss": 2.8333, + "theoretical_loss": 3.3912398090589777, + "tokens_seen": 2299592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083838383838384, + "loss": 2.8907, + "theoretical_loss": 3.391231970403743, + "tokens_seen": 2299658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030836363636363635, + "loss": 2.9655, + "theoretical_loss": 3.3912241320344387, + "tokens_seen": 2299723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030834343434343436, + "loss": 2.9461, + "theoretical_loss": 3.3912162939510457, + "tokens_seen": 2299789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003083232323232323, + "loss": 2.8861, + "theoretical_loss": 3.3912084561535467, + "tokens_seen": 2299854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030830303030303033, + "loss": 3.1383, + "theoretical_loss": 3.3912006186419217, + "tokens_seen": 2299920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082828282828283, + "loss": 2.822, + "theoretical_loss": 3.391192781416153, + "tokens_seen": 2299985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030826262626262624, + "loss": 2.8816, + "theoretical_loss": 3.3911849444762217, + "tokens_seen": 2300051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030824242424242425, + "loss": 2.901, + "theoretical_loss": 3.3911771078221093, + "tokens_seen": 2300116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030822222222222226, + "loss": 2.9067, + "theoretical_loss": 3.391169271453797, + "tokens_seen": 2300182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003082020202020202, + "loss": 2.9857, + "theoretical_loss": 3.3911614353712665, + "tokens_seen": 2300248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081818181818182, + "loss": 2.9499, + "theoretical_loss": 3.3911535995744995, + "tokens_seen": 2300313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030816161616161613, + "loss": 2.9482, + "theoretical_loss": 3.3911457640634772, + "tokens_seen": 2300379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030814141414141414, + "loss": 2.6845, + "theoretical_loss": 3.3911379288381807, + "tokens_seen": 2300444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030812121212121216, + "loss": 2.7828, + "theoretical_loss": 3.391130093898592, + "tokens_seen": 2300510208 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1296929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.806436777114868, + "objective/train/theoretical_loss": 3.391122259244692, + "objective/train/tokens_used": 659434976, + "theoretical_loss": 3.391122259244692, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003081010101010101, + "loss": 2.9523, + "theoretical_loss": 3.391122259244692, + "tokens_seen": 2300575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030808080808080807, + "loss": 2.8884, + "theoretical_loss": 3.3911144248764624, + "tokens_seen": 2300641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308060606060606, + "loss": 2.9124, + "theoretical_loss": 3.3911065907938847, + "tokens_seen": 2300706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003080404040404041, + "loss": 2.9183, + "theoretical_loss": 3.3910987569969406, + "tokens_seen": 2300772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030802020202020205, + "loss": 2.8528, + "theoretical_loss": 3.391090923485611, + "tokens_seen": 2300837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.000308, + "loss": 2.7586, + "theoretical_loss": 3.391083090259878, + "tokens_seen": 2300903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030797979797979796, + "loss": 2.9106, + "theoretical_loss": 3.391075257319722, + "tokens_seen": 2300968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003079595959595959, + "loss": 2.7864, + "theoretical_loss": 3.391067424665126, + "tokens_seen": 2301034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.000307939393939394, + "loss": 2.9043, + "theoretical_loss": 3.3910595922960693, + "tokens_seen": 2301100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030791919191919194, + "loss": 2.8346, + "theoretical_loss": 3.3910517602125356, + "tokens_seen": 2301165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078989898989899, + "loss": 2.996, + "theoretical_loss": 3.391043928414505, + "tokens_seen": 2301231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030787878787878786, + "loss": 2.9039, + "theoretical_loss": 3.3910360969019595, + "tokens_seen": 2301296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078585858585858, + "loss": 2.6772, + "theoretical_loss": 3.3910282656748807, + "tokens_seen": 2301362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003078383838383839, + "loss": 2.8175, + "theoretical_loss": 3.391020434733249, + "tokens_seen": 2301427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030781818181818183, + "loss": 2.9311, + "theoretical_loss": 3.3910126040770474, + "tokens_seen": 2301493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077979797979798, + "loss": 2.977, + "theoretical_loss": 3.3910047737062565, + "tokens_seen": 2301558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030777777777777775, + "loss": 2.7742, + "theoretical_loss": 3.390996943620858, + "tokens_seen": 2301624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077575757575758, + "loss": 2.7183, + "theoretical_loss": 3.3909891138208326, + "tokens_seen": 2301689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030773737373737377, + "loss": 2.7582, + "theoretical_loss": 3.390981284306163, + "tokens_seen": 2301755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003077171717171717, + "loss": 2.9074, + "theoretical_loss": 3.3909734550768302, + "tokens_seen": 2301820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076969696969697, + "loss": 2.8557, + "theoretical_loss": 3.3909656261328154, + "tokens_seen": 2301886464 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030767676767676764, + "loss": 2.8131, + "theoretical_loss": 3.3909577974741003, + "tokens_seen": 2301952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076565656565657, + "loss": 2.7825, + "theoretical_loss": 3.390949969100666, + "tokens_seen": 2302017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030763636363636366, + "loss": 2.8032, + "theoretical_loss": 3.390942141012495, + "tokens_seen": 2302083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003076161616161616, + "loss": 2.8209, + "theoretical_loss": 3.3909343132095673, + "tokens_seen": 2302148608 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1297386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2210633754730225, + "objective/train/theoretical_loss": 3.3909264856918657, + "objective/train/tokens_used": 661073376, + "theoretical_loss": 3.3909264856918657, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075959595959596, + "loss": 3.0899, + "theoretical_loss": 3.3909264856918657, + "tokens_seen": 2302214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075757575757576, + "loss": 2.9464, + "theoretical_loss": 3.3909186584593716, + "tokens_seen": 2302279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075555555555556, + "loss": 2.8452, + "theoretical_loss": 3.3909108315120653, + "tokens_seen": 2302345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030753535353535356, + "loss": 2.7917, + "theoretical_loss": 3.3909030048499296, + "tokens_seen": 2302410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003075151515151515, + "loss": 2.9441, + "theoretical_loss": 3.3908951784729453, + "tokens_seen": 2302476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030749494949494947, + "loss": 2.785, + "theoretical_loss": 3.3908873523810943, + "tokens_seen": 2302541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074747474747475, + "loss": 2.9039, + "theoretical_loss": 3.3908795265743574, + "tokens_seen": 2302607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074545454545455, + "loss": 2.8448, + "theoretical_loss": 3.390871701052717, + "tokens_seen": 2302672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030743434343434345, + "loss": 2.8442, + "theoretical_loss": 3.3908638758161542, + "tokens_seen": 2302738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003074141414141414, + "loss": 2.8688, + "theoretical_loss": 3.3908560508646497, + "tokens_seen": 2302803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073939393939394, + "loss": 2.9342, + "theoretical_loss": 3.3908482261981865, + "tokens_seen": 2302869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030737373737373737, + "loss": 2.7671, + "theoretical_loss": 3.390840401816745, + "tokens_seen": 2302935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073535353535354, + "loss": 2.73, + "theoretical_loss": 3.3908325777203077, + "tokens_seen": 2303000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030733333333333334, + "loss": 2.8007, + "theoretical_loss": 3.3908247539088547, + "tokens_seen": 2303066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003073131313131313, + "loss": 2.7766, + "theoretical_loss": 3.3908169303823685, + "tokens_seen": 2303131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072929292929293, + "loss": 2.7923, + "theoretical_loss": 3.390809107140831, + "tokens_seen": 2303197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030727272727272727, + "loss": 2.625, + "theoretical_loss": 3.3908012841842226, + "tokens_seen": 2303262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003072525252525253, + "loss": 2.8255, + "theoretical_loss": 3.3907934615125255, + "tokens_seen": 2303328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030723232323232323, + "loss": 2.768, + "theoretical_loss": 3.390785639125721, + "tokens_seen": 2303393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030721212121212124, + "loss": 2.9097, + "theoretical_loss": 3.3907778170237908, + "tokens_seen": 2303459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071919191919192, + "loss": 2.8545, + "theoretical_loss": 3.390769995206716, + "tokens_seen": 2303524864 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030717171717171716, + "loss": 2.7821, + "theoretical_loss": 3.390762173674479, + "tokens_seen": 2303590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030715151515151517, + "loss": 3.0028, + "theoretical_loss": 3.3907543524270602, + "tokens_seen": 2303655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003071313131313131, + "loss": 2.8898, + "theoretical_loss": 3.3907465314644414, + "tokens_seen": 2303721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030711111111111114, + "loss": 2.7646, + "theoretical_loss": 3.3907387107866054, + "tokens_seen": 2303787008 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1298736, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9739785194396973, + "objective/train/theoretical_loss": 3.390730890393532, + "objective/train/tokens_used": 662711776, + "theoretical_loss": 3.390730890393532, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070909090909091, + "loss": 2.7189, + "theoretical_loss": 3.390730890393532, + "tokens_seen": 2303852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030707070707070705, + "loss": 2.8926, + "theoretical_loss": 3.3907230702852034, + "tokens_seen": 2303918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030705050505050506, + "loss": 2.9292, + "theoretical_loss": 3.3907152504616014, + "tokens_seen": 2303983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003070303030303031, + "loss": 3.0425, + "theoretical_loss": 3.390707430922707, + "tokens_seen": 2304049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030701010101010103, + "loss": 2.891, + "theoretical_loss": 3.390699611668503, + "tokens_seen": 2304114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306989898989899, + "loss": 2.7178, + "theoretical_loss": 3.3906917926989695, + "tokens_seen": 2304180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030696969696969694, + "loss": 2.8538, + "theoretical_loss": 3.390683974014088, + "tokens_seen": 2304245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030694949494949495, + "loss": 2.8792, + "theoretical_loss": 3.3906761556138414, + "tokens_seen": 2304311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030692929292929297, + "loss": 2.8386, + "theoretical_loss": 3.39066833749821, + "tokens_seen": 2304376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003069090909090909, + "loss": 2.695, + "theoretical_loss": 3.390660519667176, + "tokens_seen": 2304442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068888888888889, + "loss": 3.0614, + "theoretical_loss": 3.3906527021207205, + "tokens_seen": 2304507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030686868686868684, + "loss": 2.7019, + "theoretical_loss": 3.3906448848588253, + "tokens_seen": 2304573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068484848484849, + "loss": 2.9211, + "theoretical_loss": 3.390637067881472, + "tokens_seen": 2304638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030682828282828286, + "loss": 2.9641, + "theoretical_loss": 3.390629251188642, + "tokens_seen": 2304704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003068080808080808, + "loss": 2.8631, + "theoretical_loss": 3.390621434780317, + "tokens_seen": 2304770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030678787878787877, + "loss": 2.7801, + "theoretical_loss": 3.3906136186564786, + "tokens_seen": 2304835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030676767676767673, + "loss": 2.7429, + "theoretical_loss": 3.3906058028171078, + "tokens_seen": 2304901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067474747474748, + "loss": 2.8184, + "theoretical_loss": 3.390597987262187, + "tokens_seen": 2304966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030672727272727275, + "loss": 2.8857, + "theoretical_loss": 3.3905901719916973, + "tokens_seen": 2305032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003067070707070707, + "loss": 2.9642, + "theoretical_loss": 3.3905823570056204, + "tokens_seen": 2305097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030668686868686866, + "loss": 2.7816, + "theoretical_loss": 3.3905745423039377, + "tokens_seen": 2305163264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066666666666667, + "loss": 2.8158, + "theoretical_loss": 3.3905667278866307, + "tokens_seen": 2305228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066464646464647, + "loss": 2.6304, + "theoretical_loss": 3.390558913753681, + "tokens_seen": 2305294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030662626262626264, + "loss": 2.8481, + "theoretical_loss": 3.3905510999050708, + "tokens_seen": 2305359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003066060606060606, + "loss": 2.8924, + "theoretical_loss": 3.3905432863407805, + "tokens_seen": 2305425408 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1299212, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3015689849853516, + "objective/train/theoretical_loss": 3.390535473060793, + "objective/train/tokens_used": 664350176, + "theoretical_loss": 3.390535473060793, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030658585858585856, + "loss": 2.8909, + "theoretical_loss": 3.390535473060793, + "tokens_seen": 2305490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030656565656565657, + "loss": 2.8251, + "theoretical_loss": 3.390527660065089, + "tokens_seen": 2305556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065454545454546, + "loss": 2.4053, + "theoretical_loss": 3.3905198473536498, + "tokens_seen": 2305622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030652525252525254, + "loss": 2.8105, + "theoretical_loss": 3.3905120349264584, + "tokens_seen": 2305687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003065050505050505, + "loss": 2.8513, + "theoretical_loss": 3.3905042227834947, + "tokens_seen": 2305753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064848484848485, + "loss": 2.7692, + "theoretical_loss": 3.390496410924741, + "tokens_seen": 2305818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030646464646464646, + "loss": 2.8283, + "theoretical_loss": 3.390488599350179, + "tokens_seen": 2305884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030644444444444447, + "loss": 2.9367, + "theoretical_loss": 3.3904807880597905, + "tokens_seen": 2305949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030642424242424243, + "loss": 2.9268, + "theoretical_loss": 3.390472977053556, + "tokens_seen": 2306015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003064040404040404, + "loss": 2.9143, + "theoretical_loss": 3.3904651663314587, + "tokens_seen": 2306080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063838383838384, + "loss": 2.7954, + "theoretical_loss": 3.390457355893479, + "tokens_seen": 2306146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030636363636363635, + "loss": 2.5713, + "theoretical_loss": 3.390449545739599, + "tokens_seen": 2306211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030634343434343437, + "loss": 3.0041, + "theoretical_loss": 3.3904417358698002, + "tokens_seen": 2306277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063232323232323, + "loss": 2.9539, + "theoretical_loss": 3.390433926284064, + "tokens_seen": 2306342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003063030303030303, + "loss": 2.8761, + "theoretical_loss": 3.390426116982372, + "tokens_seen": 2306408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062828282828283, + "loss": 2.8442, + "theoretical_loss": 3.390418307964706, + "tokens_seen": 2306473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030626262626262625, + "loss": 2.6565, + "theoretical_loss": 3.3904104992310478, + "tokens_seen": 2306539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030624242424242426, + "loss": 2.8084, + "theoretical_loss": 3.3904026907813787, + "tokens_seen": 2306605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062222222222222, + "loss": 2.7167, + "theoretical_loss": 3.39039488261568, + "tokens_seen": 2306670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003062020202020202, + "loss": 2.7921, + "theoretical_loss": 3.3903870747339333, + "tokens_seen": 2306736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061818181818182, + "loss": 2.7886, + "theoretical_loss": 3.3903792671361215, + "tokens_seen": 2306801664 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030616161616161614, + "loss": 2.975, + "theoretical_loss": 3.3903714598222243, + "tokens_seen": 2306867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030614141414141415, + "loss": 2.9884, + "theoretical_loss": 3.390363652792225, + "tokens_seen": 2306932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061212121212121, + "loss": 2.8138, + "theoretical_loss": 3.390355846046104, + "tokens_seen": 2306998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003061010101010101, + "loss": 2.7739, + "theoretical_loss": 3.3903480395838437, + "tokens_seen": 2307063808 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1299979, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7158727645874023, + "objective/train/theoretical_loss": 3.3903402334054253, + "objective/train/tokens_used": 665988576, + "theoretical_loss": 3.3903402334054253, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003060808080808081, + "loss": 2.7387, + "theoretical_loss": 3.3903402334054253, + "tokens_seen": 2307129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030606060606060603, + "loss": 2.8506, + "theoretical_loss": 3.39033242751083, + "tokens_seen": 2307194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030604040404040404, + "loss": 3.0865, + "theoretical_loss": 3.390324621900041, + "tokens_seen": 2307260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030602020202020205, + "loss": 2.8248, + "theoretical_loss": 3.390316816573038, + "tokens_seen": 2307325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.000306, + "loss": 2.7525, + "theoretical_loss": 3.390309011529804, + "tokens_seen": 2307391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030597979797979797, + "loss": 2.8518, + "theoretical_loss": 3.39030120677032, + "tokens_seen": 2307457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.000305959595959596, + "loss": 2.9155, + "theoretical_loss": 3.390293402294567, + "tokens_seen": 2307522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030593939393939394, + "loss": 2.9868, + "theoretical_loss": 3.3902855981025284, + "tokens_seen": 2307588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030591919191919195, + "loss": 2.9243, + "theoretical_loss": 3.3902777941941844, + "tokens_seen": 2307653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058989898989899, + "loss": 2.6363, + "theoretical_loss": 3.390269990569517, + "tokens_seen": 2307719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030587878787878786, + "loss": 2.7164, + "theoretical_loss": 3.3902621872285077, + "tokens_seen": 2307784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030585858585858587, + "loss": 2.931, + "theoretical_loss": 3.3902543841711386, + "tokens_seen": 2307850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003058383838383839, + "loss": 2.7162, + "theoretical_loss": 3.3902465813973905, + "tokens_seen": 2307915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030581818181818184, + "loss": 2.914, + "theoretical_loss": 3.390238778907246, + "tokens_seen": 2307981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057979797979798, + "loss": 2.9068, + "theoretical_loss": 3.3902309767006864, + "tokens_seen": 2308046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030577777777777775, + "loss": 2.7761, + "theoretical_loss": 3.390223174777693, + "tokens_seen": 2308112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030575757575757576, + "loss": 2.7264, + "theoretical_loss": 3.390215373138248, + "tokens_seen": 2308177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003057373737373738, + "loss": 2.81, + "theoretical_loss": 3.390207571782332, + "tokens_seen": 2308243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030571717171717173, + "loss": 2.8692, + "theoretical_loss": 3.390199770709928, + "tokens_seen": 2308308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056969696969697, + "loss": 2.742, + "theoretical_loss": 3.390191969921017, + "tokens_seen": 2308374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030567676767676765, + "loss": 2.8953, + "theoretical_loss": 3.3901841694155803, + "tokens_seen": 2308440064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056565656565657, + "loss": 2.707, + "theoretical_loss": 3.3901763691936, + "tokens_seen": 2308505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.00030563636363636367, + "loss": 2.7813, + "theoretical_loss": 3.390168569255058, + "tokens_seen": 2308571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003056161616161616, + "loss": 2.8963, + "theoretical_loss": 3.3901607695999356, + "tokens_seen": 2308636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0003055959595959596, + "loss": 2.8651, + "theoretical_loss": 3.390152970228214, + "tokens_seen": 2308702208 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 1301383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7038803100585938, + "objective/train/theoretical_loss": 3.390145171139876, + "objective/train/tokens_used": 667626976, + "theoretical_loss": 3.390145171139876, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030557575757575754, + "loss": 2.6377, + "theoretical_loss": 3.390145171139876, + "tokens_seen": 2308767744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055555555555556, + "loss": 2.8587, + "theoretical_loss": 3.390137372334902, + "tokens_seen": 2308833280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030553535353535356, + "loss": 2.669, + "theoretical_loss": 3.3901295738132746, + "tokens_seen": 2308898816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003055151515151515, + "loss": 2.6227, + "theoretical_loss": 3.390121775574975, + "tokens_seen": 2308964352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054949494949495, + "loss": 2.8173, + "theoretical_loss": 3.3901139776199853, + "tokens_seen": 2309029888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054747474747475, + "loss": 2.8596, + "theoretical_loss": 3.390106179948287, + "tokens_seen": 2309095424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054545454545455, + "loss": 2.7819, + "theoretical_loss": 3.390098382559861, + "tokens_seen": 2309160960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030543434343434345, + "loss": 2.8372, + "theoretical_loss": 3.39009058545469, + "tokens_seen": 2309226496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003054141414141414, + "loss": 2.8494, + "theoretical_loss": 3.390082788632755, + "tokens_seen": 2309292032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030539393939393937, + "loss": 2.81, + "theoretical_loss": 3.3900749920940383, + "tokens_seen": 2309357568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053737373737374, + "loss": 2.6886, + "theoretical_loss": 3.3900671958385207, + "tokens_seen": 2309423104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053535353535354, + "loss": 2.6857, + "theoretical_loss": 3.390059399866185, + "tokens_seen": 2309488640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030533333333333335, + "loss": 2.8404, + "theoretical_loss": 3.390051604177012, + "tokens_seen": 2309554176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003053131313131313, + "loss": 2.6981, + "theoretical_loss": 3.3900438087709834, + "tokens_seen": 2309619712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052929292929293, + "loss": 2.7511, + "theoretical_loss": 3.3900360136480816, + "tokens_seen": 2309685248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030527272727272727, + "loss": 2.8869, + "theoretical_loss": 3.3900282188082875, + "tokens_seen": 2309750784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052525252525253, + "loss": 2.7472, + "theoretical_loss": 3.3900204242515835, + "tokens_seen": 2309816320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030523232323232324, + "loss": 2.8297, + "theoretical_loss": 3.3900126299779503, + "tokens_seen": 2309881856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003052121212121212, + "loss": 2.9864, + "theoretical_loss": 3.3900048359873707, + "tokens_seen": 2309947392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051919191919192, + "loss": 2.6383, + "theoretical_loss": 3.3899970422798256, + "tokens_seen": 2310012928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030517171717171716, + "loss": 2.7804, + "theoretical_loss": 3.389989248855297, + "tokens_seen": 2310078464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003051515151515152, + "loss": 2.974, + "theoretical_loss": 3.389981455713767, + "tokens_seen": 2310144000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030513131313131313, + "loss": 2.974, + "theoretical_loss": 3.3899736628552164, + "tokens_seen": 2310209536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030511111111111114, + "loss": 2.8939, + "theoretical_loss": 3.389965870279627, + "tokens_seen": 2310275072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003050909090909091, + "loss": 2.9533, + "theoretical_loss": 3.389958077986982, + "tokens_seen": 2310340608 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1301708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898665189743042, + "objective/train/theoretical_loss": 3.3899502859772612, + "objective/train/tokens_used": 669265376, + "theoretical_loss": 3.3899502859772612, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030507070707070706, + "loss": 2.7051, + "theoretical_loss": 3.3899502859772612, + "tokens_seen": 2310406144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030505050505050507, + "loss": 2.9797, + "theoretical_loss": 3.389942494250447, + "tokens_seen": 2310471680 + }, + { + "epoch": 0.4, + "learning_rate": 0.000305030303030303, + "loss": 2.8345, + "theoretical_loss": 3.389934702806521, + "tokens_seen": 2310537216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030501010101010104, + "loss": 2.6338, + "theoretical_loss": 3.389926911645466, + "tokens_seen": 2310602752 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304989898989899, + "loss": 2.9239, + "theoretical_loss": 3.389919120767262, + "tokens_seen": 2310668288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030496969696969695, + "loss": 2.7818, + "theoretical_loss": 3.3899113301718913, + "tokens_seen": 2310733824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030494949494949496, + "loss": 2.7741, + "theoretical_loss": 3.3899035398593362, + "tokens_seen": 2310799360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030492929292929297, + "loss": 2.4895, + "theoretical_loss": 3.389895749829578, + "tokens_seen": 2310864896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030490909090909093, + "loss": 2.8555, + "theoretical_loss": 3.389887960082598, + "tokens_seen": 2310930432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048888888888889, + "loss": 2.7798, + "theoretical_loss": 3.3898801706183788, + "tokens_seen": 2310995968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030486868686868684, + "loss": 2.8986, + "theoretical_loss": 3.3898723814369016, + "tokens_seen": 2311061504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030484848484848485, + "loss": 2.669, + "theoretical_loss": 3.389864592538148, + "tokens_seen": 2311127040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030482828282828286, + "loss": 2.8341, + "theoretical_loss": 3.3898568039221004, + "tokens_seen": 2311192576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003048080808080808, + "loss": 2.9821, + "theoretical_loss": 3.3898490155887395, + "tokens_seen": 2311258112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047878787878788, + "loss": 2.7298, + "theoretical_loss": 3.3898412275380476, + "tokens_seen": 2311323648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030476767676767673, + "loss": 2.7452, + "theoretical_loss": 3.3898334397700065, + "tokens_seen": 2311389184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047474747474748, + "loss": 2.9943, + "theoretical_loss": 3.389825652284598, + "tokens_seen": 2311454720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030472727272727276, + "loss": 2.8758, + "theoretical_loss": 3.3898178650818034, + "tokens_seen": 2311520256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003047070707070707, + "loss": 2.5298, + "theoretical_loss": 3.389810078161605, + "tokens_seen": 2311585792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030468686868686867, + "loss": 2.5685, + "theoretical_loss": 3.3898022915239836, + "tokens_seen": 2311651328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030466666666666663, + "loss": 2.9617, + "theoretical_loss": 3.389794505168922, + "tokens_seen": 2311716864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046464646464647, + "loss": 2.8077, + "theoretical_loss": 3.389786719096401, + "tokens_seen": 2311782400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030462626262626265, + "loss": 2.7846, + "theoretical_loss": 3.389778933306403, + "tokens_seen": 2311847936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003046060606060606, + "loss": 2.7815, + "theoretical_loss": 3.3897711477989096, + "tokens_seen": 2311913472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030458585858585856, + "loss": 2.6722, + "theoretical_loss": 3.3897633625739028, + "tokens_seen": 2311979008 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1302992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.039232015609741, + "objective/train/theoretical_loss": 3.3897555776313637, + "objective/train/tokens_used": 670903776, + "theoretical_loss": 3.3897555776313637, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045656565656565, + "loss": 2.8617, + "theoretical_loss": 3.3897555776313637, + "tokens_seen": 2312044544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045454545454546, + "loss": 2.8611, + "theoretical_loss": 3.3897477929712743, + "tokens_seen": 2312110080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030452525252525254, + "loss": 2.8933, + "theoretical_loss": 3.3897400085936167, + "tokens_seen": 2312175616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003045050505050505, + "loss": 2.8383, + "theoretical_loss": 3.3897322244983723, + "tokens_seen": 2312241152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030448484848484846, + "loss": 2.9009, + "theoretical_loss": 3.389724440685523, + "tokens_seen": 2312306688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030446464646464647, + "loss": 2.7073, + "theoretical_loss": 3.3897166571550503, + "tokens_seen": 2312372224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044444444444445, + "loss": 2.9082, + "theoretical_loss": 3.389708873906936, + "tokens_seen": 2312437760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030442424242424243, + "loss": 2.7661, + "theoretical_loss": 3.3897010909411622, + "tokens_seen": 2312503296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003044040404040404, + "loss": 2.866, + "theoretical_loss": 3.3896933082577103, + "tokens_seen": 2312568832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030438383838383835, + "loss": 2.6364, + "theoretical_loss": 3.3896855258565624, + "tokens_seen": 2312634368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030436363636363636, + "loss": 2.6627, + "theoretical_loss": 3.3896777437376997, + "tokens_seen": 2312699904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030434343434343437, + "loss": 2.6868, + "theoretical_loss": 3.3896699619011046, + "tokens_seen": 2312765440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030432323232323233, + "loss": 2.6931, + "theoretical_loss": 3.389662180346759, + "tokens_seen": 2312830976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003043030303030303, + "loss": 2.9166, + "theoretical_loss": 3.3896543990746437, + "tokens_seen": 2312896512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042828282828283, + "loss": 2.663, + "theoretical_loss": 3.3896466180847407, + "tokens_seen": 2312962048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042626262626263, + "loss": 2.704, + "theoretical_loss": 3.3896388373770328, + "tokens_seen": 2313027584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030424242424242426, + "loss": 2.9085, + "theoretical_loss": 3.389631056951501, + "tokens_seen": 2313093120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042222222222222, + "loss": 2.8758, + "theoretical_loss": 3.389623276808127, + "tokens_seen": 2313158656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003042020202020202, + "loss": 2.6271, + "theoretical_loss": 3.389615496946892, + "tokens_seen": 2313224192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041818181818182, + "loss": 2.6938, + "theoretical_loss": 3.389607717367779, + "tokens_seen": 2313289728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041616161616162, + "loss": 2.68, + "theoretical_loss": 3.3895999380707695, + "tokens_seen": 2313355264 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030414141414141416, + "loss": 2.8948, + "theoretical_loss": 3.389592159055845, + "tokens_seen": 2313420800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041212121212121, + "loss": 2.6716, + "theoretical_loss": 3.389584380322987, + "tokens_seen": 2313486336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003041010101010101, + "loss": 2.6113, + "theoretical_loss": 3.389576601872178, + "tokens_seen": 2313551872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040808080808081, + "loss": 2.8829, + "theoretical_loss": 3.389568823703399, + "tokens_seen": 2313617408 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1303654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.849209785461426, + "objective/train/theoretical_loss": 3.3895610458166323, + "objective/train/tokens_used": 672542176, + "theoretical_loss": 3.3895610458166323, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003040606060606061, + "loss": 2.9267, + "theoretical_loss": 3.3895610458166323, + "tokens_seen": 2313682944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030404040404040405, + "loss": 2.5, + "theoretical_loss": 3.3895532682118596, + "tokens_seen": 2313748480 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304020202020202, + "loss": 2.8516, + "theoretical_loss": 3.3895454908890628, + "tokens_seen": 2313814016 + }, + { + "epoch": 0.4, + "learning_rate": 0.000304, + "loss": 2.8007, + "theoretical_loss": 3.389537713848223, + "tokens_seen": 2313879552 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303979797979798, + "loss": 2.8909, + "theoretical_loss": 3.389529937089323, + "tokens_seen": 2313945088 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303959595959596, + "loss": 2.8252, + "theoretical_loss": 3.3895221606123442, + "tokens_seen": 2314010624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030393939393939394, + "loss": 2.6123, + "theoretical_loss": 3.389514384417268, + "tokens_seen": 2314076160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030391919191919195, + "loss": 2.9422, + "theoretical_loss": 3.389506608504077, + "tokens_seen": 2314141696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038989898989899, + "loss": 2.7772, + "theoretical_loss": 3.3894988328727518, + "tokens_seen": 2314207232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030387878787878787, + "loss": 2.9047, + "theoretical_loss": 3.3894910575232755, + "tokens_seen": 2314272768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003038585858585859, + "loss": 2.6344, + "theoretical_loss": 3.389483282455629, + "tokens_seen": 2314338304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030383838383838383, + "loss": 2.8524, + "theoretical_loss": 3.389475507669794, + "tokens_seen": 2314403840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030381818181818185, + "loss": 2.8169, + "theoretical_loss": 3.3894677331657537, + "tokens_seen": 2314469376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037979797979798, + "loss": 2.9629, + "theoretical_loss": 3.3894599589434886, + "tokens_seen": 2314534912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030377777777777776, + "loss": 2.6756, + "theoretical_loss": 3.3894521850029804, + "tokens_seen": 2314600448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030375757575757577, + "loss": 2.9192, + "theoretical_loss": 3.3894444113442117, + "tokens_seen": 2314665984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003037373737373738, + "loss": 2.8393, + "theoretical_loss": 3.389436637967164, + "tokens_seen": 2314731520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030371717171717174, + "loss": 2.7371, + "theoretical_loss": 3.389428864871819, + "tokens_seen": 2314797056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036969696969697, + "loss": 3.0214, + "theoretical_loss": 3.389421092058159, + "tokens_seen": 2314862592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030367676767676765, + "loss": 2.9021, + "theoretical_loss": 3.389413319526165, + "tokens_seen": 2314928128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030365656565656566, + "loss": 2.7711, + "theoretical_loss": 3.389405547275819, + "tokens_seen": 2314993664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003036363636363637, + "loss": 2.7803, + "theoretical_loss": 3.3893977753071036, + "tokens_seen": 2315059200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030361616161616163, + "loss": 2.833, + "theoretical_loss": 3.3893900036199995, + "tokens_seen": 2315124736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035959595959596, + "loss": 2.8857, + "theoretical_loss": 3.3893822322144898, + "tokens_seen": 2315190272 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030357575757575754, + "loss": 2.7454, + "theoretical_loss": 3.389374461090555, + "tokens_seen": 2315255808 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1304892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9643783569335938, + "objective/train/theoretical_loss": 3.3893666902481776, + "objective/train/tokens_used": 674180576, + "theoretical_loss": 3.3893666902481776, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035555555555556, + "loss": 2.6829, + "theoretical_loss": 3.3893666902481776, + "tokens_seen": 2315321344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030353535353535357, + "loss": 2.9287, + "theoretical_loss": 3.38935891968734, + "tokens_seen": 2315386880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003035151515151515, + "loss": 2.7405, + "theoretical_loss": 3.389351149408023, + "tokens_seen": 2315452416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034949494949495, + "loss": 2.6223, + "theoretical_loss": 3.389343379410209, + "tokens_seen": 2315517952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030347474747474744, + "loss": 2.9421, + "theoretical_loss": 3.3893356096938794, + "tokens_seen": 2315583488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034545454545455, + "loss": 2.7328, + "theoretical_loss": 3.389327840259017, + "tokens_seen": 2315649024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030343434343434346, + "loss": 2.7606, + "theoretical_loss": 3.389320071105602, + "tokens_seen": 2315714560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003034141414141414, + "loss": 2.6953, + "theoretical_loss": 3.3893123022336176, + "tokens_seen": 2315780096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033939393939394, + "loss": 2.6156, + "theoretical_loss": 3.3893045336430454, + "tokens_seen": 2315845632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033737373737374, + "loss": 2.8277, + "theoretical_loss": 3.389296765333867, + "tokens_seen": 2315911168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033535353535354, + "loss": 2.8583, + "theoretical_loss": 3.3892889973060645, + "tokens_seen": 2315976704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030333333333333335, + "loss": 2.5963, + "theoretical_loss": 3.3892812295596193, + "tokens_seen": 2316042240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003033131313131313, + "loss": 2.6594, + "theoretical_loss": 3.3892734620945135, + "tokens_seen": 2316107776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030329292929292927, + "loss": 2.8189, + "theoretical_loss": 3.3892656949107294, + "tokens_seen": 2316173312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032727272727273, + "loss": 2.9855, + "theoretical_loss": 3.3892579280082478, + "tokens_seen": 2316238848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032525252525253, + "loss": 2.6452, + "theoretical_loss": 3.389250161387052, + "tokens_seen": 2316304384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030323232323232324, + "loss": 2.8668, + "theoretical_loss": 3.3892423950471224, + "tokens_seen": 2316369920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003032121212121212, + "loss": 2.7943, + "theoretical_loss": 3.389234628988442, + "tokens_seen": 2316435456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031919191919192, + "loss": 2.8283, + "theoretical_loss": 3.3892268632109914, + "tokens_seen": 2316500992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030317171717171717, + "loss": 3.0157, + "theoretical_loss": 3.389219097714754, + "tokens_seen": 2316566528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031515151515152, + "loss": 2.7523, + "theoretical_loss": 3.38921133249971, + "tokens_seen": 2316632064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030313131313131314, + "loss": 2.4398, + "theoretical_loss": 3.389203567565843, + "tokens_seen": 2316697600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003031111111111111, + "loss": 2.6508, + "theoretical_loss": 3.3891958029131333, + "tokens_seen": 2316763136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030909090909091, + "loss": 2.6536, + "theoretical_loss": 3.389188038541564, + "tokens_seen": 2316828672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030307070707070706, + "loss": 2.8753, + "theoretical_loss": 3.3891802744511166, + "tokens_seen": 2316894208 + }, + { + "debugging/Self-BLEU-5": 0.15659319952366813, + "debugging/distinct-1-grams": 0.7962866038742501, + "debugging/distinct-2-grams": 0.9550000000000001, + "debugging/entropy-1-grams": 4.929609510498728, + "debugging/entropy-2-grams": 5.4030103897236375, + "debugging/length": 474.3333333333333, + "debugging/num_segments": 3, + "epoch": 0.4, + "objective/train/docs_used": 1305637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6540987491607666, + "objective/train/theoretical_loss": 3.3891725106417723, + "objective/train/tokens_used": 675818976, + "theoretical_loss": 3.3891725106417723, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003030505050505051, + "loss": 2.8128, + "theoretical_loss": 3.3891725106417723, + "tokens_seen": 2316959744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030303030303030303, + "loss": 2.9295, + "theoretical_loss": 3.3891647471135133, + "tokens_seen": 2317025280 + }, + { + "epoch": 0.4, + "learning_rate": 0.000303010101010101, + "loss": 2.6501, + "theoretical_loss": 3.3891569838663225, + "tokens_seen": 2317090816 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302989898989899, + "loss": 2.9299, + "theoretical_loss": 3.38914922090018, + "tokens_seen": 2317156352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030296969696969695, + "loss": 2.9275, + "theoretical_loss": 3.389141458215069, + "tokens_seen": 2317221888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030294949494949497, + "loss": 2.6176, + "theoretical_loss": 3.3891336958109712, + "tokens_seen": 2317287424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003029292929292929, + "loss": 2.7396, + "theoretical_loss": 3.3891259336878683, + "tokens_seen": 2317352960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030290909090909093, + "loss": 2.6516, + "theoretical_loss": 3.389118171845742, + "tokens_seen": 2317418496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028888888888889, + "loss": 2.7552, + "theoretical_loss": 3.389110410284574, + "tokens_seen": 2317484032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030286868686868685, + "loss": 2.6525, + "theoretical_loss": 3.3891026490043465, + "tokens_seen": 2317549568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030284848484848486, + "loss": 2.6291, + "theoretical_loss": 3.389094888005042, + "tokens_seen": 2317615104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028282828282828, + "loss": 2.7194, + "theoretical_loss": 3.3890871272866416, + "tokens_seen": 2317680640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003028080808080808, + "loss": 2.6636, + "theoretical_loss": 3.389079366849127, + "tokens_seen": 2317746176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027878787878788, + "loss": 2.6198, + "theoretical_loss": 3.3890716066924806, + "tokens_seen": 2317811712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030276767676767674, + "loss": 2.6016, + "theoretical_loss": 3.3890638468166845, + "tokens_seen": 2317877248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030274747474747475, + "loss": 2.8698, + "theoretical_loss": 3.38905608722172, + "tokens_seen": 2317942784 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030272727272727276, + "loss": 2.8009, + "theoretical_loss": 3.3890483279075694, + "tokens_seen": 2318008320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003027070707070707, + "loss": 2.6509, + "theoretical_loss": 3.3890405688742145, + "tokens_seen": 2318073856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026868686868687, + "loss": 2.6102, + "theoretical_loss": 3.3890328101216367, + "tokens_seen": 2318139392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030266666666666663, + "loss": 2.5764, + "theoretical_loss": 3.389025051649819, + "tokens_seen": 2318204928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030264646464646464, + "loss": 2.6944, + "theoretical_loss": 3.389017293458742, + "tokens_seen": 2318270464 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030262626262626266, + "loss": 2.4496, + "theoretical_loss": 3.389009535548389, + "tokens_seen": 2318336000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003026060606060606, + "loss": 2.9531, + "theoretical_loss": 3.3890017779187405, + "tokens_seen": 2318401536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030258585858585857, + "loss": 2.6086, + "theoretical_loss": 3.3889940205697795, + "tokens_seen": 2318467072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025656565656566, + "loss": 2.7689, + "theoretical_loss": 3.3889862635014874, + "tokens_seen": 2318532608 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1307149, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.711271047592163, + "objective/train/theoretical_loss": 3.3889785067138463, + "objective/train/tokens_used": 677457376, + "theoretical_loss": 3.3889785067138463, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025454545454546, + "loss": 2.8654, + "theoretical_loss": 3.3889785067138463, + "tokens_seen": 2318598144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030252525252525255, + "loss": 2.7619, + "theoretical_loss": 3.3889707502068376, + "tokens_seen": 2318663680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003025050505050505, + "loss": 2.7765, + "theoretical_loss": 3.388962993980444, + "tokens_seen": 2318729216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030248484848484846, + "loss": 2.7251, + "theoretical_loss": 3.3889552380346473, + "tokens_seen": 2318794752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030246464646464647, + "loss": 2.6917, + "theoretical_loss": 3.3889474823694288, + "tokens_seen": 2318860288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024444444444445, + "loss": 2.6016, + "theoretical_loss": 3.3889397269847708, + "tokens_seen": 2318925824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030242424242424244, + "loss": 2.8136, + "theoretical_loss": 3.3889319718806554, + "tokens_seen": 2318991360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003024040404040404, + "loss": 2.6818, + "theoretical_loss": 3.388924217057064, + "tokens_seen": 2319056896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030238383838383835, + "loss": 2.8821, + "theoretical_loss": 3.388916462513979, + "tokens_seen": 2319122432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023636363636364, + "loss": 2.8411, + "theoretical_loss": 3.3889087082513827, + "tokens_seen": 2319187968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023434343434344, + "loss": 2.8286, + "theoretical_loss": 3.388900954269256, + "tokens_seen": 2319253504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030232323232323233, + "loss": 2.6994, + "theoretical_loss": 3.388893200567581, + "tokens_seen": 2319319040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003023030303030303, + "loss": 2.9918, + "theoretical_loss": 3.3888854471463405, + "tokens_seen": 2319384576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030228282828282825, + "loss": 2.6526, + "theoretical_loss": 3.388877694005516, + "tokens_seen": 2319450112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022626262626263, + "loss": 3.1073, + "theoretical_loss": 3.388869941145089, + "tokens_seen": 2319515648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030224242424242427, + "loss": 2.7592, + "theoretical_loss": 3.3888621885650423, + "tokens_seen": 2319581184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022222222222222, + "loss": 2.4824, + "theoretical_loss": 3.388854436265357, + "tokens_seen": 2319646720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003022020202020202, + "loss": 2.8191, + "theoretical_loss": 3.3888466842460154, + "tokens_seen": 2319712256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021818181818182, + "loss": 2.8071, + "theoretical_loss": 3.3888389325069994, + "tokens_seen": 2319777792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021616161616162, + "loss": 2.5834, + "theoretical_loss": 3.3888311810482907, + "tokens_seen": 2319843328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030214141414141416, + "loss": 2.6414, + "theoretical_loss": 3.388823429869872, + "tokens_seen": 2319908864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021212121212121, + "loss": 2.6655, + "theoretical_loss": 3.3888156789717243, + "tokens_seen": 2319974400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003021010101010101, + "loss": 2.7404, + "theoretical_loss": 3.38880792835383, + "tokens_seen": 2320039936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020808080808081, + "loss": 2.9775, + "theoretical_loss": 3.3888001780161714, + "tokens_seen": 2320105472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003020606060606061, + "loss": 2.736, + "theoretical_loss": 3.3887924279587303, + "tokens_seen": 2320171008 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1307847, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6207799911499023, + "objective/train/theoretical_loss": 3.388784678181488, + "objective/train/tokens_used": 679095776, + "theoretical_loss": 3.388784678181488, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030204040404040405, + "loss": 2.6458, + "theoretical_loss": 3.388784678181488, + "tokens_seen": 2320236544 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302020202020202, + "loss": 2.8554, + "theoretical_loss": 3.388776928684427, + "tokens_seen": 2320302080 + }, + { + "epoch": 0.4, + "learning_rate": 0.000302, + "loss": 2.9058, + "theoretical_loss": 3.388769179467529, + "tokens_seen": 2320367616 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301979797979798, + "loss": 2.7256, + "theoretical_loss": 3.3887614305307765, + "tokens_seen": 2320433152 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301959595959596, + "loss": 2.5883, + "theoretical_loss": 3.388753681874151, + "tokens_seen": 2320498688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030193939393939395, + "loss": 2.5694, + "theoretical_loss": 3.3887459334976344, + "tokens_seen": 2320564224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003019191919191919, + "loss": 2.707, + "theoretical_loss": 3.3887381854012086, + "tokens_seen": 2320629760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018989898989899, + "loss": 2.8458, + "theoretical_loss": 3.388730437584856, + "tokens_seen": 2320695296 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030187878787878787, + "loss": 2.7834, + "theoretical_loss": 3.3887226900485583, + "tokens_seen": 2320760832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003018585858585859, + "loss": 2.7129, + "theoretical_loss": 3.388714942792298, + "tokens_seen": 2320826368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030183838383838384, + "loss": 2.8178, + "theoretical_loss": 3.388707195816056, + "tokens_seen": 2320891904 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030181818181818185, + "loss": 2.6196, + "theoretical_loss": 3.388699449119815, + "tokens_seen": 2320957440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017979797979798, + "loss": 2.6758, + "theoretical_loss": 3.3886917027035572, + "tokens_seen": 2321022976 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030177777777777776, + "loss": 2.6274, + "theoretical_loss": 3.3886839565672635, + "tokens_seen": 2321088512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003017575757575758, + "loss": 2.5062, + "theoretical_loss": 3.3886762107109174, + "tokens_seen": 2321154048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030173737373737373, + "loss": 2.7094, + "theoretical_loss": 3.3886684651344994, + "tokens_seen": 2321219584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030171717171717174, + "loss": 2.7272, + "theoretical_loss": 3.3886607198379926, + "tokens_seen": 2321285120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016969696969697, + "loss": 2.5162, + "theoretical_loss": 3.388652974821378, + "tokens_seen": 2321350656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030167676767676766, + "loss": 2.7287, + "theoretical_loss": 3.3886452300846384, + "tokens_seen": 2321416192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030165656565656567, + "loss": 2.7906, + "theoretical_loss": 3.3886374856277555, + "tokens_seen": 2321481728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003016363636363637, + "loss": 2.6774, + "theoretical_loss": 3.3886297414507114, + "tokens_seen": 2321547264 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030161616161616164, + "loss": 2.7178, + "theoretical_loss": 3.388621997553488, + "tokens_seen": 2321612800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003015959595959596, + "loss": 2.6505, + "theoretical_loss": 3.388614253936067, + "tokens_seen": 2321678336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030157575757575755, + "loss": 2.715, + "theoretical_loss": 3.3886065105984304, + "tokens_seen": 2321743872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030155555555555556, + "loss": 2.5441, + "theoretical_loss": 3.3885987675405613, + "tokens_seen": 2321809408 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1309222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.696748733520508, + "objective/train/theoretical_loss": 3.38859102476244, + "objective/train/tokens_used": 680734176, + "theoretical_loss": 3.38859102476244, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030153535353535357, + "loss": 2.6524, + "theoretical_loss": 3.38859102476244, + "tokens_seen": 2321874944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030151515151515153, + "loss": 2.6731, + "theoretical_loss": 3.38858328226405, + "tokens_seen": 2321940480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014949494949495, + "loss": 2.8973, + "theoretical_loss": 3.3885755400453723, + "tokens_seen": 2322006016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030147474747474744, + "loss": 2.9603, + "theoretical_loss": 3.388567798106389, + "tokens_seen": 2322071552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014545454545455, + "loss": 2.6632, + "theoretical_loss": 3.388560056447083, + "tokens_seen": 2322137088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030143434343434346, + "loss": 2.9522, + "theoretical_loss": 3.3885523150674355, + "tokens_seen": 2322202624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003014141414141414, + "loss": 2.7009, + "theoretical_loss": 3.3885445739674283, + "tokens_seen": 2322268160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013939393939394, + "loss": 2.6803, + "theoretical_loss": 3.388536833147044, + "tokens_seen": 2322333696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030137373737373734, + "loss": 2.7527, + "theoretical_loss": 3.388529092606264, + "tokens_seen": 2322399232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013535353535354, + "loss": 2.7213, + "theoretical_loss": 3.3885213523450712, + "tokens_seen": 2322464768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030133333333333336, + "loss": 2.586, + "theoretical_loss": 3.3885136123634467, + "tokens_seen": 2322530304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003013131313131313, + "loss": 2.8505, + "theoretical_loss": 3.3885058726613733, + "tokens_seen": 2322595840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030129292929292927, + "loss": 2.7472, + "theoretical_loss": 3.388498133238832, + "tokens_seen": 2322661376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030127272727272723, + "loss": 2.7726, + "theoretical_loss": 3.388490394095806, + "tokens_seen": 2322726912 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012525252525253, + "loss": 2.6737, + "theoretical_loss": 3.3884826552322767, + "tokens_seen": 2322792448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030123232323232325, + "loss": 2.6029, + "theoretical_loss": 3.388474916648226, + "tokens_seen": 2322857984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003012121212121212, + "loss": 2.5996, + "theoretical_loss": 3.388467178343636, + "tokens_seen": 2322923520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030119191919191916, + "loss": 2.6177, + "theoretical_loss": 3.3884594403184893, + "tokens_seen": 2322989056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011717171717172, + "loss": 2.7236, + "theoretical_loss": 3.3884517025727674, + "tokens_seen": 2323054592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011515151515152, + "loss": 2.5547, + "theoretical_loss": 3.388443965106452, + "tokens_seen": 2323120128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030113131313131314, + "loss": 2.6664, + "theoretical_loss": 3.3884362279195255, + "tokens_seen": 2323185664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003011111111111111, + "loss": 2.7179, + "theoretical_loss": 3.3884284910119704, + "tokens_seen": 2323251200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030109090909090906, + "loss": 2.6245, + "theoretical_loss": 3.388420754383768, + "tokens_seen": 2323316736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030107070707070707, + "loss": 2.7555, + "theoretical_loss": 3.3884130180349006, + "tokens_seen": 2323382272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003010505050505051, + "loss": 2.5849, + "theoretical_loss": 3.3884052819653503, + "tokens_seen": 2323447808 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1309762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.047351121902466, + "objective/train/theoretical_loss": 3.388397546175099, + "objective/train/tokens_used": 682372576, + "theoretical_loss": 3.388397546175099, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030103030303030304, + "loss": 2.7856, + "theoretical_loss": 3.388397546175099, + "tokens_seen": 2323513344 + }, + { + "epoch": 0.4, + "learning_rate": 0.000301010101010101, + "loss": 2.6778, + "theoretical_loss": 3.388389810664129, + "tokens_seen": 2323578880 + }, + { + "epoch": 0.4, + "learning_rate": 0.000300989898989899, + "loss": 2.5062, + "theoretical_loss": 3.3883820754324216, + "tokens_seen": 2323644416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030096969696969696, + "loss": 2.6832, + "theoretical_loss": 3.3883743404799596, + "tokens_seen": 2323709952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030094949494949497, + "loss": 2.7284, + "theoretical_loss": 3.388366605806725, + "tokens_seen": 2323775488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030092929292929293, + "loss": 2.6938, + "theoretical_loss": 3.3883588714127, + "tokens_seen": 2323841024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003009090909090909, + "loss": 2.5292, + "theoretical_loss": 3.388351137297866, + "tokens_seen": 2323906560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008888888888889, + "loss": 2.7144, + "theoretical_loss": 3.3883434034622057, + "tokens_seen": 2323972096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030086868686868685, + "loss": 2.544, + "theoretical_loss": 3.3883356699057003, + "tokens_seen": 2324037632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030084848484848486, + "loss": 2.7695, + "theoretical_loss": 3.3883279366283325, + "tokens_seen": 2324103168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003008282828282828, + "loss": 2.9699, + "theoretical_loss": 3.3883202036300846, + "tokens_seen": 2324168704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030080808080808083, + "loss": 2.7642, + "theoretical_loss": 3.3883124709109382, + "tokens_seen": 2324234240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007878787878788, + "loss": 2.8907, + "theoretical_loss": 3.3883047384708753, + "tokens_seen": 2324299776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007676767676768, + "loss": 2.5656, + "theoretical_loss": 3.3882970063098785, + "tokens_seen": 2324365312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030074747474747476, + "loss": 2.719, + "theoretical_loss": 3.3882892744279287, + "tokens_seen": 2324430848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007272727272727, + "loss": 2.6871, + "theoretical_loss": 3.3882815428250095, + "tokens_seen": 2324496384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003007070707070707, + "loss": 2.7375, + "theoretical_loss": 3.3882738115011017, + "tokens_seen": 2324561920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006868686868687, + "loss": 2.8953, + "theoretical_loss": 3.388266080456188, + "tokens_seen": 2324627456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006666666666667, + "loss": 2.6949, + "theoretical_loss": 3.388258349690251, + "tokens_seen": 2324692992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030064646464646465, + "loss": 2.7697, + "theoretical_loss": 3.388250619203271, + "tokens_seen": 2324758528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030062626262626266, + "loss": 2.6345, + "theoretical_loss": 3.388242888995232, + "tokens_seen": 2324824064 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003006060606060606, + "loss": 2.5136, + "theoretical_loss": 3.388235159066115, + "tokens_seen": 2324889600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005858585858586, + "loss": 2.6412, + "theoretical_loss": 3.388227429415902, + "tokens_seen": 2324955136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0003005656565656566, + "loss": 2.5225, + "theoretical_loss": 3.3882197000445755, + "tokens_seen": 2325020672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00030054545454545454, + "loss": 2.7943, + "theoretical_loss": 3.388211970952118, + "tokens_seen": 2325086208 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 1311230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0581648349761963, + "objective/train/theoretical_loss": 3.3882042421385106, + "objective/train/tokens_used": 684010976, + "theoretical_loss": 3.3882042421385106, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030052525252525255, + "loss": 2.7853, + "theoretical_loss": 3.3882042421385106, + "tokens_seen": 2325151744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003005050505050505, + "loss": 2.6681, + "theoretical_loss": 3.388196513603736, + "tokens_seen": 2325217280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030048484848484847, + "loss": 2.5532, + "theoretical_loss": 3.388188785347776, + "tokens_seen": 2325282816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004646464646465, + "loss": 2.7304, + "theoretical_loss": 3.3881810573706126, + "tokens_seen": 2325348352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004444444444445, + "loss": 2.644, + "theoretical_loss": 3.3881733296722283, + "tokens_seen": 2325413888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030042424242424245, + "loss": 2.613, + "theoretical_loss": 3.388165602252605, + "tokens_seen": 2325479424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003004040404040404, + "loss": 2.9072, + "theoretical_loss": 3.388157875111725, + "tokens_seen": 2325544960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030038383838383836, + "loss": 2.5849, + "theoretical_loss": 3.38815014824957, + "tokens_seen": 2325610496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030036363636363637, + "loss": 2.8043, + "theoretical_loss": 3.3881424216661222, + "tokens_seen": 2325676032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003434343434344, + "loss": 2.7708, + "theoretical_loss": 3.3881346953613636, + "tokens_seen": 2325741568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030032323232323234, + "loss": 2.8734, + "theoretical_loss": 3.3881269693352762, + "tokens_seen": 2325807104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003003030303030303, + "loss": 2.6612, + "theoretical_loss": 3.388119243587843, + "tokens_seen": 2325872640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030028282828282825, + "loss": 2.7682, + "theoretical_loss": 3.388111518119045, + "tokens_seen": 2325938176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002626262626263, + "loss": 2.7942, + "theoretical_loss": 3.388103792928865, + "tokens_seen": 2326003712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002424242424243, + "loss": 2.6529, + "theoretical_loss": 3.3880960680172842, + "tokens_seen": 2326069248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030022222222222223, + "loss": 2.6498, + "theoretical_loss": 3.388088343384286, + "tokens_seen": 2326134784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003002020202020202, + "loss": 2.8346, + "theoretical_loss": 3.3880806190298514, + "tokens_seen": 2326200320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030018181818181815, + "loss": 2.6225, + "theoretical_loss": 3.388072894953963, + "tokens_seen": 2326265856 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001616161616162, + "loss": 2.47, + "theoretical_loss": 3.388065171156603, + "tokens_seen": 2326331392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030014141414141417, + "loss": 2.8565, + "theoretical_loss": 3.3880574476377534, + "tokens_seen": 2326396928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001212121212121, + "loss": 2.7111, + "theoretical_loss": 3.3880497243973964, + "tokens_seen": 2326462464 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003001010101010101, + "loss": 2.6814, + "theoretical_loss": 3.3880420014355135, + "tokens_seen": 2326528000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000808080808081, + "loss": 2.7583, + "theoretical_loss": 3.388034278752088, + "tokens_seen": 2326593536 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003000606060606061, + "loss": 2.6148, + "theoretical_loss": 3.388026556347101, + "tokens_seen": 2326659072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00030004040404040406, + "loss": 2.5278, + "theoretical_loss": 3.3880188342205346, + "tokens_seen": 2326724608 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1311908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8311409950256348, + "objective/train/theoretical_loss": 3.3880111123723715, + "objective/train/tokens_used": 685649376, + "theoretical_loss": 3.3880111123723715, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.41, + "learning_rate": 0.000300020202020202, + "loss": 2.7918, + "theoretical_loss": 3.3880111123723715, + "tokens_seen": 2326790144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0003, + "loss": 2.9172, + "theoretical_loss": 3.3880033908025937, + "tokens_seen": 2326855680 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299979797979798, + "loss": 2.7372, + "theoretical_loss": 3.387995669511183, + "tokens_seen": 2326921216 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299959595959596, + "loss": 2.6476, + "theoretical_loss": 3.3879879484981217, + "tokens_seen": 2326986752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029993939393939395, + "loss": 2.8169, + "theoretical_loss": 3.3879802277633924, + "tokens_seen": 2327052288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002999191919191919, + "loss": 2.8229, + "theoretical_loss": 3.3879725073069764, + "tokens_seen": 2327117824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029989898989898987, + "loss": 2.6333, + "theoretical_loss": 3.3879647871288565, + "tokens_seen": 2327183360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998787878787879, + "loss": 2.5942, + "theoretical_loss": 3.3879570672290145, + "tokens_seen": 2327248896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998585858585859, + "loss": 2.5397, + "theoretical_loss": 3.387949347607432, + "tokens_seen": 2327314432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029983838383838385, + "loss": 2.476, + "theoretical_loss": 3.3879416282640924, + "tokens_seen": 2327379968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002998181818181818, + "loss": 2.8709, + "theoretical_loss": 3.387933909198977, + "tokens_seen": 2327445504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997979797979798, + "loss": 2.6936, + "theoretical_loss": 3.3879261904120677, + "tokens_seen": 2327511040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029977777777777777, + "loss": 2.6076, + "theoretical_loss": 3.3879184719033475, + "tokens_seen": 2327576576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997575757575758, + "loss": 2.6498, + "theoretical_loss": 3.3879107536727977, + "tokens_seen": 2327642112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029973737373737374, + "loss": 2.7827, + "theoretical_loss": 3.3879030357204014, + "tokens_seen": 2327707648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002997171717171717, + "loss": 2.7339, + "theoretical_loss": 3.3878953180461395, + "tokens_seen": 2327773184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996969696969697, + "loss": 2.5974, + "theoretical_loss": 3.387887600649995, + "tokens_seen": 2327838720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029967676767676766, + "loss": 2.8081, + "theoretical_loss": 3.38787988353195, + "tokens_seen": 2327904256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002996565656565657, + "loss": 2.8524, + "theoretical_loss": 3.387872166691986, + "tokens_seen": 2327969792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029963636363636363, + "loss": 2.7491, + "theoretical_loss": 3.3878644501300865, + "tokens_seen": 2328035328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029961616161616164, + "loss": 2.7044, + "theoretical_loss": 3.3878567338462324, + "tokens_seen": 2328100864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995959595959596, + "loss": 2.6885, + "theoretical_loss": 3.387849017840406, + "tokens_seen": 2328166400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029957575757575756, + "loss": 2.7062, + "theoretical_loss": 3.3878413021125904, + "tokens_seen": 2328231936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029955555555555557, + "loss": 2.4049, + "theoretical_loss": 3.3878335866627665, + "tokens_seen": 2328297472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002995353535353535, + "loss": 2.7816, + "theoretical_loss": 3.387825871490917, + "tokens_seen": 2328363008 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1312576, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.68485426902771, + "objective/train/theoretical_loss": 3.3878181565970245, + "objective/train/tokens_used": 687287776, + "theoretical_loss": 3.3878181565970245, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029951515151515153, + "loss": 2.589, + "theoretical_loss": 3.3878181565970245, + "tokens_seen": 2328428544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002994949494949495, + "loss": 2.7493, + "theoretical_loss": 3.387810441981071, + "tokens_seen": 2328494080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029947474747474745, + "loss": 2.8649, + "theoretical_loss": 3.3878027276430376, + "tokens_seen": 2328559616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029945454545454546, + "loss": 2.5579, + "theoretical_loss": 3.387795013582908, + "tokens_seen": 2328625152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029943434343434347, + "loss": 2.7985, + "theoretical_loss": 3.387787299800663, + "tokens_seen": 2328690688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029941414141414143, + "loss": 2.5694, + "theoretical_loss": 3.387779586296286, + "tokens_seen": 2328756224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993939393939394, + "loss": 2.7215, + "theoretical_loss": 3.387771873069758, + "tokens_seen": 2328821760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029937373737373734, + "loss": 2.7543, + "theoretical_loss": 3.3877641601210624, + "tokens_seen": 2328887296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029935353535353535, + "loss": 2.6073, + "theoretical_loss": 3.3877564474501805, + "tokens_seen": 2328952832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029933333333333336, + "loss": 2.7629, + "theoretical_loss": 3.3877487350570945, + "tokens_seen": 2329018368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002993131313131313, + "loss": 2.7311, + "theoretical_loss": 3.3877410229417873, + "tokens_seen": 2329083904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992929292929293, + "loss": 2.6353, + "theoretical_loss": 3.3877333111042405, + "tokens_seen": 2329149440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029927272727272723, + "loss": 2.528, + "theoretical_loss": 3.387725599544436, + "tokens_seen": 2329214976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992525252525253, + "loss": 2.8042, + "theoretical_loss": 3.3877178882623564, + "tokens_seen": 2329280512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029923232323232326, + "loss": 2.6906, + "theoretical_loss": 3.387710177257984, + "tokens_seen": 2329346048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002992121212121212, + "loss": 2.5092, + "theoretical_loss": 3.3877024665313007, + "tokens_seen": 2329411584 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029919191919191917, + "loss": 2.661, + "theoretical_loss": 3.387694756082289, + "tokens_seen": 2329477120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991717171717171, + "loss": 2.7408, + "theoretical_loss": 3.3876870459109307, + "tokens_seen": 2329542656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991515151515152, + "loss": 2.6609, + "theoretical_loss": 3.387679336017208, + "tokens_seen": 2329608192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029913131313131315, + "loss": 2.6333, + "theoretical_loss": 3.387671626401104, + "tokens_seen": 2329673728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002991111111111111, + "loss": 2.6113, + "theoretical_loss": 3.3876639170626, + "tokens_seen": 2329739264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029909090909090906, + "loss": 2.8584, + "theoretical_loss": 3.387656208001678, + "tokens_seen": 2329804800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029907070707070713, + "loss": 2.5668, + "theoretical_loss": 3.3876484992183205, + "tokens_seen": 2329870336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002990505050505051, + "loss": 2.6858, + "theoretical_loss": 3.38764079071251, + "tokens_seen": 2329935872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029903030303030304, + "loss": 2.7323, + "theoretical_loss": 3.3876330824842285, + "tokens_seen": 2330001408 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1313793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3467326164245605, + "objective/train/theoretical_loss": 3.387625374533458, + "objective/train/tokens_used": 688926176, + "theoretical_loss": 3.387625374533458, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.41, + "learning_rate": 0.000299010101010101, + "loss": 2.6143, + "theoretical_loss": 3.387625374533458, + "tokens_seen": 2330066944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029898989898989895, + "loss": 2.6251, + "theoretical_loss": 3.387617666860181, + "tokens_seen": 2330132480 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298969696969697, + "loss": 2.7216, + "theoretical_loss": 3.3876099594643794, + "tokens_seen": 2330198016 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298949494949495, + "loss": 2.5494, + "theoretical_loss": 3.387602252346036, + "tokens_seen": 2330263552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029892929292929293, + "loss": 2.5316, + "theoretical_loss": 3.3875945455051317, + "tokens_seen": 2330329088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002989090909090909, + "loss": 2.3143, + "theoretical_loss": 3.38758683894165, + "tokens_seen": 2330394624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988888888888889, + "loss": 2.5858, + "theoretical_loss": 3.3875791326555733, + "tokens_seen": 2330460160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988686868686869, + "loss": 2.645, + "theoretical_loss": 3.3875714266468826, + "tokens_seen": 2330525696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029884848484848487, + "loss": 2.7016, + "theoretical_loss": 3.387563720915561, + "tokens_seen": 2330591232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988282828282828, + "loss": 2.6237, + "theoretical_loss": 3.3875560154615907, + "tokens_seen": 2330656768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002988080808080808, + "loss": 2.652, + "theoretical_loss": 3.3875483102849535, + "tokens_seen": 2330722304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987878787878788, + "loss": 2.623, + "theoretical_loss": 3.387540605385632, + "tokens_seen": 2330787840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987676767676768, + "loss": 2.5389, + "theoretical_loss": 3.387532900763608, + "tokens_seen": 2330853376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029874747474747476, + "loss": 2.8234, + "theoretical_loss": 3.3875251964188635, + "tokens_seen": 2330918912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002987272727272727, + "loss": 2.8477, + "theoretical_loss": 3.3875174923513818, + "tokens_seen": 2330984448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029870707070707073, + "loss": 2.7656, + "theoretical_loss": 3.387509788561144, + "tokens_seen": 2331049984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986868686868687, + "loss": 2.8256, + "theoretical_loss": 3.3875020850481334, + "tokens_seen": 2331115520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986666666666667, + "loss": 2.6292, + "theoretical_loss": 3.3874943818123313, + "tokens_seen": 2331181056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029864646464646466, + "loss": 2.5944, + "theoretical_loss": 3.387486678853721, + "tokens_seen": 2331246592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986262626262626, + "loss": 2.7204, + "theoretical_loss": 3.3874789761722828, + "tokens_seen": 2331312128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002986060606060606, + "loss": 2.7286, + "theoretical_loss": 3.387471273768001, + "tokens_seen": 2331377664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985858585858586, + "loss": 2.6603, + "theoretical_loss": 3.3874635716408568, + "tokens_seen": 2331443200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985656565656566, + "loss": 2.7477, + "theoretical_loss": 3.3874558697908324, + "tokens_seen": 2331508736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029854545454545455, + "loss": 2.8985, + "theoretical_loss": 3.3874481682179107, + "tokens_seen": 2331574272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029852525252525256, + "loss": 2.6347, + "theoretical_loss": 3.3874404669220732, + "tokens_seen": 2331639808 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1314556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6114652156829834, + "objective/train/theoretical_loss": 3.3874327659033026, + "objective/train/tokens_used": 690564576, + "theoretical_loss": 3.3874327659033026, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002985050505050505, + "loss": 2.6562, + "theoretical_loss": 3.3874327659033026, + "tokens_seen": 2331705344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029848484848484847, + "loss": 2.5718, + "theoretical_loss": 3.3874250651615814, + "tokens_seen": 2331770880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984646464646465, + "loss": 2.8089, + "theoretical_loss": 3.3874173646968906, + "tokens_seen": 2331836416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029844444444444444, + "loss": 2.766, + "theoretical_loss": 3.3874096645092138, + "tokens_seen": 2331901952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029842424242424245, + "loss": 2.5761, + "theoretical_loss": 3.3874019645985327, + "tokens_seen": 2331967488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002984040404040404, + "loss": 2.6005, + "theoretical_loss": 3.3873942649648296, + "tokens_seen": 2332033024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029838383838383837, + "loss": 2.6817, + "theoretical_loss": 3.387386565608087, + "tokens_seen": 2332098560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983636363636364, + "loss": 2.7879, + "theoretical_loss": 3.3873788665282865, + "tokens_seen": 2332164096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983434343434344, + "loss": 2.8421, + "theoretical_loss": 3.3873711677254112, + "tokens_seen": 2332229632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029832323232323234, + "loss": 2.6882, + "theoretical_loss": 3.3873634691994425, + "tokens_seen": 2332295168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002983030303030303, + "loss": 2.6318, + "theoretical_loss": 3.3873557709503634, + "tokens_seen": 2332360704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029828282828282826, + "loss": 2.6868, + "theoretical_loss": 3.3873480729781558, + "tokens_seen": 2332426240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029826262626262627, + "loss": 2.7006, + "theoretical_loss": 3.387340375282802, + "tokens_seen": 2332491776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982424242424243, + "loss": 2.6474, + "theoretical_loss": 3.3873326778642845, + "tokens_seen": 2332557312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029822222222222224, + "loss": 2.8663, + "theoretical_loss": 3.387324980722585, + "tokens_seen": 2332622848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002982020202020202, + "loss": 2.5904, + "theoretical_loss": 3.387317283857686, + "tokens_seen": 2332688384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029818181818181815, + "loss": 2.6688, + "theoretical_loss": 3.38730958726957, + "tokens_seen": 2332753920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981616161616162, + "loss": 2.8699, + "theoretical_loss": 3.387301890958219, + "tokens_seen": 2332819456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981414141414142, + "loss": 2.7421, + "theoretical_loss": 3.3872941949236157, + "tokens_seen": 2332884992 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029812121212121213, + "loss": 2.7654, + "theoretical_loss": 3.387286499165742, + "tokens_seen": 2332950528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002981010101010101, + "loss": 2.6119, + "theoretical_loss": 3.3872788036845805, + "tokens_seen": 2333016064 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029808080808080804, + "loss": 2.5544, + "theoretical_loss": 3.387271108480113, + "tokens_seen": 2333081600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002980606060606061, + "loss": 2.5303, + "theoretical_loss": 3.3872634135523225, + "tokens_seen": 2333147136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029804040404040407, + "loss": 2.6113, + "theoretical_loss": 3.3872557189011903, + "tokens_seen": 2333212672 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298020202020202, + "loss": 2.6254, + "theoretical_loss": 3.3872480245266994, + "tokens_seen": 2333278208 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1315828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.716416835784912, + "objective/train/theoretical_loss": 3.3872403304288317, + "objective/train/tokens_used": 692202976, + "theoretical_loss": 3.3872403304288317, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.41, + "learning_rate": 0.000298, + "loss": 2.6113, + "theoretical_loss": 3.3872403304288317, + "tokens_seen": 2333343744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029797979797979794, + "loss": 2.6518, + "theoretical_loss": 3.38723263660757, + "tokens_seen": 2333409280 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297959595959596, + "loss": 2.5376, + "theoretical_loss": 3.387224943062896, + "tokens_seen": 2333474816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029793939393939396, + "loss": 2.4026, + "theoretical_loss": 3.3872172497947926, + "tokens_seen": 2333540352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002979191919191919, + "loss": 2.6358, + "theoretical_loss": 3.387209556803241, + "tokens_seen": 2333605888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029789898989898987, + "loss": 2.6053, + "theoretical_loss": 3.3872018640882247, + "tokens_seen": 2333671424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978787878787879, + "loss": 2.5129, + "theoretical_loss": 3.3871941716497256, + "tokens_seen": 2333736960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978585858585859, + "loss": 2.6842, + "theoretical_loss": 3.3871864794877258, + "tokens_seen": 2333802496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029783838383838385, + "loss": 2.6486, + "theoretical_loss": 3.387178787602208, + "tokens_seen": 2333868032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002978181818181818, + "loss": 2.9502, + "theoretical_loss": 3.3871710959931534, + "tokens_seen": 2333933568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029779797979797976, + "loss": 2.5373, + "theoretical_loss": 3.387163404660546, + "tokens_seen": 2333999104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977777777777778, + "loss": 2.5886, + "theoretical_loss": 3.3871557136043666, + "tokens_seen": 2334064640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977575757575758, + "loss": 2.3405, + "theoretical_loss": 3.3871480228245985, + "tokens_seen": 2334130176 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029773737373737374, + "loss": 2.3346, + "theoretical_loss": 3.3871403323212235, + "tokens_seen": 2334195712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002977171717171717, + "loss": 2.6052, + "theoretical_loss": 3.3871326420942243, + "tokens_seen": 2334261248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976969696969697, + "loss": 2.5529, + "theoretical_loss": 3.3871249521435827, + "tokens_seen": 2334326784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029767676767676767, + "loss": 2.7719, + "theoretical_loss": 3.387117262469281, + "tokens_seen": 2334392320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976565656565657, + "loss": 2.7081, + "theoretical_loss": 3.387109573071302, + "tokens_seen": 2334457856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029763636363636364, + "loss": 2.5492, + "theoretical_loss": 3.387101883949628, + "tokens_seen": 2334523392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002976161616161616, + "loss": 2.4894, + "theoretical_loss": 3.387094195104241, + "tokens_seen": 2334588928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002975959595959596, + "loss": 2.592, + "theoretical_loss": 3.387086506535123, + "tokens_seen": 2334654464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029757575757575756, + "loss": 2.5685, + "theoretical_loss": 3.3870788182422573, + "tokens_seen": 2334720000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029755555555555557, + "loss": 2.5954, + "theoretical_loss": 3.3870711302256256, + "tokens_seen": 2334785536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029753535353535353, + "loss": 2.7264, + "theoretical_loss": 3.3870634424852097, + "tokens_seen": 2334851072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029751515151515154, + "loss": 2.6613, + "theoretical_loss": 3.387055755020993, + "tokens_seen": 2334916608 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1316418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.525245428085327, + "objective/train/theoretical_loss": 3.387048067832957, + "objective/train/tokens_used": 693841376, + "theoretical_loss": 3.387048067832957, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974949494949495, + "loss": 2.8112, + "theoretical_loss": 3.387048067832957, + "tokens_seen": 2334982144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029747474747474745, + "loss": 2.5488, + "theoretical_loss": 3.3870403809210847, + "tokens_seen": 2335047680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029745454545454546, + "loss": 2.6353, + "theoretical_loss": 3.3870326942853577, + "tokens_seen": 2335113216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002974343434343434, + "loss": 2.6202, + "theoretical_loss": 3.387025007925759, + "tokens_seen": 2335178752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029741414141414143, + "loss": 2.7684, + "theoretical_loss": 3.387017321842271, + "tokens_seen": 2335244288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973939393939394, + "loss": 2.6454, + "theoretical_loss": 3.387009636034875, + "tokens_seen": 2335309824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973737373737374, + "loss": 2.6933, + "theoretical_loss": 3.3870019505035542, + "tokens_seen": 2335375360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029735353535353536, + "loss": 2.5152, + "theoretical_loss": 3.386994265248291, + "tokens_seen": 2335440896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029733333333333337, + "loss": 2.5607, + "theoretical_loss": 3.386986580269067, + "tokens_seen": 2335506432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002973131313131313, + "loss": 2.4857, + "theoretical_loss": 3.3869788955658655, + "tokens_seen": 2335571968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972929292929293, + "loss": 2.7405, + "theoretical_loss": 3.3869712111386683, + "tokens_seen": 2335637504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972727272727273, + "loss": 2.4506, + "theoretical_loss": 3.3869635269874574, + "tokens_seen": 2335703040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029725252525252525, + "loss": 2.6377, + "theoretical_loss": 3.386955843112216, + "tokens_seen": 2335768576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029723232323232326, + "loss": 2.7311, + "theoretical_loss": 3.386948159512926, + "tokens_seen": 2335834112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002972121212121212, + "loss": 2.5078, + "theoretical_loss": 3.386940476189569, + "tokens_seen": 2335899648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971919191919192, + "loss": 2.668, + "theoretical_loss": 3.3869327931421287, + "tokens_seen": 2335965184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971717171717172, + "loss": 2.5063, + "theoretical_loss": 3.3869251103705866, + "tokens_seen": 2336030720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971515151515152, + "loss": 2.577, + "theoretical_loss": 3.3869174278749257, + "tokens_seen": 2336096256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029713131313131315, + "loss": 2.4665, + "theoretical_loss": 3.386909745655128, + "tokens_seen": 2336161792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002971111111111111, + "loss": 2.5282, + "theoretical_loss": 3.386902063711175, + "tokens_seen": 2336227328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029709090909090907, + "loss": 2.6981, + "theoretical_loss": 3.386894382043051, + "tokens_seen": 2336292864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970707070707071, + "loss": 2.5867, + "theoretical_loss": 3.386886700650736, + "tokens_seen": 2336358400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002970505050505051, + "loss": 2.6298, + "theoretical_loss": 3.3868790195342147, + "tokens_seen": 2336423936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029703030303030305, + "loss": 2.617, + "theoretical_loss": 3.3868713386934677, + "tokens_seen": 2336489472 + }, + { + "epoch": 0.41, + "learning_rate": 0.000297010101010101, + "loss": 2.6431, + "theoretical_loss": 3.386863658128478, + "tokens_seen": 2336555008 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1317715, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.574777841567993, + "objective/train/theoretical_loss": 3.386855977839228, + "objective/train/tokens_used": 695479776, + "theoretical_loss": 3.386855977839228, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029698989898989896, + "loss": 2.5958, + "theoretical_loss": 3.386855977839228, + "tokens_seen": 2336620544 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296969696969697, + "loss": 2.7116, + "theoretical_loss": 3.3868482978257006, + "tokens_seen": 2336686080 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296949494949495, + "loss": 2.7702, + "theoretical_loss": 3.386840618087877, + "tokens_seen": 2336751616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029692929292929294, + "loss": 2.6611, + "theoretical_loss": 3.3868329386257403, + "tokens_seen": 2336817152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002969090909090909, + "loss": 2.8769, + "theoretical_loss": 3.386825259439273, + "tokens_seen": 2336882688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029688888888888885, + "loss": 2.5341, + "theoretical_loss": 3.386817580528457, + "tokens_seen": 2336948224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968686868686869, + "loss": 2.7857, + "theoretical_loss": 3.386809901893275, + "tokens_seen": 2337013760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968484848484849, + "loss": 2.6569, + "theoretical_loss": 3.386802223533709, + "tokens_seen": 2337079296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029682828282828283, + "loss": 2.6928, + "theoretical_loss": 3.3867945454497423, + "tokens_seen": 2337144832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002968080808080808, + "loss": 2.5777, + "theoretical_loss": 3.386786867641356, + "tokens_seen": 2337210368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967878787878788, + "loss": 2.8281, + "theoretical_loss": 3.3867791901085336, + "tokens_seen": 2337275904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967676767676768, + "loss": 2.4311, + "theoretical_loss": 3.386771512851257, + "tokens_seen": 2337341440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029674747474747477, + "loss": 2.7262, + "theoretical_loss": 3.386763835869508, + "tokens_seen": 2337406976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967272727272727, + "loss": 2.7159, + "theoretical_loss": 3.38675615916327, + "tokens_seen": 2337472512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002967070707070707, + "loss": 2.6483, + "theoretical_loss": 3.386748482732525, + "tokens_seen": 2337538048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966868686868687, + "loss": 2.538, + "theoretical_loss": 3.3867408065772553, + "tokens_seen": 2337603584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966666666666667, + "loss": 2.5789, + "theoretical_loss": 3.3867331306974435, + "tokens_seen": 2337669120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029664646464646466, + "loss": 2.7758, + "theoretical_loss": 3.3867254550930714, + "tokens_seen": 2337734656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966262626262626, + "loss": 2.3797, + "theoretical_loss": 3.386717779764122, + "tokens_seen": 2337800192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002966060606060606, + "loss": 2.6032, + "theoretical_loss": 3.386710104710578, + "tokens_seen": 2337865728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965858585858586, + "loss": 2.7274, + "theoretical_loss": 3.386702429932421, + "tokens_seen": 2337931264 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965656565656566, + "loss": 2.6564, + "theoretical_loss": 3.386694755429634, + "tokens_seen": 2337996800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029654545454545455, + "loss": 2.6385, + "theoretical_loss": 3.3866870812021985, + "tokens_seen": 2338062336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965252525252525, + "loss": 2.5347, + "theoretical_loss": 3.3866794072500976, + "tokens_seen": 2338127872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002965050505050505, + "loss": 2.4261, + "theoretical_loss": 3.3866717335733143, + "tokens_seen": 2338193408 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1318542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8225085735321045, + "objective/train/theoretical_loss": 3.38666406017183, + "objective/train/tokens_used": 697118176, + "theoretical_loss": 3.38666406017183, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964848484848485, + "loss": 2.5863, + "theoretical_loss": 3.38666406017183, + "tokens_seen": 2338258944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964646464646465, + "loss": 2.7754, + "theoretical_loss": 3.3866563870456274, + "tokens_seen": 2338324480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029644444444444445, + "loss": 2.5538, + "theoretical_loss": 3.386648714194689, + "tokens_seen": 2338390016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964242424242424, + "loss": 2.6842, + "theoretical_loss": 3.386641041618997, + "tokens_seen": 2338455552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002964040404040404, + "loss": 2.8594, + "theoretical_loss": 3.386633369318534, + "tokens_seen": 2338521088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029638383838383837, + "loss": 2.6688, + "theoretical_loss": 3.3866256972932827, + "tokens_seen": 2338586624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963636363636364, + "loss": 2.5245, + "theoretical_loss": 3.386618025543225, + "tokens_seen": 2338652160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029634343434343434, + "loss": 2.6111, + "theoretical_loss": 3.3866103540683437, + "tokens_seen": 2338717696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029632323232323235, + "loss": 2.7836, + "theoretical_loss": 3.386602682868621, + "tokens_seen": 2338783232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002963030303030303, + "loss": 2.5031, + "theoretical_loss": 3.3865950119440393, + "tokens_seen": 2338848768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029628282828282826, + "loss": 2.5916, + "theoretical_loss": 3.3865873412945815, + "tokens_seen": 2338914304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962626262626263, + "loss": 2.7443, + "theoretical_loss": 3.386579670920229, + "tokens_seen": 2338979840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029624242424242423, + "loss": 2.6094, + "theoretical_loss": 3.386572000820965, + "tokens_seen": 2339045376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029622222222222224, + "loss": 2.6119, + "theoretical_loss": 3.386564330996772, + "tokens_seen": 2339110912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002962020202020202, + "loss": 2.6978, + "theoretical_loss": 3.3865566614476323, + "tokens_seen": 2339176448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029618181818181816, + "loss": 2.3142, + "theoretical_loss": 3.3865489921735277, + "tokens_seen": 2339241984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029616161616161617, + "loss": 2.6912, + "theoretical_loss": 3.3865413231744417, + "tokens_seen": 2339307520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961414141414142, + "loss": 2.6887, + "theoretical_loss": 3.386533654450356, + "tokens_seen": 2339373056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029612121212121214, + "loss": 2.7416, + "theoretical_loss": 3.386525986001253, + "tokens_seen": 2339438592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002961010101010101, + "loss": 2.7078, + "theoretical_loss": 3.3865183178271154, + "tokens_seen": 2339504128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029608080808080805, + "loss": 2.5197, + "theoretical_loss": 3.386510649927926, + "tokens_seen": 2339569664 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029606060606060606, + "loss": 2.7871, + "theoretical_loss": 3.386502982303667, + "tokens_seen": 2339635200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029604040404040407, + "loss": 2.5326, + "theoretical_loss": 3.38649531495432, + "tokens_seen": 2339700736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029602020202020203, + "loss": 2.6778, + "theoretical_loss": 3.3864876478798687, + "tokens_seen": 2339766272 + }, + { + "epoch": 0.41, + "learning_rate": 0.000296, + "loss": 2.4329, + "theoretical_loss": 3.386479981080295, + "tokens_seen": 2339831808 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1319951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.179854154586792, + "objective/train/theoretical_loss": 3.3864723145555806, + "objective/train/tokens_used": 698756576, + "theoretical_loss": 3.3864723145555806, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029597979797979794, + "loss": 3.0214, + "theoretical_loss": 3.3864723145555806, + "tokens_seen": 2339897344 + }, + { + "epoch": 0.41, + "learning_rate": 0.000295959595959596, + "loss": 2.6822, + "theoretical_loss": 3.3864646483057093, + "tokens_seen": 2339962880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029593939393939396, + "loss": 2.1883, + "theoretical_loss": 3.386456982330663, + "tokens_seen": 2340028416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002959191919191919, + "loss": 2.4673, + "theoretical_loss": 3.3864493166304235, + "tokens_seen": 2340093952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958989898989899, + "loss": 2.5995, + "theoretical_loss": 3.3864416512049744, + "tokens_seen": 2340159488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029587878787878783, + "loss": 2.7869, + "theoretical_loss": 3.386433986054297, + "tokens_seen": 2340225024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958585858585859, + "loss": 2.5513, + "theoretical_loss": 3.386426321178375, + "tokens_seen": 2340290560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029583838383838386, + "loss": 2.8497, + "theoretical_loss": 3.38641865657719, + "tokens_seen": 2340356096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002958181818181818, + "loss": 2.5766, + "theoretical_loss": 3.3864109922507244, + "tokens_seen": 2340421632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029579797979797977, + "loss": 2.4274, + "theoretical_loss": 3.386403328198961, + "tokens_seen": 2340487168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957777777777778, + "loss": 2.7949, + "theoretical_loss": 3.3863956644218822, + "tokens_seen": 2340552704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957575757575758, + "loss": 2.6733, + "theoretical_loss": 3.386388000919471, + "tokens_seen": 2340618240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029573737373737375, + "loss": 2.6649, + "theoretical_loss": 3.3863803376917083, + "tokens_seen": 2340683776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002957171717171717, + "loss": 2.6137, + "theoretical_loss": 3.3863726747385785, + "tokens_seen": 2340749312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029569696969696966, + "loss": 2.7176, + "theoretical_loss": 3.386365012060063, + "tokens_seen": 2340814848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029567676767676773, + "loss": 2.5201, + "theoretical_loss": 3.386357349656144, + "tokens_seen": 2340880384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956565656565657, + "loss": 2.7437, + "theoretical_loss": 3.3863496875268044, + "tokens_seen": 2340945920 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029563636363636364, + "loss": 2.5785, + "theoretical_loss": 3.386342025672027, + "tokens_seen": 2341011456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002956161616161616, + "loss": 2.7471, + "theoretical_loss": 3.386334364091794, + "tokens_seen": 2341076992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955959595959596, + "loss": 2.4495, + "theoretical_loss": 3.386326702786087, + "tokens_seen": 2341142528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955757575757576, + "loss": 2.3624, + "theoretical_loss": 3.38631904175489, + "tokens_seen": 2341208064 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955555555555556, + "loss": 2.6221, + "theoretical_loss": 3.386311380998185, + "tokens_seen": 2341273600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00029553535353535353, + "loss": 2.6063, + "theoretical_loss": 3.386303720515954, + "tokens_seen": 2341339136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002955151515151515, + "loss": 2.4995, + "theoretical_loss": 3.3862960603081795, + "tokens_seen": 2341404672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002954949494949495, + "loss": 2.8646, + "theoretical_loss": 3.3862884003748444, + "tokens_seen": 2341470208 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 1320683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.422520399093628, + "objective/train/theoretical_loss": 3.386280740715931, + "objective/train/tokens_used": 700394976, + "theoretical_loss": 3.386280740715931, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.41, + "learning_rate": 0.0002954747474747475, + "loss": 2.5639, + "theoretical_loss": 3.386280740715931, + "tokens_seen": 2341535744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029545454545454547, + "loss": 2.636, + "theoretical_loss": 3.3862730813314217, + "tokens_seen": 2341601280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029543434343434343, + "loss": 2.516, + "theoretical_loss": 3.3862654222212996, + "tokens_seen": 2341666816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029541414141414144, + "loss": 2.6289, + "theoretical_loss": 3.386257763385546, + "tokens_seen": 2341732352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953939393939394, + "loss": 2.5459, + "theoretical_loss": 3.3862501048241445, + "tokens_seen": 2341797888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953737373737374, + "loss": 2.6831, + "theoretical_loss": 3.386242446537077, + "tokens_seen": 2341863424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029535353535353536, + "loss": 2.7485, + "theoretical_loss": 3.386234788524326, + "tokens_seen": 2341928960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002953333333333333, + "loss": 2.5413, + "theoretical_loss": 3.3862271307858745, + "tokens_seen": 2341994496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029531313131313133, + "loss": 2.521, + "theoretical_loss": 3.3862194733217046, + "tokens_seen": 2342060032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952929292929293, + "loss": 2.7846, + "theoretical_loss": 3.386211816131799, + "tokens_seen": 2342125568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952727272727273, + "loss": 2.7343, + "theoretical_loss": 3.3862041592161396, + "tokens_seen": 2342191104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029525252525252526, + "loss": 2.5494, + "theoretical_loss": 3.38619650257471, + "tokens_seen": 2342256640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029523232323232327, + "loss": 2.7587, + "theoretical_loss": 3.3861888462074914, + "tokens_seen": 2342322176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002952121212121212, + "loss": 2.6206, + "theoretical_loss": 3.3861811901144674, + "tokens_seen": 2342387712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951919191919192, + "loss": 2.8098, + "theoretical_loss": 3.38617353429562, + "tokens_seen": 2342453248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951717171717172, + "loss": 2.6267, + "theoretical_loss": 3.3861658787509317, + "tokens_seen": 2342518784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029515151515151515, + "loss": 2.5424, + "theoretical_loss": 3.3861582234803853, + "tokens_seen": 2342584320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029513131313131316, + "loss": 2.4622, + "theoretical_loss": 3.3861505684839632, + "tokens_seen": 2342649856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002951111111111111, + "loss": 2.7357, + "theoretical_loss": 3.386142913761648, + "tokens_seen": 2342715392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950909090909091, + "loss": 2.4491, + "theoretical_loss": 3.3861352593134217, + "tokens_seen": 2342780928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950707070707071, + "loss": 2.5825, + "theoretical_loss": 3.3861276051392672, + "tokens_seen": 2342846464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002950505050505051, + "loss": 2.5191, + "theoretical_loss": 3.3861199512391673, + "tokens_seen": 2342912000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029503030303030305, + "loss": 2.6143, + "theoretical_loss": 3.386112297613104, + "tokens_seen": 2342977536 + }, + { + "epoch": 0.42, + "learning_rate": 0.000295010101010101, + "loss": 2.6888, + "theoretical_loss": 3.38610464426106, + "tokens_seen": 2343043072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029498989898989897, + "loss": 2.5973, + "theoretical_loss": 3.386096991183018, + "tokens_seen": 2343108608 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1321245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0052030086517334, + "objective/train/theoretical_loss": 3.3860893383789605, + "objective/train/tokens_used": 702033376, + "theoretical_loss": 3.3860893383789605, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294969696969697, + "loss": 2.801, + "theoretical_loss": 3.3860893383789605, + "tokens_seen": 2343174144 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294949494949495, + "loss": 2.5194, + "theoretical_loss": 3.38608168584887, + "tokens_seen": 2343239680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029492929292929295, + "loss": 2.8026, + "theoretical_loss": 3.386074033592729, + "tokens_seen": 2343305216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002949090909090909, + "loss": 2.4889, + "theoretical_loss": 3.3860663816105196, + "tokens_seen": 2343370752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029488888888888886, + "loss": 2.5535, + "theoretical_loss": 3.386058729902225, + "tokens_seen": 2343436288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948686868686869, + "loss": 2.629, + "theoretical_loss": 3.3860510784678275, + "tokens_seen": 2343501824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948484848484849, + "loss": 2.9178, + "theoretical_loss": 3.3860434273073095, + "tokens_seen": 2343567360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029482828282828284, + "loss": 2.3001, + "theoretical_loss": 3.3860357764206537, + "tokens_seen": 2343632896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002948080808080808, + "loss": 2.6427, + "theoretical_loss": 3.3860281258078424, + "tokens_seen": 2343698432 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029478787878787875, + "loss": 2.609, + "theoretical_loss": 3.3860204754688583, + "tokens_seen": 2343763968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947676767676768, + "loss": 2.6246, + "theoretical_loss": 3.3860128254036845, + "tokens_seen": 2343829504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947474747474748, + "loss": 2.638, + "theoretical_loss": 3.3860051756123024, + "tokens_seen": 2343895040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029472727272727273, + "loss": 2.9118, + "theoretical_loss": 3.3859975260946955, + "tokens_seen": 2343960576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002947070707070707, + "loss": 2.7692, + "theoretical_loss": 3.385989876850846, + "tokens_seen": 2344026112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029468686868686864, + "loss": 2.7103, + "theoretical_loss": 3.3859822278807363, + "tokens_seen": 2344091648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946666666666667, + "loss": 2.7128, + "theoretical_loss": 3.3859745791843494, + "tokens_seen": 2344157184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029464646464646467, + "loss": 2.6473, + "theoretical_loss": 3.385966930761667, + "tokens_seen": 2344222720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946262626262626, + "loss": 2.6653, + "theoretical_loss": 3.3859592826126725, + "tokens_seen": 2344288256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002946060606060606, + "loss": 2.6009, + "theoretical_loss": 3.3859516347373484, + "tokens_seen": 2344353792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945858585858586, + "loss": 2.5443, + "theoretical_loss": 3.3859439871356765, + "tokens_seen": 2344419328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945656565656566, + "loss": 2.7419, + "theoretical_loss": 3.3859363398076407, + "tokens_seen": 2344484864 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029454545454545456, + "loss": 2.7701, + "theoretical_loss": 3.3859286927532217, + "tokens_seen": 2344550400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002945252525252525, + "loss": 2.5697, + "theoretical_loss": 3.385921045972404, + "tokens_seen": 2344615936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029450505050505047, + "loss": 2.571, + "theoretical_loss": 3.385913399465169, + "tokens_seen": 2344681472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944848484848485, + "loss": 2.6611, + "theoretical_loss": 3.3859057532314996, + "tokens_seen": 2344747008 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1322388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.708512306213379, + "objective/train/theoretical_loss": 3.385898107271378, + "objective/train/tokens_used": 703671776, + "theoretical_loss": 3.385898107271378, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944646464646465, + "loss": 2.9322, + "theoretical_loss": 3.385898107271378, + "tokens_seen": 2344812544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029444444444444445, + "loss": 2.6482, + "theoretical_loss": 3.385890461584787, + "tokens_seen": 2344878080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944242424242424, + "loss": 2.5017, + "theoretical_loss": 3.3858828161717094, + "tokens_seen": 2344943616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002944040404040404, + "loss": 2.7315, + "theoretical_loss": 3.385875171032128, + "tokens_seen": 2345009152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943838383838384, + "loss": 2.7529, + "theoretical_loss": 3.3858675261660247, + "tokens_seen": 2345074688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943636363636364, + "loss": 2.4727, + "theoretical_loss": 3.3858598815733822, + "tokens_seen": 2345140224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029434343434343434, + "loss": 2.7133, + "theoretical_loss": 3.3858522372541833, + "tokens_seen": 2345205760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943232323232323, + "loss": 2.4668, + "theoretical_loss": 3.385844593208411, + "tokens_seen": 2345271296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002943030303030303, + "loss": 2.7062, + "theoretical_loss": 3.3858369494360465, + "tokens_seen": 2345336832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029428282828282827, + "loss": 2.5438, + "theoretical_loss": 3.385829305937074, + "tokens_seen": 2345402368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942626262626263, + "loss": 2.7066, + "theoretical_loss": 3.385821662711475, + "tokens_seen": 2345467904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029424242424242424, + "loss": 2.7555, + "theoretical_loss": 3.385814019759233, + "tokens_seen": 2345533440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029422222222222225, + "loss": 2.5234, + "theoretical_loss": 3.3858063770803293, + "tokens_seen": 2345598976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002942020202020202, + "loss": 2.5084, + "theoretical_loss": 3.3857987346747476, + "tokens_seen": 2345664512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029418181818181816, + "loss": 2.7225, + "theoretical_loss": 3.38579109254247, + "tokens_seen": 2345730048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941616161616162, + "loss": 2.4881, + "theoretical_loss": 3.3857834506834794, + "tokens_seen": 2345795584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029414141414141413, + "loss": 2.7698, + "theoretical_loss": 3.385775809097758, + "tokens_seen": 2345861120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029412121212121214, + "loss": 2.7729, + "theoretical_loss": 3.3857681677852884, + "tokens_seen": 2345926656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002941010101010101, + "loss": 2.8765, + "theoretical_loss": 3.3857605267460538, + "tokens_seen": 2345992192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029408080808080805, + "loss": 2.5235, + "theoretical_loss": 3.385752885980036, + "tokens_seen": 2346057728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029406060606060607, + "loss": 2.5943, + "theoretical_loss": 3.385745245487218, + "tokens_seen": 2346123264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002940404040404041, + "loss": 2.4821, + "theoretical_loss": 3.385737605267583, + "tokens_seen": 2346188800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029402020202020203, + "loss": 2.671, + "theoretical_loss": 3.3857299653211124, + "tokens_seen": 2346254336 + }, + { + "epoch": 0.42, + "learning_rate": 0.000294, + "loss": 2.578, + "theoretical_loss": 3.3857223256477895, + "tokens_seen": 2346319872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029397979797979795, + "loss": 2.6153, + "theoretical_loss": 3.3857146862475966, + "tokens_seen": 2346385408 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1323122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.686946392059326, + "objective/train/theoretical_loss": 3.385707047120517, + "objective/train/tokens_used": 705310176, + "theoretical_loss": 3.385707047120517, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029395959595959596, + "loss": 2.5544, + "theoretical_loss": 3.385707047120517, + "tokens_seen": 2346450944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029393939393939397, + "loss": 2.6632, + "theoretical_loss": 3.3856994082665324, + "tokens_seen": 2346516480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002939191919191919, + "loss": 2.8116, + "theoretical_loss": 3.385691769685626, + "tokens_seen": 2346582016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938989898989899, + "loss": 2.7437, + "theoretical_loss": 3.38568413137778, + "tokens_seen": 2346647552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938787878787879, + "loss": 2.5964, + "theoretical_loss": 3.3856764933429773, + "tokens_seen": 2346713088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938585858585859, + "loss": 2.408, + "theoretical_loss": 3.3856688555812005, + "tokens_seen": 2346778624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029383838383838386, + "loss": 2.7538, + "theoretical_loss": 3.3856612180924324, + "tokens_seen": 2346844160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002938181818181818, + "loss": 2.8753, + "theoretical_loss": 3.385653580876655, + "tokens_seen": 2346909696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937979797979798, + "loss": 2.674, + "theoretical_loss": 3.3856459439338513, + "tokens_seen": 2346975232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937777777777778, + "loss": 2.7639, + "theoretical_loss": 3.3856383072640046, + "tokens_seen": 2347040768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937575757575758, + "loss": 2.6402, + "theoretical_loss": 3.385630670867096, + "tokens_seen": 2347106304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029373737373737375, + "loss": 2.5126, + "theoretical_loss": 3.3856230347431095, + "tokens_seen": 2347171840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002937171717171717, + "loss": 2.7758, + "theoretical_loss": 3.385615398892027, + "tokens_seen": 2347237376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029369696969696967, + "loss": 2.5912, + "theoretical_loss": 3.3856077633138315, + "tokens_seen": 2347302912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029367676767676773, + "loss": 2.7393, + "theoretical_loss": 3.3856001280085053, + "tokens_seen": 2347368448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936565656565657, + "loss": 2.5351, + "theoretical_loss": 3.3855924929760315, + "tokens_seen": 2347433984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029363636363636365, + "loss": 2.6031, + "theoretical_loss": 3.385584858216392, + "tokens_seen": 2347499520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002936161616161616, + "loss": 2.7566, + "theoretical_loss": 3.38557722372957, + "tokens_seen": 2347565056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029359595959595956, + "loss": 2.5678, + "theoretical_loss": 3.3855695895155478, + "tokens_seen": 2347630592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935757575757576, + "loss": 2.7319, + "theoretical_loss": 3.3855619555743086, + "tokens_seen": 2347696128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935555555555556, + "loss": 2.6191, + "theoretical_loss": 3.3855543219058344, + "tokens_seen": 2347761664 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029353535353535354, + "loss": 2.5899, + "theoretical_loss": 3.385546688510108, + "tokens_seen": 2347827200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002935151515151515, + "loss": 2.7846, + "theoretical_loss": 3.3855390553871128, + "tokens_seen": 2347892736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934949494949495, + "loss": 2.5968, + "theoretical_loss": 3.3855314225368303, + "tokens_seen": 2347958272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934747474747475, + "loss": 2.7216, + "theoretical_loss": 3.385523789959244, + "tokens_seen": 2348023808 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1324638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6331121921539307, + "objective/train/theoretical_loss": 3.385516157654336, + "objective/train/tokens_used": 706948576, + "theoretical_loss": 3.385516157654336, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934545454545455, + "loss": 2.6591, + "theoretical_loss": 3.385516157654336, + "tokens_seen": 2348089344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029343434343434343, + "loss": 2.7627, + "theoretical_loss": 3.3855085256220887, + "tokens_seen": 2348154880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002934141414141414, + "loss": 2.548, + "theoretical_loss": 3.3855008938624858, + "tokens_seen": 2348220416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933939393939394, + "loss": 2.784, + "theoretical_loss": 3.3854932623755087, + "tokens_seen": 2348285952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933737373737374, + "loss": 2.518, + "theoretical_loss": 3.385485631161141, + "tokens_seen": 2348351488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029335353535353537, + "loss": 2.3231, + "theoretical_loss": 3.3854780002193654, + "tokens_seen": 2348417024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933333333333333, + "loss": 2.5679, + "theoretical_loss": 3.3854703695501636, + "tokens_seen": 2348482560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002933131313131313, + "loss": 2.6462, + "theoretical_loss": 3.3854627391535193, + "tokens_seen": 2348548096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932929292929293, + "loss": 2.8078, + "theoretical_loss": 3.3854551090294143, + "tokens_seen": 2348613632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932727272727273, + "loss": 2.6492, + "theoretical_loss": 3.385447479177832, + "tokens_seen": 2348679168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029325252525252526, + "loss": 2.5405, + "theoretical_loss": 3.3854398495987548, + "tokens_seen": 2348744704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002932323232323232, + "loss": 2.5035, + "theoretical_loss": 3.385432220292165, + "tokens_seen": 2348810240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029321212121212123, + "loss": 2.4862, + "theoretical_loss": 3.3854245912580456, + "tokens_seen": 2348875776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931919191919192, + "loss": 2.321, + "theoretical_loss": 3.3854169624963797, + "tokens_seen": 2348941312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931717171717172, + "loss": 2.3676, + "theoretical_loss": 3.385409334007149, + "tokens_seen": 2349006848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029315151515151515, + "loss": 2.4442, + "theoretical_loss": 3.385401705790337, + "tokens_seen": 2349072384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931313131313131, + "loss": 2.5083, + "theoretical_loss": 3.385394077845926, + "tokens_seen": 2349137920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002931111111111111, + "loss": 2.6836, + "theoretical_loss": 3.3853864501738986, + "tokens_seen": 2349203456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930909090909091, + "loss": 2.6238, + "theoretical_loss": 3.3853788227742374, + "tokens_seen": 2349268992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002930707070707071, + "loss": 2.6135, + "theoretical_loss": 3.385371195646926, + "tokens_seen": 2349334528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029305050505050505, + "loss": 2.8295, + "theoretical_loss": 3.3853635687919454, + "tokens_seen": 2349400064 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029303030303030306, + "loss": 2.8278, + "theoretical_loss": 3.38535594220928, + "tokens_seen": 2349465600 + }, + { + "epoch": 0.42, + "learning_rate": 0.000293010101010101, + "loss": 2.6805, + "theoretical_loss": 3.3853483158989115, + "tokens_seen": 2349531136 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029298989898989897, + "loss": 2.6131, + "theoretical_loss": 3.3853406898608225, + "tokens_seen": 2349596672 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292969696969697, + "loss": 2.6318, + "theoretical_loss": 3.385333064094996, + "tokens_seen": 2349662208 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1325195, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3687126636505127, + "objective/train/theoretical_loss": 3.385325438601415, + "objective/train/tokens_used": 708586976, + "theoretical_loss": 3.385325438601415, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029294949494949494, + "loss": 2.6364, + "theoretical_loss": 3.385325438601415, + "tokens_seen": 2349727744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029292929292929295, + "loss": 2.8055, + "theoretical_loss": 3.3853178133800617, + "tokens_seen": 2349793280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002929090909090909, + "loss": 2.4316, + "theoretical_loss": 3.385310188430919, + "tokens_seen": 2349858816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029288888888888886, + "loss": 2.6335, + "theoretical_loss": 3.3853025637539695, + "tokens_seen": 2349924352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928686868686869, + "loss": 2.6266, + "theoretical_loss": 3.385294939349196, + "tokens_seen": 2349989888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928484848484849, + "loss": 2.3951, + "theoretical_loss": 3.385287315216581, + "tokens_seen": 2350055424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029282828282828284, + "loss": 2.3585, + "theoretical_loss": 3.385279691356107, + "tokens_seen": 2350120960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002928080808080808, + "loss": 2.8629, + "theoretical_loss": 3.3852720677677577, + "tokens_seen": 2350186496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029278787878787876, + "loss": 2.5454, + "theoretical_loss": 3.385264444451515, + "tokens_seen": 2350252032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029276767676767677, + "loss": 2.7257, + "theoretical_loss": 3.3852568214073617, + "tokens_seen": 2350317568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927474747474748, + "loss": 2.517, + "theoretical_loss": 3.38524919863528, + "tokens_seen": 2350383104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029272727272727274, + "loss": 2.4239, + "theoretical_loss": 3.3852415761352534, + "tokens_seen": 2350448640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002927070707070707, + "loss": 2.947, + "theoretical_loss": 3.3852339539072647, + "tokens_seen": 2350514176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029268686868686865, + "loss": 2.58, + "theoretical_loss": 3.3852263319512956, + "tokens_seen": 2350579712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926666666666667, + "loss": 2.6567, + "theoretical_loss": 3.38521871026733, + "tokens_seen": 2350645248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029264646464646467, + "loss": 2.629, + "theoretical_loss": 3.3852110888553497, + "tokens_seen": 2350710784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029262626262626263, + "loss": 2.6805, + "theoretical_loss": 3.385203467715338, + "tokens_seen": 2350776320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002926060606060606, + "loss": 2.7369, + "theoretical_loss": 3.385195846847277, + "tokens_seen": 2350841856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029258585858585854, + "loss": 2.5131, + "theoretical_loss": 3.38518822625115, + "tokens_seen": 2350907392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925656565656566, + "loss": 2.7112, + "theoretical_loss": 3.38518060592694, + "tokens_seen": 2350972928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029254545454545456, + "loss": 2.3817, + "theoretical_loss": 3.3851729858746284, + "tokens_seen": 2351038464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925252525252525, + "loss": 2.5142, + "theoretical_loss": 3.385165366094199, + "tokens_seen": 2351104000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002925050505050505, + "loss": 2.533, + "theoretical_loss": 3.3851577465856346, + "tokens_seen": 2351169536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924848484848485, + "loss": 2.6736, + "theoretical_loss": 3.3851501273489175, + "tokens_seen": 2351235072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924646464646465, + "loss": 2.8991, + "theoretical_loss": 3.38514250838403, + "tokens_seen": 2351300608 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1325921, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6260910034179688, + "objective/train/theoretical_loss": 3.3851348896909563, + "objective/train/tokens_used": 710225376, + "theoretical_loss": 3.3851348896909563, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029244444444444446, + "loss": 2.5636, + "theoretical_loss": 3.3851348896909563, + "tokens_seen": 2351366144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002924242424242424, + "loss": 2.6198, + "theoretical_loss": 3.3851272712696776, + "tokens_seen": 2351431680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029240404040404037, + "loss": 2.5822, + "theoretical_loss": 3.385119653120177, + "tokens_seen": 2351497216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923838383838384, + "loss": 2.6212, + "theoretical_loss": 3.3851120352424378, + "tokens_seen": 2351562752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923636363636364, + "loss": 2.7307, + "theoretical_loss": 3.385104417636442, + "tokens_seen": 2351628288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029234343434343435, + "loss": 2.3506, + "theoretical_loss": 3.385096800302173, + "tokens_seen": 2351693824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923232323232323, + "loss": 2.57, + "theoretical_loss": 3.385089183239613, + "tokens_seen": 2351759360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002923030303030303, + "loss": 2.4977, + "theoretical_loss": 3.3850815664487452, + "tokens_seen": 2351824896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922828282828283, + "loss": 2.63, + "theoretical_loss": 3.385073949929552, + "tokens_seen": 2351890432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922626262626263, + "loss": 2.7815, + "theoretical_loss": 3.3850663336820164, + "tokens_seen": 2351955968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029224242424242424, + "loss": 2.7007, + "theoretical_loss": 3.3850587177061207, + "tokens_seen": 2352021504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922222222222222, + "loss": 2.6827, + "theoretical_loss": 3.385051102001848, + "tokens_seen": 2352087040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002922020202020202, + "loss": 2.7474, + "theoretical_loss": 3.385043486569181, + "tokens_seen": 2352152576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921818181818182, + "loss": 2.7234, + "theoretical_loss": 3.3850358714081024, + "tokens_seen": 2352218112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921616161616162, + "loss": 2.6846, + "theoretical_loss": 3.385028256518595, + "tokens_seen": 2352283648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029214141414141414, + "loss": 2.6508, + "theoretical_loss": 3.385020641900642, + "tokens_seen": 2352349184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029212121212121215, + "loss": 2.7721, + "theoretical_loss": 3.385013027554225, + "tokens_seen": 2352414720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002921010101010101, + "loss": 2.7745, + "theoretical_loss": 3.3850054134793277, + "tokens_seen": 2352480256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002920808080808081, + "loss": 2.5833, + "theoretical_loss": 3.3849977996759324, + "tokens_seen": 2352545792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029206060606060607, + "loss": 2.6717, + "theoretical_loss": 3.384990186144022, + "tokens_seen": 2352611328 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029204040404040403, + "loss": 2.6104, + "theoretical_loss": 3.3849825728835796, + "tokens_seen": 2352676864 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029202020202020204, + "loss": 2.6631, + "theoretical_loss": 3.3849749598945875, + "tokens_seen": 2352742400 + }, + { + "epoch": 0.42, + "learning_rate": 0.000292, + "loss": 2.4635, + "theoretical_loss": 3.3849673471770285, + "tokens_seen": 2352807936 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291979797979798, + "loss": 2.6463, + "theoretical_loss": 3.3849597347308853, + "tokens_seen": 2352873472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029195959595959596, + "loss": 2.6798, + "theoretical_loss": 3.3849521225561414, + "tokens_seen": 2352939008 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1327039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.520237684249878, + "objective/train/theoretical_loss": 3.3849445106527787, + "objective/train/tokens_used": 711863776, + "theoretical_loss": 3.3849445106527787, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291939393939394, + "loss": 2.7601, + "theoretical_loss": 3.3849445106527787, + "tokens_seen": 2353004544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029191919191919193, + "loss": 2.4533, + "theoretical_loss": 3.38493689902078, + "tokens_seen": 2353070080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918989898989899, + "loss": 2.587, + "theoretical_loss": 3.3849292876601287, + "tokens_seen": 2353135616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918787878787879, + "loss": 2.5119, + "theoretical_loss": 3.384921676570807, + "tokens_seen": 2353201152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029185858585858586, + "loss": 2.3691, + "theoretical_loss": 3.384914065752798, + "tokens_seen": 2353266688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029183838383838387, + "loss": 2.6674, + "theoretical_loss": 3.3849064552060844, + "tokens_seen": 2353332224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002918181818181818, + "loss": 2.595, + "theoretical_loss": 3.3848988449306487, + "tokens_seen": 2353397760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917979797979798, + "loss": 2.4805, + "theoretical_loss": 3.3848912349264744, + "tokens_seen": 2353463296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917777777777778, + "loss": 2.5534, + "theoretical_loss": 3.384883625193543, + "tokens_seen": 2353528832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917575757575758, + "loss": 2.9125, + "theoretical_loss": 3.3848760157318387, + "tokens_seen": 2353594368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029173737373737376, + "loss": 2.5711, + "theoretical_loss": 3.3848684065413437, + "tokens_seen": 2353659904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002917171717171717, + "loss": 2.5661, + "theoretical_loss": 3.38486079762204, + "tokens_seen": 2353725440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916969696969697, + "loss": 2.6526, + "theoretical_loss": 3.3848531889739117, + "tokens_seen": 2353790976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916767676767677, + "loss": 2.5088, + "theoretical_loss": 3.3848455805969406, + "tokens_seen": 2353856512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916565656565657, + "loss": 2.5215, + "theoretical_loss": 3.3848379724911104, + "tokens_seen": 2353922048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029163636363636365, + "loss": 2.5553, + "theoretical_loss": 3.384830364656403, + "tokens_seen": 2353987584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002916161616161616, + "loss": 2.4631, + "theoretical_loss": 3.3848227570928016, + "tokens_seen": 2354053120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029159595959595957, + "loss": 2.5388, + "theoretical_loss": 3.384815149800289, + "tokens_seen": 2354118656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029157575757575763, + "loss": 2.6675, + "theoretical_loss": 3.384807542778848, + "tokens_seen": 2354184192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915555555555556, + "loss": 2.5001, + "theoretical_loss": 3.384799936028461, + "tokens_seen": 2354249728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029153535353535355, + "loss": 2.6005, + "theoretical_loss": 3.3847923295491116, + "tokens_seen": 2354315264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002915151515151515, + "loss": 2.6781, + "theoretical_loss": 3.384784723340782, + "tokens_seen": 2354380800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029149494949494946, + "loss": 2.6723, + "theoretical_loss": 3.384777117403455, + "tokens_seen": 2354446336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914747474747475, + "loss": 2.4633, + "theoretical_loss": 3.3847695117371135, + "tokens_seen": 2354511872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914545454545455, + "loss": 2.9726, + "theoretical_loss": 3.384761906341741, + "tokens_seen": 2354577408 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1327759, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5912680625915527, + "objective/train/theoretical_loss": 3.3847543012173187, + "objective/train/tokens_used": 713502176, + "theoretical_loss": 3.3847543012173187, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029143434343434344, + "loss": 2.5972, + "theoretical_loss": 3.3847543012173187, + "tokens_seen": 2354642944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002914141414141414, + "loss": 2.4997, + "theoretical_loss": 3.384746696363831, + "tokens_seen": 2354708480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029139393939393935, + "loss": 2.5865, + "theoretical_loss": 3.3847390917812596, + "tokens_seen": 2354774016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913737373737374, + "loss": 2.5449, + "theoretical_loss": 3.3847314874695877, + "tokens_seen": 2354839552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913535353535354, + "loss": 2.4667, + "theoretical_loss": 3.3847238834287987, + "tokens_seen": 2354905088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029133333333333333, + "loss": 2.7011, + "theoretical_loss": 3.384716279658875, + "tokens_seen": 2354970624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002913131313131313, + "loss": 2.6486, + "theoretical_loss": 3.3847086761597986, + "tokens_seen": 2355036160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912929292929293, + "loss": 2.4565, + "theoretical_loss": 3.3847010729315534, + "tokens_seen": 2355101696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912727272727273, + "loss": 2.5485, + "theoretical_loss": 3.384693469974122, + "tokens_seen": 2355167232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029125252525252527, + "loss": 2.6018, + "theoretical_loss": 3.3846858672874864, + "tokens_seen": 2355232768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912323232323232, + "loss": 2.5269, + "theoretical_loss": 3.384678264871631, + "tokens_seen": 2355298304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002912121212121212, + "loss": 2.5937, + "theoretical_loss": 3.384670662726537, + "tokens_seen": 2355363840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911919191919192, + "loss": 2.5792, + "theoretical_loss": 3.3846630608521884, + "tokens_seen": 2355429376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911717171717172, + "loss": 2.7147, + "theoretical_loss": 3.384655459248567, + "tokens_seen": 2355494912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029115151515151516, + "loss": 2.514, + "theoretical_loss": 3.3846478579156565, + "tokens_seen": 2355560448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002911313131313131, + "loss": 2.5482, + "theoretical_loss": 3.3846402568534395, + "tokens_seen": 2355625984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029111111111111113, + "loss": 2.6443, + "theoretical_loss": 3.3846326560618984, + "tokens_seen": 2355691520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910909090909091, + "loss": 2.6769, + "theoretical_loss": 3.384625055541017, + "tokens_seen": 2355757056 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002910707070707071, + "loss": 2.6481, + "theoretical_loss": 3.3846174552907766, + "tokens_seen": 2355822592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029105050505050505, + "loss": 2.7579, + "theoretical_loss": 3.3846098553111617, + "tokens_seen": 2355888128 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291030303030303, + "loss": 2.6744, + "theoretical_loss": 3.3846022556021538, + "tokens_seen": 2355953664 + }, + { + "epoch": 0.42, + "learning_rate": 0.000291010101010101, + "loss": 2.4761, + "theoretical_loss": 3.3845946561637366, + "tokens_seen": 2356019200 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290989898989899, + "loss": 2.5549, + "theoretical_loss": 3.3845870569958927, + "tokens_seen": 2356084736 + }, + { + "epoch": 0.42, + "learning_rate": 0.000290969696969697, + "loss": 2.7423, + "theoretical_loss": 3.384579458098605, + "tokens_seen": 2356150272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029094949494949495, + "loss": 2.6243, + "theoretical_loss": 3.3845718594718557, + "tokens_seen": 2356215808 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1329159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6554901599884033, + "objective/train/theoretical_loss": 3.3845642611156284, + "objective/train/tokens_used": 715140576, + "theoretical_loss": 3.3845642611156284, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029092929292929296, + "loss": 2.5994, + "theoretical_loss": 3.3845642611156284, + "tokens_seen": 2356281344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002909090909090909, + "loss": 2.6802, + "theoretical_loss": 3.3845566630299055, + "tokens_seen": 2356346880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029088888888888887, + "loss": 2.8462, + "theoretical_loss": 3.384549065214671, + "tokens_seen": 2356412416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908686868686869, + "loss": 2.3845, + "theoretical_loss": 3.384541467669906, + "tokens_seen": 2356477952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029084848484848484, + "loss": 2.6576, + "theoretical_loss": 3.384533870395594, + "tokens_seen": 2356543488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029082828282828285, + "loss": 2.7148, + "theoretical_loss": 3.3845262733917183, + "tokens_seen": 2356609024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002908080808080808, + "loss": 2.4904, + "theoretical_loss": 3.384518676658262, + "tokens_seen": 2356674560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029078787878787876, + "loss": 2.5257, + "theoretical_loss": 3.3845110801952063, + "tokens_seen": 2356740096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907676767676768, + "loss": 2.6002, + "theoretical_loss": 3.384503484002536, + "tokens_seen": 2356805632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907474747474748, + "loss": 2.6237, + "theoretical_loss": 3.3844958880802327, + "tokens_seen": 2356871168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029072727272727274, + "loss": 2.6998, + "theoretical_loss": 3.38448829242828, + "tokens_seen": 2356936704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002907070707070707, + "loss": 2.5187, + "theoretical_loss": 3.3844806970466603, + "tokens_seen": 2357002240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029068686868686866, + "loss": 2.6719, + "theoretical_loss": 3.384473101935357, + "tokens_seen": 2357067776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029066666666666667, + "loss": 2.5666, + "theoretical_loss": 3.384465507094352, + "tokens_seen": 2357133312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906464646464647, + "loss": 2.6901, + "theoretical_loss": 3.384457912523629, + "tokens_seen": 2357198848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029062626262626263, + "loss": 2.647, + "theoretical_loss": 3.3844503182231707, + "tokens_seen": 2357264384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002906060606060606, + "loss": 2.5895, + "theoretical_loss": 3.3844427241929598, + "tokens_seen": 2357329920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029058585858585855, + "loss": 2.859, + "theoretical_loss": 3.384435130432979, + "tokens_seen": 2357395456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905656565656566, + "loss": 2.7494, + "theoretical_loss": 3.3844275369432113, + "tokens_seen": 2357460992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029054545454545457, + "loss": 2.5824, + "theoretical_loss": 3.3844199437236404, + "tokens_seen": 2357526528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029052525252525253, + "loss": 2.728, + "theoretical_loss": 3.384412350774248, + "tokens_seen": 2357592064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002905050505050505, + "loss": 2.5033, + "theoretical_loss": 3.384404758095018, + "tokens_seen": 2357657600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904848484848485, + "loss": 2.6595, + "theoretical_loss": 3.3843971656859315, + "tokens_seen": 2357723136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904646464646465, + "loss": 2.7494, + "theoretical_loss": 3.384389573546974, + "tokens_seen": 2357788672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00029044444444444446, + "loss": 2.5222, + "theoretical_loss": 3.384381981678126, + "tokens_seen": 2357854208 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 1329871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.564392566680908, + "objective/train/theoretical_loss": 3.3843743900793717, + "objective/train/tokens_used": 716778976, + "theoretical_loss": 3.3843743900793717, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0002904242424242424, + "loss": 2.8592, + "theoretical_loss": 3.3843743900793717, + "tokens_seen": 2357919744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002904040404040404, + "loss": 2.7474, + "theoretical_loss": 3.384366798750694, + "tokens_seen": 2357985280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029038383838383844, + "loss": 2.945, + "theoretical_loss": 3.384359207692075, + "tokens_seen": 2358050816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903636363636364, + "loss": 2.5182, + "theoretical_loss": 3.384351616903498, + "tokens_seen": 2358116352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029034343434343436, + "loss": 2.6945, + "theoretical_loss": 3.3843440263849462, + "tokens_seen": 2358181888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002903232323232323, + "loss": 2.5126, + "theoretical_loss": 3.3843364361364023, + "tokens_seen": 2358247424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029030303030303027, + "loss": 2.7411, + "theoretical_loss": 3.3843288461578487, + "tokens_seen": 2358312960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029028282828282833, + "loss": 2.6637, + "theoretical_loss": 3.384321256449269, + "tokens_seen": 2358378496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902626262626263, + "loss": 2.4749, + "theoretical_loss": 3.3843136670106455, + "tokens_seen": 2358444032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029024242424242425, + "loss": 2.6405, + "theoretical_loss": 3.384306077841962, + "tokens_seen": 2358509568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902222222222222, + "loss": 2.5541, + "theoretical_loss": 3.3842984889432, + "tokens_seen": 2358575104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002902020202020202, + "loss": 2.6431, + "theoretical_loss": 3.3842909003143435, + "tokens_seen": 2358640640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029018181818181823, + "loss": 2.5677, + "theoretical_loss": 3.3842833119553752, + "tokens_seen": 2358706176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901616161616162, + "loss": 2.6305, + "theoretical_loss": 3.3842757238662777, + "tokens_seen": 2358771712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029014141414141414, + "loss": 2.659, + "theoretical_loss": 3.384268136047034, + "tokens_seen": 2358837248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901212121212121, + "loss": 2.6598, + "theoretical_loss": 3.3842605484976276, + "tokens_seen": 2358902784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002901010101010101, + "loss": 2.6775, + "theoretical_loss": 3.3842529612180408, + "tokens_seen": 2358968320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900808080808081, + "loss": 2.4984, + "theoretical_loss": 3.3842453742082563, + "tokens_seen": 2359033856 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002900606060606061, + "loss": 2.5174, + "theoretical_loss": 3.3842377874682574, + "tokens_seen": 2359099392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029004040404040403, + "loss": 2.4017, + "theoretical_loss": 3.3842302009980267, + "tokens_seen": 2359164928 + }, + { + "epoch": 0.43, + "learning_rate": 0.000290020202020202, + "loss": 2.6207, + "theoretical_loss": 3.384222614797548, + "tokens_seen": 2359230464 + }, + { + "epoch": 0.43, + "learning_rate": 0.00029, + "loss": 2.7849, + "theoretical_loss": 3.384215028866803, + "tokens_seen": 2359296000 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289979797979798, + "loss": 2.8801, + "theoretical_loss": 3.3842074432057756, + "tokens_seen": 2359361536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028995959595959597, + "loss": 2.74, + "theoretical_loss": 3.384199857814448, + "tokens_seen": 2359427072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002899393939393939, + "loss": 2.6468, + "theoretical_loss": 3.3841922726928035, + "tokens_seen": 2359492608 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1331053, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2432167530059814, + "objective/train/theoretical_loss": 3.384184687840825, + "objective/train/tokens_used": 718417376, + "theoretical_loss": 3.384184687840825, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028991919191919194, + "loss": 2.4206, + "theoretical_loss": 3.384184687840825, + "tokens_seen": 2359558144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898989898989899, + "loss": 2.4144, + "theoretical_loss": 3.3841771032584953, + "tokens_seen": 2359623680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898787878787879, + "loss": 2.7486, + "theoretical_loss": 3.3841695189457974, + "tokens_seen": 2359689216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028985858585858586, + "loss": 2.7796, + "theoretical_loss": 3.384161934902714, + "tokens_seen": 2359754752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002898383838383838, + "loss": 2.5324, + "theoretical_loss": 3.384154351129229, + "tokens_seen": 2359820288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028981818181818183, + "loss": 2.7376, + "theoretical_loss": 3.384146767625324, + "tokens_seen": 2359885824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897979797979798, + "loss": 2.5864, + "theoretical_loss": 3.3841391843909827, + "tokens_seen": 2359951360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897777777777778, + "loss": 2.6643, + "theoretical_loss": 3.384131601426188, + "tokens_seen": 2360016896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028975757575757576, + "loss": 2.7257, + "theoretical_loss": 3.384124018730922, + "tokens_seen": 2360082432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028973737373737377, + "loss": 2.598, + "theoretical_loss": 3.3841164363051686, + "tokens_seen": 2360147968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002897171717171717, + "loss": 2.8115, + "theoretical_loss": 3.384108854148911, + "tokens_seen": 2360213504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896969696969697, + "loss": 2.6195, + "theoretical_loss": 3.3841012722621313, + "tokens_seen": 2360279040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896767676767677, + "loss": 2.6485, + "theoretical_loss": 3.3840936906448125, + "tokens_seen": 2360344576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028965656565656565, + "loss": 2.5936, + "theoretical_loss": 3.3840861092969377, + "tokens_seen": 2360410112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028963636363636366, + "loss": 2.8064, + "theoretical_loss": 3.38407852821849, + "tokens_seen": 2360475648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002896161616161616, + "loss": 2.6709, + "theoretical_loss": 3.3840709474094526, + "tokens_seen": 2360541184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028959595959595957, + "loss": 2.5756, + "theoretical_loss": 3.3840633668698077, + "tokens_seen": 2360606720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895757575757576, + "loss": 2.7978, + "theoretical_loss": 3.384055786599539, + "tokens_seen": 2360672256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895555555555556, + "loss": 2.7116, + "theoretical_loss": 3.384048206598629, + "tokens_seen": 2360737792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028953535353535355, + "loss": 2.7063, + "theoretical_loss": 3.3840406268670606, + "tokens_seen": 2360803328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002895151515151515, + "loss": 2.7249, + "theoretical_loss": 3.384033047404817, + "tokens_seen": 2360868864 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028949494949494947, + "loss": 2.4109, + "theoretical_loss": 3.384025468211881, + "tokens_seen": 2360934400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894747474747475, + "loss": 2.4331, + "theoretical_loss": 3.384017889288236, + "tokens_seen": 2360999936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894545454545455, + "loss": 2.5951, + "theoretical_loss": 3.384010310633864, + "tokens_seen": 2361065472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028943434343434344, + "loss": 2.921, + "theoretical_loss": 3.384002732248749, + "tokens_seen": 2361131008 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1331743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.630728006362915, + "objective/train/theoretical_loss": 3.383995154132873, + "objective/train/tokens_used": 720055776, + "theoretical_loss": 3.383995154132873, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002894141414141414, + "loss": 2.6102, + "theoretical_loss": 3.383995154132873, + "tokens_seen": 2361196544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028939393939393936, + "loss": 2.7887, + "theoretical_loss": 3.38398757628622, + "tokens_seen": 2361262080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893737373737374, + "loss": 2.5303, + "theoretical_loss": 3.383979998708772, + "tokens_seen": 2361327616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893535353535354, + "loss": 2.7375, + "theoretical_loss": 3.3839724214005127, + "tokens_seen": 2361393152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028933333333333334, + "loss": 2.795, + "theoretical_loss": 3.3839648443614245, + "tokens_seen": 2361458688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002893131313131313, + "loss": 2.7, + "theoretical_loss": 3.383957267591491, + "tokens_seen": 2361524224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028929292929292925, + "loss": 2.669, + "theoretical_loss": 3.3839496910906943, + "tokens_seen": 2361589760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892727272727273, + "loss": 2.7168, + "theoretical_loss": 3.3839421148590185, + "tokens_seen": 2361655296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028925252525252527, + "loss": 2.5876, + "theoretical_loss": 3.3839345388964457, + "tokens_seen": 2361720832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028923232323232323, + "loss": 2.7653, + "theoretical_loss": 3.383926963202959, + "tokens_seen": 2361786368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002892121212121212, + "loss": 2.5536, + "theoretical_loss": 3.3839193877785414, + "tokens_seen": 2361851904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891919191919192, + "loss": 2.9061, + "theoretical_loss": 3.3839118126231758, + "tokens_seen": 2361917440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891717171717172, + "loss": 2.7059, + "theoretical_loss": 3.3839042377368456, + "tokens_seen": 2361982976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028915151515151517, + "loss": 2.7124, + "theoretical_loss": 3.3838966631195335, + "tokens_seen": 2362048512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891313131313131, + "loss": 2.6146, + "theoretical_loss": 3.3838890887712223, + "tokens_seen": 2362114048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002891111111111111, + "loss": 2.5341, + "theoretical_loss": 3.383881514691896, + "tokens_seen": 2362179584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890909090909091, + "loss": 2.5501, + "theoretical_loss": 3.3838739408815357, + "tokens_seen": 2362245120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002890707070707071, + "loss": 2.507, + "theoretical_loss": 3.383866367340126, + "tokens_seen": 2362310656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028905050505050506, + "loss": 2.3466, + "theoretical_loss": 3.3838587940676494, + "tokens_seen": 2362376192 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289030303030303, + "loss": 2.6251, + "theoretical_loss": 3.3838512210640888, + "tokens_seen": 2362441728 + }, + { + "epoch": 0.43, + "learning_rate": 0.000289010101010101, + "loss": 2.4988, + "theoretical_loss": 3.383843648329427, + "tokens_seen": 2362507264 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288989898989899, + "loss": 2.4288, + "theoretical_loss": 3.3838360758636474, + "tokens_seen": 2362572800 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288969696969697, + "loss": 2.4331, + "theoretical_loss": 3.383828503666733, + "tokens_seen": 2362638336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028894949494949495, + "loss": 2.7551, + "theoretical_loss": 3.3838209317386667, + "tokens_seen": 2362703872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889292929292929, + "loss": 2.5403, + "theoretical_loss": 3.383813360079431, + "tokens_seen": 2362769408 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1333010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7640528678894043, + "objective/train/theoretical_loss": 3.3838057886890094, + "objective/train/tokens_used": 721694176, + "theoretical_loss": 3.3838057886890094, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002889090909090909, + "loss": 2.6939, + "theoretical_loss": 3.3838057886890094, + "tokens_seen": 2362834944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888888888888889, + "loss": 2.6569, + "theoretical_loss": 3.383798217567385, + "tokens_seen": 2362900480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888686868686869, + "loss": 2.5571, + "theoretical_loss": 3.3837906467145404, + "tokens_seen": 2362966016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028884848484848484, + "loss": 2.3732, + "theoretical_loss": 3.383783076130459, + "tokens_seen": 2363031552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028882828282828285, + "loss": 2.7462, + "theoretical_loss": 3.383775505815124, + "tokens_seen": 2363097088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002888080808080808, + "loss": 2.7517, + "theoretical_loss": 3.3837679357685175, + "tokens_seen": 2363162624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887878787878788, + "loss": 2.6005, + "theoretical_loss": 3.3837603659906232, + "tokens_seen": 2363228160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887676767676768, + "loss": 2.4719, + "theoretical_loss": 3.383752796481424, + "tokens_seen": 2363293696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028874747474747474, + "loss": 2.4749, + "theoretical_loss": 3.3837452272409028, + "tokens_seen": 2363359232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028872727272727275, + "loss": 2.7368, + "theoretical_loss": 3.3837376582690424, + "tokens_seen": 2363424768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002887070707070707, + "loss": 2.6306, + "theoretical_loss": 3.3837300895658267, + "tokens_seen": 2363490304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886868686868687, + "loss": 2.7372, + "theoretical_loss": 3.3837225211312374, + "tokens_seen": 2363555840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028866666666666667, + "loss": 2.6388, + "theoretical_loss": 3.3837149529652586, + "tokens_seen": 2363621376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886464646464647, + "loss": 2.8235, + "theoretical_loss": 3.383707385067873, + "tokens_seen": 2363686912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028862626262626264, + "loss": 2.6592, + "theoretical_loss": 3.383699817439064, + "tokens_seen": 2363752448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002886060606060606, + "loss": 2.6099, + "theoretical_loss": 3.3836922500788136, + "tokens_seen": 2363817984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885858585858586, + "loss": 2.708, + "theoretical_loss": 3.3836846829871057, + "tokens_seen": 2363883520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028856565656565656, + "loss": 2.6726, + "theoretical_loss": 3.3836771161639225, + "tokens_seen": 2363949056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885454545454546, + "loss": 2.6458, + "theoretical_loss": 3.383669549609248, + "tokens_seen": 2364014592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028852525252525253, + "loss": 2.6351, + "theoretical_loss": 3.383661983323065, + "tokens_seen": 2364080128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002885050505050505, + "loss": 2.5166, + "theoretical_loss": 3.383654417305356, + "tokens_seen": 2364145664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884848484848485, + "loss": 2.6553, + "theoretical_loss": 3.3836468515561045, + "tokens_seen": 2364211200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884646464646465, + "loss": 2.3542, + "theoretical_loss": 3.3836392860752933, + "tokens_seen": 2364276736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028844444444444447, + "loss": 2.6269, + "theoretical_loss": 3.3836317208629056, + "tokens_seen": 2364342272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884242424242424, + "loss": 2.6966, + "theoretical_loss": 3.383624155918924, + "tokens_seen": 2364407808 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1333580, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4547007083892822, + "objective/train/theoretical_loss": 3.3836165912433325, + "objective/train/tokens_used": 723332576, + "theoretical_loss": 3.3836165912433325, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002884040404040404, + "loss": 2.5975, + "theoretical_loss": 3.3836165912433325, + "tokens_seen": 2364473344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883838383838384, + "loss": 2.5716, + "theoretical_loss": 3.3836090268361136, + "tokens_seen": 2364538880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883636363636364, + "loss": 2.7567, + "theoretical_loss": 3.38360146269725, + "tokens_seen": 2364604416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028834343434343436, + "loss": 2.6352, + "theoretical_loss": 3.383593898826725, + "tokens_seen": 2364669952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883232323232323, + "loss": 2.4256, + "theoretical_loss": 3.3835863352245217, + "tokens_seen": 2364735488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002883030303030303, + "loss": 2.5112, + "theoretical_loss": 3.383578771890623, + "tokens_seen": 2364801024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028828282828282834, + "loss": 2.3881, + "theoretical_loss": 3.3835712088250123, + "tokens_seen": 2364866560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882626262626263, + "loss": 2.6387, + "theoretical_loss": 3.383563646027672, + "tokens_seen": 2364932096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028824242424242425, + "loss": 2.6682, + "theoretical_loss": 3.383556083498586, + "tokens_seen": 2364997632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002882222222222222, + "loss": 2.4543, + "theoretical_loss": 3.3835485212377368, + "tokens_seen": 2365063168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028820202020202017, + "loss": 2.4292, + "theoretical_loss": 3.3835409592451073, + "tokens_seen": 2365128704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028818181818181823, + "loss": 2.6991, + "theoretical_loss": 3.383533397520681, + "tokens_seen": 2365194240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881616161616162, + "loss": 2.6921, + "theoretical_loss": 3.383525836064441, + "tokens_seen": 2365259776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028814141414141415, + "loss": 2.5551, + "theoretical_loss": 3.38351827487637, + "tokens_seen": 2365325312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002881212121212121, + "loss": 2.6294, + "theoretical_loss": 3.383510713956451, + "tokens_seen": 2365390848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028810101010101006, + "loss": 2.6882, + "theoretical_loss": 3.3835031533046678, + "tokens_seen": 2365456384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880808080808081, + "loss": 2.6836, + "theoretical_loss": 3.3834955929210024, + "tokens_seen": 2365521920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002880606060606061, + "loss": 2.6534, + "theoretical_loss": 3.3834880328054386, + "tokens_seen": 2365587456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028804040404040404, + "loss": 2.5824, + "theoretical_loss": 3.383480472957959, + "tokens_seen": 2365652992 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288020202020202, + "loss": 2.4562, + "theoretical_loss": 3.3834729133785473, + "tokens_seen": 2365718528 + }, + { + "epoch": 0.43, + "learning_rate": 0.000288, + "loss": 2.5657, + "theoretical_loss": 3.383465354067186, + "tokens_seen": 2365784064 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287979797979798, + "loss": 2.8811, + "theoretical_loss": 3.3834577950238582, + "tokens_seen": 2365849600 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287959595959596, + "loss": 2.5979, + "theoretical_loss": 3.3834502362485472, + "tokens_seen": 2365915136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028793939393939393, + "loss": 2.4926, + "theoretical_loss": 3.383442677741236, + "tokens_seen": 2365980672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002879191919191919, + "loss": 2.8622, + "theoretical_loss": 3.383435119501908, + "tokens_seen": 2366046208 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1334672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.595120668411255, + "objective/train/theoretical_loss": 3.383427561530546, + "objective/train/tokens_used": 724970976, + "theoretical_loss": 3.383427561530546, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878989898989899, + "loss": 2.7888, + "theoretical_loss": 3.383427561530546, + "tokens_seen": 2366111744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878787878787879, + "loss": 2.6816, + "theoretical_loss": 3.3834200038271325, + "tokens_seen": 2366177280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028785858585858587, + "loss": 2.6647, + "theoretical_loss": 3.3834124463916515, + "tokens_seen": 2366242816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002878383838383838, + "loss": 2.7434, + "theoretical_loss": 3.383404889224085, + "tokens_seen": 2366308352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028781818181818184, + "loss": 2.618, + "theoretical_loss": 3.3833973323244178, + "tokens_seen": 2366373888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877979797979798, + "loss": 2.9136, + "theoretical_loss": 3.383389775692631, + "tokens_seen": 2366439424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877777777777778, + "loss": 2.4533, + "theoretical_loss": 3.3833822193287095, + "tokens_seen": 2366504960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028775757575757576, + "loss": 2.5231, + "theoretical_loss": 3.383374663232635, + "tokens_seen": 2366570496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002877373737373737, + "loss": 2.6097, + "theoretical_loss": 3.3833671074043914, + "tokens_seen": 2366636032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028771717171717173, + "loss": 2.8139, + "theoretical_loss": 3.383359551843961, + "tokens_seen": 2366701568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876969696969697, + "loss": 2.7426, + "theoretical_loss": 3.3833519965513283, + "tokens_seen": 2366767104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876767676767677, + "loss": 2.6497, + "theoretical_loss": 3.383344441526475, + "tokens_seen": 2366832640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028765656565656565, + "loss": 2.525, + "theoretical_loss": 3.3833368867693845, + "tokens_seen": 2366898176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028763636363636366, + "loss": 2.6037, + "theoretical_loss": 3.38332933228004, + "tokens_seen": 2366963712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002876161616161616, + "loss": 2.5639, + "theoretical_loss": 3.3833217780584253, + "tokens_seen": 2367029248 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875959595959596, + "loss": 2.407, + "theoretical_loss": 3.3833142241045224, + "tokens_seen": 2367094784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875757575757576, + "loss": 2.5668, + "theoretical_loss": 3.3833066704183152, + "tokens_seen": 2367160320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028755555555555555, + "loss": 2.5001, + "theoretical_loss": 3.3832991169997864, + "tokens_seen": 2367225856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028753535353535356, + "loss": 2.742, + "theoretical_loss": 3.383291563848919, + "tokens_seen": 2367291392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002875151515151515, + "loss": 2.4955, + "theoretical_loss": 3.3832840109656965, + "tokens_seen": 2367356928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028749494949494947, + "loss": 2.6003, + "theoretical_loss": 3.383276458350102, + "tokens_seen": 2367422464 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874747474747475, + "loss": 2.661, + "theoretical_loss": 3.383268906002118, + "tokens_seen": 2367488000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874545454545455, + "loss": 2.498, + "theoretical_loss": 3.383261353921728, + "tokens_seen": 2367553536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028743434343434345, + "loss": 2.7659, + "theoretical_loss": 3.383253802108915, + "tokens_seen": 2367619072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002874141414141414, + "loss": 2.8517, + "theoretical_loss": 3.383246250563663, + "tokens_seen": 2367684608 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1335172, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.526048183441162, + "objective/train/theoretical_loss": 3.383238699285954, + "objective/train/tokens_used": 726609376, + "theoretical_loss": 3.383238699285954, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028739393939393936, + "loss": 2.6943, + "theoretical_loss": 3.383238699285954, + "tokens_seen": 2367750144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873737373737374, + "loss": 2.6169, + "theoretical_loss": 3.383231148275771, + "tokens_seen": 2367815680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873535353535354, + "loss": 2.7908, + "theoretical_loss": 3.383223597533098, + "tokens_seen": 2367881216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028733333333333334, + "loss": 2.5811, + "theoretical_loss": 3.383216047057918, + "tokens_seen": 2367946752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002873131313131313, + "loss": 2.7512, + "theoretical_loss": 3.383208496850213, + "tokens_seen": 2368012288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028729292929292926, + "loss": 2.3928, + "theoretical_loss": 3.3832009469099678, + "tokens_seen": 2368077824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872727272727273, + "loss": 2.65, + "theoretical_loss": 3.3831933972371644, + "tokens_seen": 2368143360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872525252525253, + "loss": 2.6437, + "theoretical_loss": 3.383185847831786, + "tokens_seen": 2368208896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028723232323232324, + "loss": 2.6425, + "theoretical_loss": 3.3831782986938164, + "tokens_seen": 2368274432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002872121212121212, + "loss": 2.7894, + "theoretical_loss": 3.383170749823238, + "tokens_seen": 2368339968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028719191919191915, + "loss": 2.6946, + "theoretical_loss": 3.383163201220034, + "tokens_seen": 2368405504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871717171717172, + "loss": 2.641, + "theoretical_loss": 3.3831556528841884, + "tokens_seen": 2368471040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028715151515151517, + "loss": 2.7502, + "theoretical_loss": 3.383148104815683, + "tokens_seen": 2368536576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028713131313131313, + "loss": 2.649, + "theoretical_loss": 3.383140557014502, + "tokens_seen": 2368602112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002871111111111111, + "loss": 2.6079, + "theoretical_loss": 3.383133009480628, + "tokens_seen": 2368667648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870909090909091, + "loss": 2.424, + "theoretical_loss": 3.383125462214044, + "tokens_seen": 2368733184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002870707070707071, + "loss": 2.6845, + "theoretical_loss": 3.3831179152147337, + "tokens_seen": 2368798720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028705050505050506, + "loss": 2.6333, + "theoretical_loss": 3.3831103684826798, + "tokens_seen": 2368864256 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287030303030303, + "loss": 2.5584, + "theoretical_loss": 3.383102822017866, + "tokens_seen": 2368929792 + }, + { + "epoch": 0.43, + "learning_rate": 0.000287010101010101, + "loss": 2.7274, + "theoretical_loss": 3.3830952758202746, + "tokens_seen": 2368995328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028698989898989904, + "loss": 2.725, + "theoretical_loss": 3.383087729889889, + "tokens_seen": 2369060864 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286969696969697, + "loss": 2.6992, + "theoretical_loss": 3.383080184226693, + "tokens_seen": 2369126400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028694949494949496, + "loss": 2.5341, + "theoretical_loss": 3.383072638830669, + "tokens_seen": 2369191936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869292929292929, + "loss": 2.563, + "theoretical_loss": 3.3830650937018008, + "tokens_seen": 2369257472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002869090909090909, + "loss": 2.4638, + "theoretical_loss": 3.3830575488400707, + "tokens_seen": 2369323008 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1336475, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4415109157562256, + "objective/train/theoretical_loss": 3.3830500042454625, + "objective/train/tokens_used": 728247776, + "theoretical_loss": 3.3830500042454625, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028688888888888894, + "loss": 2.494, + "theoretical_loss": 3.3830500042454625, + "tokens_seen": 2369388544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868686868686869, + "loss": 2.7024, + "theoretical_loss": 3.3830424599179594, + "tokens_seen": 2369454080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028684848484848485, + "loss": 2.671, + "theoretical_loss": 3.383034915857544, + "tokens_seen": 2369519616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868282828282828, + "loss": 2.6477, + "theoretical_loss": 3.3830273720642, + "tokens_seen": 2369585152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002868080808080808, + "loss": 2.5131, + "theoretical_loss": 3.3830198285379103, + "tokens_seen": 2369650688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028678787878787883, + "loss": 2.7548, + "theoretical_loss": 3.3830122852786584, + "tokens_seen": 2369716224 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867676767676768, + "loss": 2.634, + "theoretical_loss": 3.3830047422864267, + "tokens_seen": 2369781760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028674747474747474, + "loss": 2.4429, + "theoretical_loss": 3.382997199561199, + "tokens_seen": 2369847296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867272727272727, + "loss": 2.3356, + "theoretical_loss": 3.382989657102959, + "tokens_seen": 2369912832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002867070707070707, + "loss": 2.7688, + "theoretical_loss": 3.3829821149116883, + "tokens_seen": 2369978368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866868686868687, + "loss": 2.7545, + "theoretical_loss": 3.3829745729873713, + "tokens_seen": 2370043904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866666666666667, + "loss": 2.5716, + "theoretical_loss": 3.382967031329991, + "tokens_seen": 2370109440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028664646464646463, + "loss": 2.5847, + "theoretical_loss": 3.38295948993953, + "tokens_seen": 2370174976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028662626262626265, + "loss": 2.5743, + "theoretical_loss": 3.382951948815972, + "tokens_seen": 2370240512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002866060606060606, + "loss": 2.5629, + "theoretical_loss": 3.3829444079593003, + "tokens_seen": 2370306048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865858585858586, + "loss": 2.4643, + "theoretical_loss": 3.3829368673694975, + "tokens_seen": 2370371584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028656565656565657, + "loss": 2.706, + "theoretical_loss": 3.382929327046547, + "tokens_seen": 2370437120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028654545454545453, + "loss": 2.5235, + "theoretical_loss": 3.3829217869904324, + "tokens_seen": 2370502656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028652525252525254, + "loss": 2.6642, + "theoretical_loss": 3.3829142472011364, + "tokens_seen": 2370568192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002865050505050505, + "loss": 2.7696, + "theoretical_loss": 3.382906707678642, + "tokens_seen": 2370633728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864848484848485, + "loss": 2.6744, + "theoretical_loss": 3.382899168422933, + "tokens_seen": 2370699264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028646464646464646, + "loss": 2.6552, + "theoretical_loss": 3.382891629433993, + "tokens_seen": 2370764800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864444444444445, + "loss": 2.6204, + "theoretical_loss": 3.382884090711803, + "tokens_seen": 2370830336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028642424242424243, + "loss": 2.5095, + "theoretical_loss": 3.3828765522563486, + "tokens_seen": 2370895872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002864040404040404, + "loss": 2.7346, + "theoretical_loss": 3.3828690140676123, + "tokens_seen": 2370961408 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1336773, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8766262531280518, + "objective/train/theoretical_loss": 3.382861476145577, + "objective/train/tokens_used": 729886176, + "theoretical_loss": 3.382861476145577, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863838383838384, + "loss": 2.4819, + "theoretical_loss": 3.382861476145577, + "tokens_seen": 2371026944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028636363636363636, + "loss": 2.7626, + "theoretical_loss": 3.3828539384902254, + "tokens_seen": 2371092480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028634343434343437, + "loss": 2.663, + "theoretical_loss": 3.3828464011015416, + "tokens_seen": 2371158016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863232323232323, + "loss": 2.3645, + "theoretical_loss": 3.3828388639795084, + "tokens_seen": 2371223552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002863030303030303, + "loss": 2.8217, + "theoretical_loss": 3.382831327124109, + "tokens_seen": 2371289088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862828282828283, + "loss": 2.7529, + "theoretical_loss": 3.3828237905353267, + "tokens_seen": 2371354624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862626262626263, + "loss": 2.7941, + "theoretical_loss": 3.382816254213145, + "tokens_seen": 2371420160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028624242424242426, + "loss": 2.4945, + "theoretical_loss": 3.3828087181575457, + "tokens_seen": 2371485696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862222222222222, + "loss": 2.8856, + "theoretical_loss": 3.382801182368514, + "tokens_seen": 2371551232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002862020202020202, + "loss": 2.6388, + "theoretical_loss": 3.382793646846032, + "tokens_seen": 2371616768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861818181818182, + "loss": 2.7119, + "theoretical_loss": 3.3827861115900824, + "tokens_seen": 2371682304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861616161616162, + "loss": 2.5793, + "theoretical_loss": 3.3827785766006495, + "tokens_seen": 2371747840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028614141414141415, + "loss": 2.4514, + "theoretical_loss": 3.3827710418777164, + "tokens_seen": 2371813376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002861212121212121, + "loss": 2.8599, + "theoretical_loss": 3.3827635074212656, + "tokens_seen": 2371878912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028610101010101007, + "loss": 2.5846, + "theoretical_loss": 3.382755973231281, + "tokens_seen": 2371944448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028608080808080813, + "loss": 2.8522, + "theoretical_loss": 3.382748439307745, + "tokens_seen": 2372009984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002860606060606061, + "loss": 2.5963, + "theoretical_loss": 3.3827409056506417, + "tokens_seen": 2372075520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028604040404040404, + "loss": 2.5024, + "theoretical_loss": 3.382733372259954, + "tokens_seen": 2372141056 + }, + { + "epoch": 0.43, + "learning_rate": 0.000286020202020202, + "loss": 2.687, + "theoretical_loss": 3.3827258391356647, + "tokens_seen": 2372206592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028599999999999996, + "loss": 2.8025, + "theoretical_loss": 3.3827183062777575, + "tokens_seen": 2372272128 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285979797979798, + "loss": 2.6644, + "theoretical_loss": 3.382710773686216, + "tokens_seen": 2372337664 + }, + { + "epoch": 0.43, + "learning_rate": 0.000285959595959596, + "loss": 2.5193, + "theoretical_loss": 3.3827032413610225, + "tokens_seen": 2372403200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028593939393939394, + "loss": 2.7166, + "theoretical_loss": 3.3826957093021606, + "tokens_seen": 2372468736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002859191919191919, + "loss": 2.342, + "theoretical_loss": 3.3826881775096136, + "tokens_seen": 2372534272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858989898989899, + "loss": 2.6417, + "theoretical_loss": 3.3826806459833647, + "tokens_seen": 2372599808 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1338329, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7846832275390625, + "objective/train/theoretical_loss": 3.3826731147233975, + "objective/train/tokens_used": 731524576, + "theoretical_loss": 3.3826731147233975, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858787878787879, + "loss": 2.7451, + "theoretical_loss": 3.3826731147233975, + "tokens_seen": 2372665344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858585858585859, + "loss": 2.5056, + "theoretical_loss": 3.3826655837296946, + "tokens_seen": 2372730880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028583838383838383, + "loss": 2.7293, + "theoretical_loss": 3.382658053002239, + "tokens_seen": 2372796416 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002858181818181818, + "loss": 2.5888, + "theoretical_loss": 3.3826505225410153, + "tokens_seen": 2372861952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857979797979798, + "loss": 2.6055, + "theoretical_loss": 3.3826429923460055, + "tokens_seen": 2372927488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857777777777778, + "loss": 2.422, + "theoretical_loss": 3.382635462417193, + "tokens_seen": 2372993024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028575757575757577, + "loss": 2.5993, + "theoretical_loss": 3.382627932754562, + "tokens_seen": 2373058560 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002857373737373737, + "loss": 2.6244, + "theoretical_loss": 3.3826204033580938, + "tokens_seen": 2373124096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028571717171717173, + "loss": 2.7692, + "theoretical_loss": 3.3826128742277737, + "tokens_seen": 2373189632 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856969696969697, + "loss": 2.7395, + "theoretical_loss": 3.382605345363584, + "tokens_seen": 2373255168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856767676767677, + "loss": 2.5881, + "theoretical_loss": 3.3825978167655077, + "tokens_seen": 2373320704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028565656565656566, + "loss": 2.9856, + "theoretical_loss": 3.382590288433528, + "tokens_seen": 2373386240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856363636363636, + "loss": 2.5799, + "theoretical_loss": 3.3825827603676295, + "tokens_seen": 2373451776 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002856161616161616, + "loss": 2.6056, + "theoretical_loss": 3.3825752325677936, + "tokens_seen": 2373517312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855959595959596, + "loss": 2.5279, + "theoretical_loss": 3.382567705034005, + "tokens_seen": 2373582848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855757575757576, + "loss": 2.7878, + "theoretical_loss": 3.382560177766246, + "tokens_seen": 2373648384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028555555555555555, + "loss": 2.7268, + "theoretical_loss": 3.3825526507645005, + "tokens_seen": 2373713920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028553535353535356, + "loss": 2.569, + "theoretical_loss": 3.382545124028751, + "tokens_seen": 2373779456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002855151515151515, + "loss": 2.6998, + "theoretical_loss": 3.3825375975589815, + "tokens_seen": 2373844992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854949494949495, + "loss": 2.6353, + "theoretical_loss": 3.382530071355175, + "tokens_seen": 2373910528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854747474747475, + "loss": 2.7844, + "theoretical_loss": 3.3825225454173147, + "tokens_seen": 2373976064 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028545454545454544, + "loss": 2.512, + "theoretical_loss": 3.382515019745384, + "tokens_seen": 2374041600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028543434343434346, + "loss": 2.7988, + "theoretical_loss": 3.3825074943393663, + "tokens_seen": 2374107136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002854141414141414, + "loss": 2.6382, + "theoretical_loss": 3.382499969199244, + "tokens_seen": 2374172672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00028539393939393937, + "loss": 2.8049, + "theoretical_loss": 3.3824924443250013, + "tokens_seen": 2374238208 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 1338943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8074584007263184, + "objective/train/theoretical_loss": 3.3824849197166214, + "objective/train/tokens_used": 733162976, + "theoretical_loss": 3.3824849197166214, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0002853737373737374, + "loss": 2.6096, + "theoretical_loss": 3.3824849197166214, + "tokens_seen": 2374303744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853535353535354, + "loss": 2.7635, + "theoretical_loss": 3.382477395374087, + "tokens_seen": 2374369280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028533333333333335, + "loss": 2.4954, + "theoretical_loss": 3.3824698712973817, + "tokens_seen": 2374434816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002853131313131313, + "loss": 2.4033, + "theoretical_loss": 3.3824623474864888, + "tokens_seen": 2374500352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852929292929293, + "loss": 2.6762, + "theoretical_loss": 3.3824548239413916, + "tokens_seen": 2374565888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028527272727272727, + "loss": 2.7028, + "theoretical_loss": 3.3824473006620734, + "tokens_seen": 2374631424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852525252525253, + "loss": 2.6912, + "theoretical_loss": 3.3824397776485173, + "tokens_seen": 2374696960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028523232323232324, + "loss": 2.6387, + "theoretical_loss": 3.382432254900707, + "tokens_seen": 2374762496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002852121212121212, + "loss": 2.7545, + "theoretical_loss": 3.382424732418625, + "tokens_seen": 2374828032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851919191919192, + "loss": 2.5495, + "theoretical_loss": 3.382417210202255, + "tokens_seen": 2374893568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851717171717172, + "loss": 2.6781, + "theoretical_loss": 3.3824096882515806, + "tokens_seen": 2374959104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851515151515152, + "loss": 2.7711, + "theoretical_loss": 3.382402166566585, + "tokens_seen": 2375024640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028513131313131313, + "loss": 2.6937, + "theoretical_loss": 3.382394645147251, + "tokens_seen": 2375090176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002851111111111111, + "loss": 2.4823, + "theoretical_loss": 3.382387123993562, + "tokens_seen": 2375155712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850909090909091, + "loss": 2.7585, + "theoretical_loss": 3.382379603105502, + "tokens_seen": 2375221248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002850707070707071, + "loss": 2.7415, + "theoretical_loss": 3.382372082483053, + "tokens_seen": 2375286784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028505050505050507, + "loss": 2.5106, + "theoretical_loss": 3.3823645621261997, + "tokens_seen": 2375352320 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285030303030303, + "loss": 2.7746, + "theoretical_loss": 3.3823570420349243, + "tokens_seen": 2375417856 + }, + { + "epoch": 0.44, + "learning_rate": 0.000285010101010101, + "loss": 2.5345, + "theoretical_loss": 3.3823495222092106, + "tokens_seen": 2375483392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028498989898989905, + "loss": 2.5362, + "theoretical_loss": 3.3823420026490423, + "tokens_seen": 2375548928 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284969696969697, + "loss": 2.2615, + "theoretical_loss": 3.3823344833544016, + "tokens_seen": 2375614464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028494949494949496, + "loss": 2.4126, + "theoretical_loss": 3.382326964325273, + "tokens_seen": 2375680000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849292929292929, + "loss": 2.5093, + "theoretical_loss": 3.3823194455616385, + "tokens_seen": 2375745536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002849090909090909, + "loss": 2.6896, + "theoretical_loss": 3.3823119270634825, + "tokens_seen": 2375811072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028488888888888894, + "loss": 2.7733, + "theoretical_loss": 3.3823044088307883, + "tokens_seen": 2375876608 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1339545, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4100427627563477, + "objective/train/theoretical_loss": 3.3822968908635382, + "objective/train/tokens_used": 734801376, + "theoretical_loss": 3.3822968908635382, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848686868686869, + "loss": 2.5206, + "theoretical_loss": 3.3822968908635382, + "tokens_seen": 2375942144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028484848484848485, + "loss": 2.6557, + "theoretical_loss": 3.3822893731617167, + "tokens_seen": 2376007680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002848282828282828, + "loss": 2.8389, + "theoretical_loss": 3.382281855725306, + "tokens_seen": 2376073216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028480808080808077, + "loss": 2.6174, + "theoretical_loss": 3.38227433855429, + "tokens_seen": 2376138752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028478787878787883, + "loss": 2.4613, + "theoretical_loss": 3.3822668216486527, + "tokens_seen": 2376204288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847676767676768, + "loss": 2.522, + "theoretical_loss": 3.3822593050083762, + "tokens_seen": 2376269824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028474747474747475, + "loss": 2.778, + "theoretical_loss": 3.382251788633444, + "tokens_seen": 2376335360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847272727272727, + "loss": 2.6014, + "theoretical_loss": 3.38224427252384, + "tokens_seen": 2376400896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002847070707070707, + "loss": 2.6706, + "theoretical_loss": 3.3822367566795473, + "tokens_seen": 2376466432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846868686868687, + "loss": 2.532, + "theoretical_loss": 3.382229241100549, + "tokens_seen": 2376531968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846666666666667, + "loss": 2.7145, + "theoretical_loss": 3.382221725786829, + "tokens_seen": 2376597504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028464646464646464, + "loss": 2.7233, + "theoretical_loss": 3.3822142107383697, + "tokens_seen": 2376663040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846262626262626, + "loss": 2.7088, + "theoretical_loss": 3.382206695955155, + "tokens_seen": 2376728576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002846060606060606, + "loss": 2.7441, + "theoretical_loss": 3.3821991814371684, + "tokens_seen": 2376794112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845858585858586, + "loss": 2.4408, + "theoretical_loss": 3.3821916671843923, + "tokens_seen": 2376859648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845656565656566, + "loss": 2.4441, + "theoretical_loss": 3.3821841531968113, + "tokens_seen": 2376925184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028454545454545453, + "loss": 2.4608, + "theoretical_loss": 3.382176639474408, + "tokens_seen": 2376990720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028452525252525254, + "loss": 2.5849, + "theoretical_loss": 3.382169126017166, + "tokens_seen": 2377056256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002845050505050505, + "loss": 2.4665, + "theoretical_loss": 3.382161612825068, + "tokens_seen": 2377121792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844848484848485, + "loss": 2.7051, + "theoretical_loss": 3.3821540998980986, + "tokens_seen": 2377187328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028446464646464647, + "loss": 2.6452, + "theoretical_loss": 3.3821465872362397, + "tokens_seen": 2377252864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844444444444444, + "loss": 2.6103, + "theoretical_loss": 3.3821390748394755, + "tokens_seen": 2377318400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028442424242424244, + "loss": 2.7096, + "theoretical_loss": 3.382131562707789, + "tokens_seen": 2377383936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002844040404040404, + "loss": 2.6147, + "theoretical_loss": 3.382124050841164, + "tokens_seen": 2377449472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843838383838384, + "loss": 2.3126, + "theoretical_loss": 3.382116539239583, + "tokens_seen": 2377515008 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1340666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8909237384796143, + "objective/train/theoretical_loss": 3.38210902790303, + "objective/train/tokens_used": 736439776, + "theoretical_loss": 3.38210902790303, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028436363636363636, + "loss": 2.7469, + "theoretical_loss": 3.38210902790303, + "tokens_seen": 2377580544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028434343434343437, + "loss": 2.7506, + "theoretical_loss": 3.3821015168314883, + "tokens_seen": 2377646080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028432323232323233, + "loss": 2.6125, + "theoretical_loss": 3.3820940060249414, + "tokens_seen": 2377711616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002843030303030303, + "loss": 2.6742, + "theoretical_loss": 3.382086495483372, + "tokens_seen": 2377777152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842828282828283, + "loss": 2.7199, + "theoretical_loss": 3.382078985206764, + "tokens_seen": 2377842688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028426262626262625, + "loss": 2.8026, + "theoretical_loss": 3.3820714751951004, + "tokens_seen": 2377908224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028424242424242427, + "loss": 2.5453, + "theoretical_loss": 3.3820639654483653, + "tokens_seen": 2377973760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842222222222222, + "loss": 2.7569, + "theoretical_loss": 3.382056455966541, + "tokens_seen": 2378039296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002842020202020202, + "loss": 2.6902, + "theoretical_loss": 3.3820489467496113, + "tokens_seen": 2378104832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841818181818182, + "loss": 2.5194, + "theoretical_loss": 3.38204143779756, + "tokens_seen": 2378170368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841616161616162, + "loss": 2.6451, + "theoretical_loss": 3.38203392911037, + "tokens_seen": 2378235904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028414141414141416, + "loss": 2.7505, + "theoretical_loss": 3.382026420688024, + "tokens_seen": 2378301440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002841212121212121, + "loss": 2.5913, + "theoretical_loss": 3.3820189125305067, + "tokens_seen": 2378366976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028410101010101007, + "loss": 2.5304, + "theoretical_loss": 3.382011404637801, + "tokens_seen": 2378432512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840808080808081, + "loss": 2.5318, + "theoretical_loss": 3.38200389700989, + "tokens_seen": 2378498048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002840606060606061, + "loss": 2.6922, + "theoretical_loss": 3.381996389646757, + "tokens_seen": 2378563584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028404040404040405, + "loss": 2.6977, + "theoretical_loss": 3.3819888825483853, + "tokens_seen": 2378629120 + }, + { + "epoch": 0.44, + "learning_rate": 0.000284020202020202, + "loss": 2.6107, + "theoretical_loss": 3.381981375714759, + "tokens_seen": 2378694656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028399999999999996, + "loss": 2.676, + "theoretical_loss": 3.3819738691458605, + "tokens_seen": 2378760192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028397979797979803, + "loss": 2.675, + "theoretical_loss": 3.3819663628416743, + "tokens_seen": 2378825728 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283959595959596, + "loss": 2.569, + "theoretical_loss": 3.3819588568021826, + "tokens_seen": 2378891264 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028393939393939394, + "loss": 2.776, + "theoretical_loss": 3.3819513510273698, + "tokens_seen": 2378956800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002839191919191919, + "loss": 2.6587, + "theoretical_loss": 3.381943845517218, + "tokens_seen": 2379022336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028389898989898986, + "loss": 2.5299, + "theoretical_loss": 3.3819363402717117, + "tokens_seen": 2379087872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838787878787879, + "loss": 2.6231, + "theoretical_loss": 3.381928835290834, + "tokens_seen": 2379153408 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1341274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6137118339538574, + "objective/train/theoretical_loss": 3.3819213305745683, + "objective/train/tokens_used": 738078176, + "theoretical_loss": 3.3819213305745683, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838585858585859, + "loss": 2.5341, + "theoretical_loss": 3.3819213305745683, + "tokens_seen": 2379218944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028383838383838384, + "loss": 2.4865, + "theoretical_loss": 3.381913826122898, + "tokens_seen": 2379284480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002838181818181818, + "loss": 2.7311, + "theoretical_loss": 3.3819063219358063, + "tokens_seen": 2379350016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837979797979798, + "loss": 2.7818, + "theoretical_loss": 3.3818988180132763, + "tokens_seen": 2379415552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837777777777778, + "loss": 2.6341, + "theoretical_loss": 3.381891314355292, + "tokens_seen": 2379481088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028375757575757577, + "loss": 2.6738, + "theoretical_loss": 3.3818838109618365, + "tokens_seen": 2379546624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028373737373737373, + "loss": 2.587, + "theoretical_loss": 3.3818763078328935, + "tokens_seen": 2379612160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002837171717171717, + "loss": 2.8507, + "theoretical_loss": 3.3818688049684456, + "tokens_seen": 2379677696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836969696969697, + "loss": 2.5939, + "theoretical_loss": 3.381861302368477, + "tokens_seen": 2379743232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836767676767677, + "loss": 2.7202, + "theoretical_loss": 3.381853800032971, + "tokens_seen": 2379808768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028365656565656566, + "loss": 2.7032, + "theoretical_loss": 3.3818462979619106, + "tokens_seen": 2379874304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002836363636363636, + "loss": 2.7006, + "theoretical_loss": 3.381838796155279, + "tokens_seen": 2379939840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028361616161616163, + "loss": 2.5243, + "theoretical_loss": 3.3818312946130606, + "tokens_seen": 2380005376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028359595959595964, + "loss": 2.5968, + "theoretical_loss": 3.381823793335238, + "tokens_seen": 2380070912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835757575757576, + "loss": 2.6744, + "theoretical_loss": 3.381816292321795, + "tokens_seen": 2380136448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028355555555555556, + "loss": 2.8344, + "theoretical_loss": 3.3818087915727144, + "tokens_seen": 2380201984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835353535353535, + "loss": 2.6277, + "theoretical_loss": 3.3818012910879798, + "tokens_seen": 2380267520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002835151515151515, + "loss": 2.7584, + "theoretical_loss": 3.3817937908675755, + "tokens_seen": 2380333056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028349494949494954, + "loss": 2.6666, + "theoretical_loss": 3.3817862909114837, + "tokens_seen": 2380398592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834747474747475, + "loss": 2.8667, + "theoretical_loss": 3.3817787912196886, + "tokens_seen": 2380464128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028345454545454545, + "loss": 2.7041, + "theoretical_loss": 3.381771291792173, + "tokens_seen": 2380529664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834343434343434, + "loss": 2.5067, + "theoretical_loss": 3.381763792628921, + "tokens_seen": 2380595200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002834141414141414, + "loss": 2.7781, + "theoretical_loss": 3.3817562937299153, + "tokens_seen": 2380660736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028339393939393943, + "loss": 2.7537, + "theoretical_loss": 3.38174879509514, + "tokens_seen": 2380726272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833737373737374, + "loss": 2.6713, + "theoretical_loss": 3.381741296724578, + "tokens_seen": 2380791808 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1342636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.429597854614258, + "objective/train/theoretical_loss": 3.381733798618213, + "objective/train/tokens_used": 739716576, + "theoretical_loss": 3.381733798618213, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028335353535353534, + "loss": 2.4813, + "theoretical_loss": 3.381733798618213, + "tokens_seen": 2380857344 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028333333333333335, + "loss": 2.6592, + "theoretical_loss": 3.3817263007760285, + "tokens_seen": 2380922880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002833131313131313, + "loss": 2.6638, + "theoretical_loss": 3.3817188031980074, + "tokens_seen": 2380988416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832929292929293, + "loss": 2.5007, + "theoretical_loss": 3.3817113058841337, + "tokens_seen": 2381053952 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832727272727273, + "loss": 2.5037, + "theoretical_loss": 3.3817038088343905, + "tokens_seen": 2381119488 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028325252525252524, + "loss": 2.5039, + "theoretical_loss": 3.3816963120487613, + "tokens_seen": 2381185024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028323232323232325, + "loss": 2.5619, + "theoretical_loss": 3.3816888155272298, + "tokens_seen": 2381250560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002832121212121212, + "loss": 2.6469, + "theoretical_loss": 3.381681319269779, + "tokens_seen": 2381316096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831919191919192, + "loss": 2.4035, + "theoretical_loss": 3.381673823276392, + "tokens_seen": 2381381632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028317171717171717, + "loss": 2.3633, + "theoretical_loss": 3.3816663275470535, + "tokens_seen": 2381447168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831515151515152, + "loss": 2.6394, + "theoretical_loss": 3.3816588320817456, + "tokens_seen": 2381512704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028313131313131314, + "loss": 2.6701, + "theoretical_loss": 3.381651336880453, + "tokens_seen": 2381578240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002831111111111111, + "loss": 2.4469, + "theoretical_loss": 3.381643841943158, + "tokens_seen": 2381643776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830909090909091, + "loss": 2.6025, + "theoretical_loss": 3.3816363472698443, + "tokens_seen": 2381709312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028307070707070706, + "loss": 2.7981, + "theoretical_loss": 3.3816288528604956, + "tokens_seen": 2381774848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002830505050505051, + "loss": 2.7202, + "theoretical_loss": 3.3816213587150954, + "tokens_seen": 2381840384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028303030303030303, + "loss": 2.6423, + "theoretical_loss": 3.3816138648336267, + "tokens_seen": 2381905920 + }, + { + "epoch": 0.44, + "learning_rate": 0.000283010101010101, + "loss": 2.6257, + "theoretical_loss": 3.3816063712160735, + "tokens_seen": 2381971456 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282989898989899, + "loss": 2.3863, + "theoretical_loss": 3.381598877862419, + "tokens_seen": 2382036992 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282969696969697, + "loss": 2.5864, + "theoretical_loss": 3.381591384772647, + "tokens_seen": 2382102528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028294949494949497, + "loss": 2.6779, + "theoretical_loss": 3.38158389194674, + "tokens_seen": 2382168064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829292929292929, + "loss": 2.6215, + "theoretical_loss": 3.381576399384682, + "tokens_seen": 2382233600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002829090909090909, + "loss": 2.6309, + "theoretical_loss": 3.3815689070864567, + "tokens_seen": 2382299136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828888888888889, + "loss": 2.7794, + "theoretical_loss": 3.3815614150520474, + "tokens_seen": 2382364672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828686868686869, + "loss": 2.7342, + "theoretical_loss": 3.3815539232814373, + "tokens_seen": 2382430208 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1343061, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8310749530792236, + "objective/train/theoretical_loss": 3.3815464317746105, + "objective/train/tokens_used": 741354976, + "theoretical_loss": 3.3815464317746105, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028284848484848486, + "loss": 2.6754, + "theoretical_loss": 3.3815464317746105, + "tokens_seen": 2382495744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828282828282828, + "loss": 2.9134, + "theoretical_loss": 3.3815389405315495, + "tokens_seen": 2382561280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002828080808080808, + "loss": 2.803, + "theoretical_loss": 3.3815314495522384, + "tokens_seen": 2382626816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028278787878787884, + "loss": 2.7242, + "theoretical_loss": 3.3815239588366603, + "tokens_seen": 2382692352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827676767676768, + "loss": 2.8574, + "theoretical_loss": 3.381516468384799, + "tokens_seen": 2382757888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028274747474747475, + "loss": 2.4683, + "theoretical_loss": 3.381508978196638, + "tokens_seen": 2382823424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002827272727272727, + "loss": 2.6794, + "theoretical_loss": 3.3815014882721606, + "tokens_seen": 2382888960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028270707070707067, + "loss": 2.8465, + "theoretical_loss": 3.38149399861135, + "tokens_seen": 2382954496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028268686868686873, + "loss": 2.5972, + "theoretical_loss": 3.38148650921419, + "tokens_seen": 2383020032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826666666666667, + "loss": 2.594, + "theoretical_loss": 3.3814790200806644, + "tokens_seen": 2383085568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028264646464646465, + "loss": 2.7432, + "theoretical_loss": 3.381471531210756, + "tokens_seen": 2383151104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826262626262626, + "loss": 2.8432, + "theoretical_loss": 3.3814640426044487, + "tokens_seen": 2383216640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002826060606060606, + "loss": 2.528, + "theoretical_loss": 3.3814565542617254, + "tokens_seen": 2383282176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825858585858586, + "loss": 2.8575, + "theoretical_loss": 3.3814490661825705, + "tokens_seen": 2383347712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825656565656566, + "loss": 2.6797, + "theoretical_loss": 3.3814415783669665, + "tokens_seen": 2383413248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028254545454545454, + "loss": 2.5891, + "theoretical_loss": 3.3814340908148974, + "tokens_seen": 2383478784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825252525252525, + "loss": 2.7026, + "theoretical_loss": 3.381426603526347, + "tokens_seen": 2383544320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002825050505050505, + "loss": 2.5379, + "theoretical_loss": 3.381419116501298, + "tokens_seen": 2383609856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824848484848485, + "loss": 2.5709, + "theoretical_loss": 3.381411629739735, + "tokens_seen": 2383675392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824646464646465, + "loss": 2.728, + "theoretical_loss": 3.3814041432416397, + "tokens_seen": 2383740928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028244444444444443, + "loss": 2.8895, + "theoretical_loss": 3.3813966570069973, + "tokens_seen": 2383806464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028242424242424244, + "loss": 2.5061, + "theoretical_loss": 3.381389171035791, + "tokens_seen": 2383872000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002824040404040404, + "loss": 2.7659, + "theoretical_loss": 3.3813816853280034, + "tokens_seen": 2383937536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823838383838384, + "loss": 2.7107, + "theoretical_loss": 3.381374199883618, + "tokens_seen": 2384003072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028236363636363637, + "loss": 2.6017, + "theoretical_loss": 3.3813667147026196, + "tokens_seen": 2384068608 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1344107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2553467750549316, + "objective/train/theoretical_loss": 3.381359229784991, + "objective/train/tokens_used": 742993376, + "theoretical_loss": 3.381359229784991, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823434343434343, + "loss": 2.6357, + "theoretical_loss": 3.381359229784991, + "tokens_seen": 2384134144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028232323232323233, + "loss": 2.7234, + "theoretical_loss": 3.3813517451307153, + "tokens_seen": 2384199680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002823030303030303, + "loss": 2.7633, + "theoretical_loss": 3.3813442607397763, + "tokens_seen": 2384265216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822828282828283, + "loss": 2.7837, + "theoretical_loss": 3.381336776612158, + "tokens_seen": 2384330752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028226262626262626, + "loss": 2.9255, + "theoretical_loss": 3.3813292927478424, + "tokens_seen": 2384396288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028224242424242427, + "loss": 2.6371, + "theoretical_loss": 3.3813218091468147, + "tokens_seen": 2384461824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028222222222222223, + "loss": 2.706, + "theoretical_loss": 3.3813143258090577, + "tokens_seen": 2384527360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002822020202020202, + "loss": 2.6351, + "theoretical_loss": 3.3813068427345545, + "tokens_seen": 2384592896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821818181818182, + "loss": 2.8155, + "theoretical_loss": 3.3812993599232892, + "tokens_seen": 2384658432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028216161616161615, + "loss": 2.7971, + "theoretical_loss": 3.381291877375245, + "tokens_seen": 2384723968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028214141414141416, + "loss": 2.7587, + "theoretical_loss": 3.3812843950904057, + "tokens_seen": 2384789504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821212121212121, + "loss": 2.7534, + "theoretical_loss": 3.3812769130687546, + "tokens_seen": 2384855040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002821010101010101, + "loss": 2.6236, + "theoretical_loss": 3.381269431310275, + "tokens_seen": 2384920576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820808080808081, + "loss": 2.7819, + "theoretical_loss": 3.3812619498149505, + "tokens_seen": 2384986112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002820606060606061, + "loss": 2.6947, + "theoretical_loss": 3.3812544685827652, + "tokens_seen": 2385051648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028204040404040406, + "loss": 2.6212, + "theoretical_loss": 3.381246987613702, + "tokens_seen": 2385117184 + }, + { + "epoch": 0.44, + "learning_rate": 0.000282020202020202, + "loss": 2.8765, + "theoretical_loss": 3.3812395069077446, + "tokens_seen": 2385182720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028199999999999997, + "loss": 2.6303, + "theoretical_loss": 3.3812320264648767, + "tokens_seen": 2385248256 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281979797979798, + "loss": 2.7603, + "theoretical_loss": 3.381224546285081, + "tokens_seen": 2385313792 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281959595959596, + "loss": 2.6785, + "theoretical_loss": 3.3812170663683423, + "tokens_seen": 2385379328 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028193939393939395, + "loss": 2.4783, + "theoretical_loss": 3.3812095867146432, + "tokens_seen": 2385444864 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002819191919191919, + "loss": 2.6099, + "theoretical_loss": 3.3812021073239675, + "tokens_seen": 2385510400 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028189898989898986, + "loss": 2.7676, + "theoretical_loss": 3.381194628196299, + "tokens_seen": 2385575936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028187878787878793, + "loss": 2.5202, + "theoretical_loss": 3.3811871493316206, + "tokens_seen": 2385641472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818585858585859, + "loss": 2.7459, + "theoretical_loss": 3.3811796707299164, + "tokens_seen": 2385707008 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7913143634796143, + "objective/train/theoretical_loss": 3.3811721923911695, + "objective/train/tokens_used": 744631776, + "theoretical_loss": 3.3811721923911695, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028183838383838384, + "loss": 2.5842, + "theoretical_loss": 3.3811721923911695, + "tokens_seen": 2385772544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002818181818181818, + "loss": 2.8691, + "theoretical_loss": 3.381164714315364, + "tokens_seen": 2385838080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817979797979798, + "loss": 2.8892, + "theoretical_loss": 3.3811572365024825, + "tokens_seen": 2385903616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817777777777778, + "loss": 2.8021, + "theoretical_loss": 3.3811497589525095, + "tokens_seen": 2385969152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817575757575758, + "loss": 2.6422, + "theoretical_loss": 3.381142281665428, + "tokens_seen": 2386034688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028173737373737373, + "loss": 2.8206, + "theoretical_loss": 3.3811348046412215, + "tokens_seen": 2386100224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002817171717171717, + "loss": 2.6503, + "theoretical_loss": 3.3811273278798737, + "tokens_seen": 2386165760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028169696969696976, + "loss": 2.6991, + "theoretical_loss": 3.3811198513813685, + "tokens_seen": 2386231296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816767676767677, + "loss": 2.7846, + "theoretical_loss": 3.3811123751456886, + "tokens_seen": 2386296832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028165656565656567, + "loss": 2.4621, + "theoretical_loss": 3.3811048991728185, + "tokens_seen": 2386362368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816363636363636, + "loss": 2.6425, + "theoretical_loss": 3.3810974234627413, + "tokens_seen": 2386427904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002816161616161616, + "loss": 2.7658, + "theoretical_loss": 3.38108994801544, + "tokens_seen": 2386493440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028159595959595965, + "loss": 2.8343, + "theoretical_loss": 3.3810824728308995, + "tokens_seen": 2386558976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815757575757576, + "loss": 2.5458, + "theoretical_loss": 3.381074997909102, + "tokens_seen": 2386624512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028155555555555556, + "loss": 2.7797, + "theoretical_loss": 3.3810675232500316, + "tokens_seen": 2386690048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815353535353535, + "loss": 2.8826, + "theoretical_loss": 3.3810600488536715, + "tokens_seen": 2386755584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002815151515151515, + "loss": 2.2708, + "theoretical_loss": 3.381052574720006, + "tokens_seen": 2386821120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028149494949494954, + "loss": 2.6853, + "theoretical_loss": 3.3810451008490183, + "tokens_seen": 2386886656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814747474747475, + "loss": 2.8127, + "theoretical_loss": 3.3810376272406915, + "tokens_seen": 2386952192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028145454545454546, + "loss": 2.6413, + "theoretical_loss": 3.38103015389501, + "tokens_seen": 2387017728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814343434343434, + "loss": 2.8092, + "theoretical_loss": 3.381022680811957, + "tokens_seen": 2387083264 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002814141414141414, + "loss": 2.9097, + "theoretical_loss": 3.3810152079915152, + "tokens_seen": 2387148800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028139393939393943, + "loss": 2.5821, + "theoretical_loss": 3.3810077354336694, + "tokens_seen": 2387214336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813737373737374, + "loss": 2.9044, + "theoretical_loss": 3.381000263138403, + "tokens_seen": 2387279872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028135353535353535, + "loss": 2.8871, + "theoretical_loss": 3.3809927911056987, + "tokens_seen": 2387345408 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1344624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.891153573989868, + "objective/train/theoretical_loss": 3.380985319335541, + "objective/train/tokens_used": 746270176, + "theoretical_loss": 3.380985319335541, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813333333333333, + "loss": 2.7788, + "theoretical_loss": 3.380985319335541, + "tokens_seen": 2387410944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002813131313131313, + "loss": 2.8385, + "theoretical_loss": 3.380977847827913, + "tokens_seen": 2387476480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028129292929292933, + "loss": 2.7813, + "theoretical_loss": 3.3809703765827983, + "tokens_seen": 2387542016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812727272727273, + "loss": 2.9214, + "theoretical_loss": 3.3809629056001804, + "tokens_seen": 2387607552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028125252525252524, + "loss": 2.7556, + "theoretical_loss": 3.3809554348800432, + "tokens_seen": 2387673088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028123232323232325, + "loss": 2.8873, + "theoretical_loss": 3.38094796442237, + "tokens_seen": 2387738624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002812121212121212, + "loss": 2.8258, + "theoretical_loss": 3.3809404942271444, + "tokens_seen": 2387804160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811919191919192, + "loss": 2.9525, + "theoretical_loss": 3.38093302429435, + "tokens_seen": 2387869696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811717171717172, + "loss": 2.8633, + "theoretical_loss": 3.3809255546239707, + "tokens_seen": 2387935232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028115151515151513, + "loss": 2.7035, + "theoretical_loss": 3.3809180852159892, + "tokens_seen": 2388000768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028113131313131314, + "loss": 2.9471, + "theoretical_loss": 3.3809106160703903, + "tokens_seen": 2388066304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002811111111111111, + "loss": 2.947, + "theoretical_loss": 3.3809031471871567, + "tokens_seen": 2388131840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810909090909091, + "loss": 2.9078, + "theoretical_loss": 3.3808956785662723, + "tokens_seen": 2388197376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028107070707070707, + "loss": 2.9766, + "theoretical_loss": 3.3808882102077207, + "tokens_seen": 2388262912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002810505050505051, + "loss": 2.9183, + "theoretical_loss": 3.3808807421114855, + "tokens_seen": 2388328448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028103030303030304, + "loss": 2.7963, + "theoretical_loss": 3.38087327427755, + "tokens_seen": 2388393984 + }, + { + "epoch": 0.44, + "learning_rate": 0.000281010101010101, + "loss": 2.7585, + "theoretical_loss": 3.3808658067058976, + "tokens_seen": 2388459520 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280989898989899, + "loss": 2.4382, + "theoretical_loss": 3.380858339396513, + "tokens_seen": 2388525056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028096969696969696, + "loss": 2.8906, + "theoretical_loss": 3.3808508723493786, + "tokens_seen": 2388590592 + }, + { + "epoch": 0.44, + "learning_rate": 0.000280949494949495, + "loss": 2.8671, + "theoretical_loss": 3.3808434055644785, + "tokens_seen": 2388656128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028092929292929293, + "loss": 2.6652, + "theoretical_loss": 3.3808359390417966, + "tokens_seen": 2388721664 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002809090909090909, + "loss": 2.9219, + "theoretical_loss": 3.380828472781316, + "tokens_seen": 2388787200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808888888888889, + "loss": 2.7757, + "theoretical_loss": 3.3808210067830204, + "tokens_seen": 2388852736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808686868686869, + "loss": 2.7762, + "theoretical_loss": 3.3808135410468934, + "tokens_seen": 2388918272 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028084848484848487, + "loss": 2.9678, + "theoretical_loss": 3.380806075572919, + "tokens_seen": 2388983808 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1345276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.697154998779297, + "objective/train/theoretical_loss": 3.3807986103610803, + "objective/train/tokens_used": 747908576, + "theoretical_loss": 3.3807986103610803, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808282828282828, + "loss": 2.7107, + "theoretical_loss": 3.3807986103610803, + "tokens_seen": 2389049344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002808080808080808, + "loss": 2.8248, + "theoretical_loss": 3.3807911454113606, + "tokens_seen": 2389114880 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807878787878788, + "loss": 2.8628, + "theoretical_loss": 3.3807836807237446, + "tokens_seen": 2389180416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807676767676768, + "loss": 2.6826, + "theoretical_loss": 3.3807762162982153, + "tokens_seen": 2389245952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028074747474747476, + "loss": 2.9763, + "theoretical_loss": 3.380768752134756, + "tokens_seen": 2389311488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002807272727272727, + "loss": 2.4131, + "theoretical_loss": 3.3807612882333506, + "tokens_seen": 2389377024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028070707070707067, + "loss": 2.8533, + "theoretical_loss": 3.3807538245939828, + "tokens_seen": 2389442560 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028068686868686874, + "loss": 2.7356, + "theoretical_loss": 3.3807463612166364, + "tokens_seen": 2389508096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806666666666667, + "loss": 2.8053, + "theoretical_loss": 3.3807388981012942, + "tokens_seen": 2389573632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028064646464646465, + "loss": 2.9143, + "theoretical_loss": 3.3807314352479407, + "tokens_seen": 2389639168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002806262626262626, + "loss": 2.7772, + "theoretical_loss": 3.3807239726565594, + "tokens_seen": 2389704704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028060606060606057, + "loss": 3.043, + "theoretical_loss": 3.3807165103271335, + "tokens_seen": 2389770240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028058585858585863, + "loss": 2.9485, + "theoretical_loss": 3.380709048259647, + "tokens_seen": 2389835776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805656565656566, + "loss": 2.8931, + "theoretical_loss": 3.380701586454083, + "tokens_seen": 2389901312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028054545454545454, + "loss": 3.0718, + "theoretical_loss": 3.3806941249104256, + "tokens_seen": 2389966848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805252525252525, + "loss": 2.7618, + "theoretical_loss": 3.3806866636286586, + "tokens_seen": 2390032384 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002805050505050505, + "loss": 2.7515, + "theoretical_loss": 3.380679202608765, + "tokens_seen": 2390097920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804848484848485, + "loss": 2.8109, + "theoretical_loss": 3.380671741850729, + "tokens_seen": 2390163456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804646464646465, + "loss": 2.8763, + "theoretical_loss": 3.380664281354534, + "tokens_seen": 2390228992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028044444444444444, + "loss": 2.8202, + "theoretical_loss": 3.380656821120163, + "tokens_seen": 2390294528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804242424242424, + "loss": 2.7161, + "theoretical_loss": 3.3806493611476007, + "tokens_seen": 2390360064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002804040404040404, + "loss": 2.8798, + "theoretical_loss": 3.3806419014368303, + "tokens_seen": 2390425600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0002803838383838384, + "loss": 2.7269, + "theoretical_loss": 3.3806344419878354, + "tokens_seen": 2390491136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028036363636363637, + "loss": 2.7582, + "theoretical_loss": 3.3806269828005995, + "tokens_seen": 2390556672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00028034343434343433, + "loss": 2.9424, + "theoretical_loss": 3.3806195238751067, + "tokens_seen": 2390622208 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 1345792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.686504364013672, + "objective/train/theoretical_loss": 3.38061206521134, + "objective/train/tokens_used": 749546976, + "theoretical_loss": 3.38061206521134, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803232323232323, + "loss": 2.9271, + "theoretical_loss": 3.38061206521134, + "tokens_seen": 2390687744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002803030303030303, + "loss": 3.0383, + "theoretical_loss": 3.3806046068092837, + "tokens_seen": 2390753280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802828282828283, + "loss": 2.8676, + "theoretical_loss": 3.380597148668921, + "tokens_seen": 2390818816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028026262626262627, + "loss": 2.7188, + "theoretical_loss": 3.380589690790236, + "tokens_seen": 2390884352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802424242424242, + "loss": 2.8192, + "theoretical_loss": 3.380582233173212, + "tokens_seen": 2390949888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028022222222222223, + "loss": 2.9077, + "theoretical_loss": 3.380574775817832, + "tokens_seen": 2391015424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002802020202020202, + "loss": 2.7946, + "theoretical_loss": 3.3805673187240806, + "tokens_seen": 2391080960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801818181818182, + "loss": 2.8086, + "theoretical_loss": 3.3805598618919412, + "tokens_seen": 2391146496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028016161616161616, + "loss": 2.7448, + "theoretical_loss": 3.3805524053213976, + "tokens_seen": 2391212032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801414141414141, + "loss": 2.7065, + "theoretical_loss": 3.3805449490124335, + "tokens_seen": 2391277568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002801212121212121, + "loss": 2.8793, + "theoretical_loss": 3.380537492965032, + "tokens_seen": 2391343104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028010101010101014, + "loss": 2.7938, + "theoretical_loss": 3.3805300371791764, + "tokens_seen": 2391408640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002800808080808081, + "loss": 2.8832, + "theoretical_loss": 3.3805225816548523, + "tokens_seen": 2391474176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028006060606060605, + "loss": 2.9433, + "theoretical_loss": 3.380515126392041, + "tokens_seen": 2391539712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028004040404040406, + "loss": 2.8627, + "theoretical_loss": 3.380507671390728, + "tokens_seen": 2391605248 + }, + { + "epoch": 0.45, + "learning_rate": 0.000280020202020202, + "loss": 3.0073, + "theoretical_loss": 3.380500216650896, + "tokens_seen": 2391670784 + }, + { + "epoch": 0.45, + "learning_rate": 0.00028000000000000003, + "loss": 2.8601, + "theoretical_loss": 3.380492762172529, + "tokens_seen": 2391736320 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279979797979798, + "loss": 2.7146, + "theoretical_loss": 3.3804853079556105, + "tokens_seen": 2391801856 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027995959595959594, + "loss": 2.8338, + "theoretical_loss": 3.380477854000124, + "tokens_seen": 2391867392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027993939393939395, + "loss": 2.7368, + "theoretical_loss": 3.3804704003060535, + "tokens_seen": 2391932928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002799191919191919, + "loss": 2.7327, + "theoretical_loss": 3.380462946873383, + "tokens_seen": 2391998464 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798989898989899, + "loss": 2.8838, + "theoretical_loss": 3.380455493702095, + "tokens_seen": 2392064000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798787878787879, + "loss": 2.8482, + "theoretical_loss": 3.3804480407921744, + "tokens_seen": 2392129536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798585858585859, + "loss": 3.0375, + "theoretical_loss": 3.380440588143604, + "tokens_seen": 2392195072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027983838383838385, + "loss": 2.9375, + "theoretical_loss": 3.380433135756368, + "tokens_seen": 2392260608 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1347158, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466533899307251, + "objective/train/theoretical_loss": 3.3804256836304503, + "objective/train/tokens_used": 751185376, + "theoretical_loss": 3.3804256836304503, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002798181818181818, + "loss": 2.7443, + "theoretical_loss": 3.3804256836304503, + "tokens_seen": 2392326144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797979797979798, + "loss": 2.6552, + "theoretical_loss": 3.380418231765834, + "tokens_seen": 2392391680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027977777777777777, + "loss": 2.6006, + "theoretical_loss": 3.380410780162503, + "tokens_seen": 2392457216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797575757575758, + "loss": 2.7975, + "theoretical_loss": 3.3804033288204405, + "tokens_seen": 2392522752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027973737373737374, + "loss": 2.8489, + "theoretical_loss": 3.3803958777396312, + "tokens_seen": 2392588288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002797171717171717, + "loss": 2.703, + "theoretical_loss": 3.380388426920058, + "tokens_seen": 2392653824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796969696969697, + "loss": 2.7171, + "theoretical_loss": 3.380380976361705, + "tokens_seen": 2392719360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796767676767677, + "loss": 2.8282, + "theoretical_loss": 3.3803735260645555, + "tokens_seen": 2392784896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796565656565657, + "loss": 2.7735, + "theoretical_loss": 3.3803660760285936, + "tokens_seen": 2392850432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027963636363636363, + "loss": 2.7002, + "theoretical_loss": 3.3803586262538023, + "tokens_seen": 2392915968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002796161616161616, + "loss": 2.8109, + "theoretical_loss": 3.3803511767401666, + "tokens_seen": 2392981504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795959595959596, + "loss": 2.8573, + "theoretical_loss": 3.3803437274876686, + "tokens_seen": 2393047040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795757575757576, + "loss": 2.7173, + "theoretical_loss": 3.3803362784962934, + "tokens_seen": 2393112576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027955555555555557, + "loss": 2.6513, + "theoretical_loss": 3.380328829766024, + "tokens_seen": 2393178112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795353535353535, + "loss": 2.9696, + "theoretical_loss": 3.380321381296844, + "tokens_seen": 2393243648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002795151515151515, + "loss": 2.8708, + "theoretical_loss": 3.380313933088737, + "tokens_seen": 2393309184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027949494949494955, + "loss": 2.6608, + "theoretical_loss": 3.380306485141687, + "tokens_seen": 2393374720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794747474747475, + "loss": 2.7505, + "theoretical_loss": 3.380299037455678, + "tokens_seen": 2393440256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027945454545454546, + "loss": 2.6837, + "theoretical_loss": 3.3802915900306933, + "tokens_seen": 2393505792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794343434343434, + "loss": 2.8354, + "theoretical_loss": 3.3802841428667163, + "tokens_seen": 2393571328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002794141414141414, + "loss": 2.8529, + "theoretical_loss": 3.3802766959637314, + "tokens_seen": 2393636864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027939393939393944, + "loss": 2.791, + "theoretical_loss": 3.380269249321722, + "tokens_seen": 2393702400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793737373737374, + "loss": 2.7908, + "theoretical_loss": 3.3802618029406717, + "tokens_seen": 2393767936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027935353535353535, + "loss": 2.7979, + "theoretical_loss": 3.3802543568205645, + "tokens_seen": 2393833472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793333333333333, + "loss": 2.6807, + "theoretical_loss": 3.3802469109613837, + "tokens_seen": 2393899008 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1347750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0710670948028564, + "objective/train/theoretical_loss": 3.3802394653631134, + "objective/train/tokens_used": 752823776, + "theoretical_loss": 3.3802394653631134, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002793131313131313, + "loss": 2.9893, + "theoretical_loss": 3.3802394653631134, + "tokens_seen": 2393964544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027929292929292933, + "loss": 2.9364, + "theoretical_loss": 3.380232020025737, + "tokens_seen": 2394030080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792727272727273, + "loss": 2.6371, + "theoretical_loss": 3.3802245749492386, + "tokens_seen": 2394095616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027925252525252525, + "loss": 2.7877, + "theoretical_loss": 3.3802171301336017, + "tokens_seen": 2394161152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792323232323232, + "loss": 2.8156, + "theoretical_loss": 3.38020968557881, + "tokens_seen": 2394226688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002792121212121212, + "loss": 2.8113, + "theoretical_loss": 3.380202241284847, + "tokens_seen": 2394292224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791919191919192, + "loss": 2.7002, + "theoretical_loss": 3.3801947972516966, + "tokens_seen": 2394357760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791717171717172, + "loss": 2.8451, + "theoretical_loss": 3.3801873534793425, + "tokens_seen": 2394423296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027915151515151514, + "loss": 2.5973, + "theoretical_loss": 3.3801799099677687, + "tokens_seen": 2394488832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027913131313131315, + "loss": 2.7503, + "theoretical_loss": 3.380172466716959, + "tokens_seen": 2394554368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002791111111111111, + "loss": 2.6047, + "theoretical_loss": 3.3801650237268968, + "tokens_seen": 2394619904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790909090909091, + "loss": 2.3902, + "theoretical_loss": 3.3801575809975652, + "tokens_seen": 2394685440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002790707070707071, + "loss": 2.7321, + "theoretical_loss": 3.380150138528949, + "tokens_seen": 2394750976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027905050505050503, + "loss": 2.6298, + "theoretical_loss": 3.380142696321032, + "tokens_seen": 2394816512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027903030303030304, + "loss": 2.7848, + "theoretical_loss": 3.3801352543737972, + "tokens_seen": 2394882048 + }, + { + "epoch": 0.45, + "learning_rate": 0.000279010101010101, + "loss": 2.8815, + "theoretical_loss": 3.3801278126872285, + "tokens_seen": 2394947584 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278989898989899, + "loss": 2.6512, + "theoretical_loss": 3.3801203712613095, + "tokens_seen": 2395013120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027896969696969697, + "loss": 2.6708, + "theoretical_loss": 3.3801129300960246, + "tokens_seen": 2395078656 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278949494949495, + "loss": 2.7605, + "theoretical_loss": 3.380105489191357, + "tokens_seen": 2395144192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027892929292929294, + "loss": 2.9109, + "theoretical_loss": 3.380098048547291, + "tokens_seen": 2395209728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002789090909090909, + "loss": 2.7514, + "theoretical_loss": 3.380090608163809, + "tokens_seen": 2395275264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788888888888889, + "loss": 2.723, + "theoretical_loss": 3.3800831680408963, + "tokens_seen": 2395340800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027886868686868686, + "loss": 2.8103, + "theoretical_loss": 3.380075728178536, + "tokens_seen": 2395406336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027884848484848487, + "loss": 2.9746, + "theoretical_loss": 3.380068288576712, + "tokens_seen": 2395471872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027882828282828283, + "loss": 2.6027, + "theoretical_loss": 3.3800608492354076, + "tokens_seen": 2395537408 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1348886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.808833122253418, + "objective/train/theoretical_loss": 3.3800534101546065, + "objective/train/tokens_used": 754462176, + "theoretical_loss": 3.3800534101546065, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002788080808080808, + "loss": 2.7522, + "theoretical_loss": 3.3800534101546065, + "tokens_seen": 2395602944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787878787878788, + "loss": 2.9644, + "theoretical_loss": 3.3800459713342934, + "tokens_seen": 2395668480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787676767676768, + "loss": 2.8169, + "theoretical_loss": 3.380038532774451, + "tokens_seen": 2395734016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027874747474747476, + "loss": 2.9274, + "theoretical_loss": 3.380031094475064, + "tokens_seen": 2395799552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787272727272727, + "loss": 2.5923, + "theoretical_loss": 3.3800236564361157, + "tokens_seen": 2395865088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002787070707070707, + "loss": 2.7969, + "theoretical_loss": 3.3800162186575893, + "tokens_seen": 2395930624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786868686868687, + "loss": 2.8698, + "theoretical_loss": 3.3800087811394697, + "tokens_seen": 2395996160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786666666666667, + "loss": 2.7117, + "theoretical_loss": 3.3800013438817396, + "tokens_seen": 2396061696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027864646464646466, + "loss": 2.8496, + "theoretical_loss": 3.3799939068843834, + "tokens_seen": 2396127232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002786262626262626, + "loss": 2.7473, + "theoretical_loss": 3.3799864701473847, + "tokens_seen": 2396192768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027860606060606057, + "loss": 2.9282, + "theoretical_loss": 3.379979033670727, + "tokens_seen": 2396258304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027858585858585864, + "loss": 2.6832, + "theoretical_loss": 3.3799715974543947, + "tokens_seen": 2396323840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785656565656566, + "loss": 2.482, + "theoretical_loss": 3.379964161498371, + "tokens_seen": 2396389376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027854545454545455, + "loss": 2.8163, + "theoretical_loss": 3.3799567258026397, + "tokens_seen": 2396454912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002785252525252525, + "loss": 2.8279, + "theoretical_loss": 3.3799492903671844, + "tokens_seen": 2396520448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027850505050505046, + "loss": 2.7673, + "theoretical_loss": 3.3799418551919898, + "tokens_seen": 2396585984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027848484848484853, + "loss": 2.6174, + "theoretical_loss": 3.379934420277039, + "tokens_seen": 2396651520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784646464646465, + "loss": 2.8973, + "theoretical_loss": 3.3799269856223155, + "tokens_seen": 2396717056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027844444444444444, + "loss": 2.8768, + "theoretical_loss": 3.379919551227804, + "tokens_seen": 2396782592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784242424242424, + "loss": 2.82, + "theoretical_loss": 3.379912117093487, + "tokens_seen": 2396848128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002784040404040404, + "loss": 2.9526, + "theoretical_loss": 3.3799046832193493, + "tokens_seen": 2396913664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783838383838384, + "loss": 2.6483, + "theoretical_loss": 3.3798972496053743, + "tokens_seen": 2396979200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783636363636364, + "loss": 2.7374, + "theoretical_loss": 3.379889816251546, + "tokens_seen": 2397044736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027834343434343434, + "loss": 2.7603, + "theoretical_loss": 3.3798823831578475, + "tokens_seen": 2397110272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002783232323232323, + "loss": 2.9141, + "theoretical_loss": 3.3798749503242638, + "tokens_seen": 2397175808 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1349689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.743565797805786, + "objective/train/theoretical_loss": 3.3798675177507773, + "objective/train/tokens_used": 756100576, + "theoretical_loss": 3.3798675177507773, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027830303030303036, + "loss": 2.6788, + "theoretical_loss": 3.3798675177507773, + "tokens_seen": 2397241344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782828282828283, + "loss": 2.66, + "theoretical_loss": 3.379860085437373, + "tokens_seen": 2397306880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027826262626262627, + "loss": 2.8103, + "theoretical_loss": 3.3798526533840336, + "tokens_seen": 2397372416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027824242424242423, + "loss": 2.9672, + "theoretical_loss": 3.379845221590744, + "tokens_seen": 2397437952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002782222222222222, + "loss": 2.6704, + "theoretical_loss": 3.379837790057487, + "tokens_seen": 2397503488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027820202020202025, + "loss": 2.7552, + "theoretical_loss": 3.3798303587842473, + "tokens_seen": 2397569024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781818181818182, + "loss": 2.6536, + "theoretical_loss": 3.3798229277710083, + "tokens_seen": 2397634560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027816161616161616, + "loss": 2.6757, + "theoretical_loss": 3.3798154970177534, + "tokens_seen": 2397700096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002781414141414141, + "loss": 2.5656, + "theoretical_loss": 3.3798080665244665, + "tokens_seen": 2397765632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027812121212121213, + "loss": 2.4767, + "theoretical_loss": 3.379800636291132, + "tokens_seen": 2397831168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027810101010101014, + "loss": 2.7956, + "theoretical_loss": 3.379793206317734, + "tokens_seen": 2397896704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002780808080808081, + "loss": 2.6946, + "theoretical_loss": 3.3797857766042547, + "tokens_seen": 2397962240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027806060606060606, + "loss": 2.762, + "theoretical_loss": 3.379778347150679, + "tokens_seen": 2398027776 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278040404040404, + "loss": 2.8496, + "theoretical_loss": 3.3797709179569906, + "tokens_seen": 2398093312 + }, + { + "epoch": 0.45, + "learning_rate": 0.000278020202020202, + "loss": 2.7841, + "theoretical_loss": 3.379763489023173, + "tokens_seen": 2398158848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027800000000000004, + "loss": 2.7467, + "theoretical_loss": 3.3797560603492105, + "tokens_seen": 2398224384 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277979797979798, + "loss": 2.5992, + "theoretical_loss": 3.3797486319350867, + "tokens_seen": 2398289920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027795959595959595, + "loss": 2.6982, + "theoretical_loss": 3.3797412037807857, + "tokens_seen": 2398355456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027793939393939396, + "loss": 2.8855, + "theoretical_loss": 3.3797337758862906, + "tokens_seen": 2398420992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002779191919191919, + "loss": 2.767, + "theoretical_loss": 3.3797263482515856, + "tokens_seen": 2398486528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027789898989898993, + "loss": 2.8747, + "theoretical_loss": 3.3797189208766545, + "tokens_seen": 2398552064 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778787878787879, + "loss": 2.7415, + "theoretical_loss": 3.3797114937614814, + "tokens_seen": 2398617600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027785858585858584, + "loss": 2.7703, + "theoretical_loss": 3.37970406690605, + "tokens_seen": 2398683136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027783838383838385, + "loss": 2.8808, + "theoretical_loss": 3.3796966403103434, + "tokens_seen": 2398748672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002778181818181818, + "loss": 2.4705, + "theoretical_loss": 3.3796892139743466, + "tokens_seen": 2398814208 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1350650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8804543018341064, + "objective/train/theoretical_loss": 3.3796817878980425, + "objective/train/tokens_used": 757738976, + "theoretical_loss": 3.3796817878980425, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777979797979798, + "loss": 2.6102, + "theoretical_loss": 3.3796817878980425, + "tokens_seen": 2398879744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777777777777778, + "loss": 2.6403, + "theoretical_loss": 3.379674362081415, + "tokens_seen": 2398945280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777575757575758, + "loss": 2.7599, + "theoretical_loss": 3.3796669365244485, + "tokens_seen": 2399010816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027773737373737375, + "loss": 2.7357, + "theoretical_loss": 3.3796595112271266, + "tokens_seen": 2399076352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002777171717171717, + "loss": 2.7456, + "theoretical_loss": 3.379652086189433, + "tokens_seen": 2399141888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776969696969697, + "loss": 2.7646, + "theoretical_loss": 3.3796446614113513, + "tokens_seen": 2399207424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027767676767676767, + "loss": 2.6643, + "theoretical_loss": 3.379637236892866, + "tokens_seen": 2399272960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776565656565657, + "loss": 2.5843, + "theoretical_loss": 3.37962981263396, + "tokens_seen": 2399338496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027763636363636364, + "loss": 2.7863, + "theoretical_loss": 3.379622388634618, + "tokens_seen": 2399404032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002776161616161616, + "loss": 3.0152, + "theoretical_loss": 3.3796149648948237, + "tokens_seen": 2399469568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775959595959596, + "loss": 2.951, + "theoretical_loss": 3.3796075414145603, + "tokens_seen": 2399535104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775757575757576, + "loss": 2.663, + "theoretical_loss": 3.3796001181938125, + "tokens_seen": 2399600640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775555555555556, + "loss": 2.8216, + "theoretical_loss": 3.3795926952325637, + "tokens_seen": 2399666176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027753535353535353, + "loss": 2.7095, + "theoretical_loss": 3.3795852725307975, + "tokens_seen": 2399731712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002775151515151515, + "loss": 2.6633, + "theoretical_loss": 3.3795778500884976, + "tokens_seen": 2399797248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774949494949495, + "loss": 2.6636, + "theoretical_loss": 3.379570427905649, + "tokens_seen": 2399862784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774747474747475, + "loss": 2.6945, + "theoretical_loss": 3.3795630059822344, + "tokens_seen": 2399928320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027745454545454547, + "loss": 2.9088, + "theoretical_loss": 3.379555584318238, + "tokens_seen": 2399993856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774343434343434, + "loss": 2.7572, + "theoretical_loss": 3.379548162913644, + "tokens_seen": 2400059392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002774141414141414, + "loss": 2.9269, + "theoretical_loss": 3.379540741768436, + "tokens_seen": 2400124928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027739393939393945, + "loss": 2.8035, + "theoretical_loss": 3.3795333208825973, + "tokens_seen": 2400190464 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773737373737374, + "loss": 2.6372, + "theoretical_loss": 3.3795259002561124, + "tokens_seen": 2400256000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027735353535353536, + "loss": 2.7549, + "theoretical_loss": 3.379518479888965, + "tokens_seen": 2400321536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773333333333333, + "loss": 2.3375, + "theoretical_loss": 3.379511059781139, + "tokens_seen": 2400387072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002773131313131313, + "loss": 2.3997, + "theoretical_loss": 3.3795036399326186, + "tokens_seen": 2400452608 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1352096, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.710724115371704, + "objective/train/theoretical_loss": 3.3794962203433867, + "objective/train/tokens_used": 759377376, + "theoretical_loss": 3.3794962203433867, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027729292929292934, + "loss": 2.684, + "theoretical_loss": 3.3794962203433867, + "tokens_seen": 2400518144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772727272727273, + "loss": 2.631, + "theoretical_loss": 3.3794888010134283, + "tokens_seen": 2400583680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027725252525252525, + "loss": 2.7944, + "theoretical_loss": 3.379481381942726, + "tokens_seen": 2400649216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772323232323232, + "loss": 2.9077, + "theoretical_loss": 3.379473963131265, + "tokens_seen": 2400714752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002772121212121212, + "loss": 2.9504, + "theoretical_loss": 3.3794665445790284, + "tokens_seen": 2400780288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027719191919191923, + "loss": 2.7789, + "theoretical_loss": 3.379459126286, + "tokens_seen": 2400845824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771717171717172, + "loss": 2.6824, + "theoretical_loss": 3.379451708252164, + "tokens_seen": 2400911360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027715151515151514, + "loss": 2.6346, + "theoretical_loss": 3.3794442904775037, + "tokens_seen": 2400976896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771313131313131, + "loss": 2.8917, + "theoretical_loss": 3.3794368729620037, + "tokens_seen": 2401042432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002771111111111111, + "loss": 2.732, + "theoretical_loss": 3.3794294557056475, + "tokens_seen": 2401107968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770909090909091, + "loss": 2.6541, + "theoretical_loss": 3.3794220387084195, + "tokens_seen": 2401173504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002770707070707071, + "loss": 2.5943, + "theoretical_loss": 3.3794146219703025, + "tokens_seen": 2401239040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027705050505050504, + "loss": 2.8257, + "theoretical_loss": 3.3794072054912814, + "tokens_seen": 2401304576 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277030303030303, + "loss": 2.6517, + "theoretical_loss": 3.379399789271339, + "tokens_seen": 2401370112 + }, + { + "epoch": 0.45, + "learning_rate": 0.000277010101010101, + "loss": 2.6323, + "theoretical_loss": 3.379392373310461, + "tokens_seen": 2401435648 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276989898989899, + "loss": 2.6133, + "theoretical_loss": 3.3793849576086297, + "tokens_seen": 2401501184 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276969696969697, + "loss": 2.6089, + "theoretical_loss": 3.379377542165829, + "tokens_seen": 2401566720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027694949494949493, + "loss": 2.7038, + "theoretical_loss": 3.3793701269820433, + "tokens_seen": 2401632256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027692929292929294, + "loss": 2.7019, + "theoretical_loss": 3.3793627120572567, + "tokens_seen": 2401697792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002769090909090909, + "loss": 2.576, + "theoretical_loss": 3.3793552973914522, + "tokens_seen": 2401763328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768888888888889, + "loss": 2.99, + "theoretical_loss": 3.379347882984615, + "tokens_seen": 2401828864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027686868686868687, + "loss": 2.8093, + "theoretical_loss": 3.3793404688367277, + "tokens_seen": 2401894400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768484848484848, + "loss": 2.8093, + "theoretical_loss": 3.379333054947775, + "tokens_seen": 2401959936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027682828282828283, + "loss": 2.7734, + "theoretical_loss": 3.3793256413177404, + "tokens_seen": 2402025472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002768080808080808, + "loss": 2.7772, + "theoretical_loss": 3.3793182279466083, + "tokens_seen": 2402091008 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1352871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.017096996307373, + "objective/train/theoretical_loss": 3.379310814834362, + "objective/train/tokens_used": 761015776, + "theoretical_loss": 3.379310814834362, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767878787878788, + "loss": 2.5526, + "theoretical_loss": 3.379310814834362, + "tokens_seen": 2402156544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027676767676767676, + "loss": 2.7388, + "theoretical_loss": 3.3793034019809856, + "tokens_seen": 2402222080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027674747474747477, + "loss": 2.8766, + "theoretical_loss": 3.3792959893864625, + "tokens_seen": 2402287616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002767272727272727, + "loss": 2.6665, + "theoretical_loss": 3.3792885770507777, + "tokens_seen": 2402353152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027670707070707074, + "loss": 2.6875, + "theoretical_loss": 3.3792811649739143, + "tokens_seen": 2402418688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766868686868687, + "loss": 2.76, + "theoretical_loss": 3.3792737531558568, + "tokens_seen": 2402484224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027666666666666665, + "loss": 2.9441, + "theoretical_loss": 3.3792663415965882, + "tokens_seen": 2402549760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027664646464646466, + "loss": 2.7497, + "theoretical_loss": 3.379258930296093, + "tokens_seen": 2402615296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002766262626262626, + "loss": 2.7292, + "theoretical_loss": 3.3792515192543555, + "tokens_seen": 2402680832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027660606060606063, + "loss": 2.6678, + "theoretical_loss": 3.379244108471359, + "tokens_seen": 2402746368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765858585858586, + "loss": 2.7613, + "theoretical_loss": 3.3792366979470874, + "tokens_seen": 2402811904 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765656565656566, + "loss": 2.5349, + "theoretical_loss": 3.3792292876815244, + "tokens_seen": 2402877440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027654545454545456, + "loss": 2.6541, + "theoretical_loss": 3.379221877674655, + "tokens_seen": 2402942976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765252525252525, + "loss": 2.3858, + "theoretical_loss": 3.3792144679264617, + "tokens_seen": 2403008512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002765050505050505, + "loss": 2.662, + "theoretical_loss": 3.3792070584369296, + "tokens_seen": 2403074048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764848484848485, + "loss": 2.7191, + "theoretical_loss": 3.3791996492060417, + "tokens_seen": 2403139584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764646464646465, + "loss": 2.6336, + "theoretical_loss": 3.379192240233783, + "tokens_seen": 2403205120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027644444444444445, + "loss": 2.6553, + "theoretical_loss": 3.3791848315201363, + "tokens_seen": 2403270656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764242424242424, + "loss": 2.7668, + "theoretical_loss": 3.379177423065086, + "tokens_seen": 2403336192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002764040404040404, + "loss": 2.6935, + "theoretical_loss": 3.379170014868616, + "tokens_seen": 2403401728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763838383838384, + "loss": 2.6467, + "theoretical_loss": 3.37916260693071, + "tokens_seen": 2403467264 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763636363636364, + "loss": 2.8631, + "theoretical_loss": 3.3791551992513527, + "tokens_seen": 2403532800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027634343434343434, + "loss": 2.6837, + "theoretical_loss": 3.3791477918305275, + "tokens_seen": 2403598336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763232323232323, + "loss": 2.8844, + "theoretical_loss": 3.3791403846682178, + "tokens_seen": 2403663872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002763030303030303, + "loss": 2.8265, + "theoretical_loss": 3.3791329777644084, + "tokens_seen": 2403729408 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1354213, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9096786975860596, + "objective/train/theoretical_loss": 3.379125571119083, + "objective/train/tokens_used": 762654176, + "theoretical_loss": 3.379125571119083, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762828282828283, + "loss": 2.6829, + "theoretical_loss": 3.379125571119083, + "tokens_seen": 2403794944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762626262626263, + "loss": 2.6302, + "theoretical_loss": 3.379118164732225, + "tokens_seen": 2403860480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027624242424242423, + "loss": 2.5693, + "theoretical_loss": 3.3791107586038187, + "tokens_seen": 2403926016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002762222222222222, + "loss": 2.6304, + "theoretical_loss": 3.379103352733848, + "tokens_seen": 2403991552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027620202020202026, + "loss": 2.63, + "theoretical_loss": 3.3790959471222974, + "tokens_seen": 2404057088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761818181818182, + "loss": 2.6509, + "theoretical_loss": 3.3790885417691503, + "tokens_seen": 2404122624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027616161616161617, + "loss": 2.9735, + "theoretical_loss": 3.3790811366743903, + "tokens_seen": 2404188160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761414141414141, + "loss": 2.7154, + "theoretical_loss": 3.379073731838002, + "tokens_seen": 2404253696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002761212121212121, + "loss": 2.9187, + "theoretical_loss": 3.3790663272599692, + "tokens_seen": 2404319232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027610101010101015, + "loss": 2.7147, + "theoretical_loss": 3.3790589229402754, + "tokens_seen": 2404384768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002760808080808081, + "loss": 2.5673, + "theoretical_loss": 3.379051518878905, + "tokens_seen": 2404450304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027606060606060606, + "loss": 2.5827, + "theoretical_loss": 3.3790441150758417, + "tokens_seen": 2404515840 + }, + { + "epoch": 0.45, + "learning_rate": 0.000276040404040404, + "loss": 2.8213, + "theoretical_loss": 3.3790367115310698, + "tokens_seen": 2404581376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027602020202020203, + "loss": 2.677, + "theoretical_loss": 3.379029308244573, + "tokens_seen": 2404646912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027600000000000004, + "loss": 2.7966, + "theoretical_loss": 3.379021905216335, + "tokens_seen": 2404712448 + }, + { + "epoch": 0.45, + "learning_rate": 0.000275979797979798, + "loss": 2.527, + "theoretical_loss": 3.3790145024463403, + "tokens_seen": 2404777984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027595959595959595, + "loss": 2.8183, + "theoretical_loss": 3.3790070999345723, + "tokens_seen": 2404843520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759393939393939, + "loss": 2.6689, + "theoretical_loss": 3.3789996976810155, + "tokens_seen": 2404909056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002759191919191919, + "loss": 2.8761, + "theoretical_loss": 3.378992295685653, + "tokens_seen": 2404974592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027589898989898993, + "loss": 2.8458, + "theoretical_loss": 3.37898489394847, + "tokens_seen": 2405040128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758787878787879, + "loss": 2.6567, + "theoretical_loss": 3.3789774924694496, + "tokens_seen": 2405105664 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027585858585858585, + "loss": 2.7401, + "theoretical_loss": 3.378970091248576, + "tokens_seen": 2405171200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027583838383838386, + "loss": 2.7418, + "theoretical_loss": 3.378962690285833, + "tokens_seen": 2405236736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002758181818181818, + "loss": 2.7206, + "theoretical_loss": 3.3789552895812047, + "tokens_seen": 2405302272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757979797979798, + "loss": 2.7005, + "theoretical_loss": 3.378947889134675, + "tokens_seen": 2405367808 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1354954, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6834776401519775, + "objective/train/theoretical_loss": 3.378940488946228, + "objective/train/tokens_used": 764292576, + "theoretical_loss": 3.378940488946228, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757777777777778, + "loss": 2.5833, + "theoretical_loss": 3.378940488946228, + "tokens_seen": 2405433344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027575757575757574, + "loss": 2.9802, + "theoretical_loss": 3.3789330890158475, + "tokens_seen": 2405498880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027573737373737375, + "loss": 2.7339, + "theoretical_loss": 3.378925689343518, + "tokens_seen": 2405564416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002757171717171717, + "loss": 2.7226, + "theoretical_loss": 3.3789182899292225, + "tokens_seen": 2405629952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756969696969697, + "loss": 2.934, + "theoretical_loss": 3.3789108907729455, + "tokens_seen": 2405695488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756767676767677, + "loss": 2.6487, + "theoretical_loss": 3.378903491874671, + "tokens_seen": 2405761024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756565656565657, + "loss": 2.7297, + "theoretical_loss": 3.3788960932343834, + "tokens_seen": 2405826560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027563636363636364, + "loss": 2.9472, + "theoretical_loss": 3.378888694852066, + "tokens_seen": 2405892096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002756161616161616, + "loss": 2.54, + "theoretical_loss": 3.3788812967277027, + "tokens_seen": 2405957632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755959595959596, + "loss": 2.5544, + "theoretical_loss": 3.3788738988612783, + "tokens_seen": 2406023168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027557575757575757, + "loss": 2.5582, + "theoretical_loss": 3.3788665012527757, + "tokens_seen": 2406088704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755555555555556, + "loss": 2.5839, + "theoretical_loss": 3.37885910390218, + "tokens_seen": 2406154240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027553535353535354, + "loss": 2.8294, + "theoretical_loss": 3.378851706809474, + "tokens_seen": 2406219776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002755151515151515, + "loss": 2.9212, + "theoretical_loss": 3.3788443099746432, + "tokens_seen": 2406285312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754949494949495, + "loss": 2.8336, + "theoretical_loss": 3.37883691339767, + "tokens_seen": 2406350848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754747474747475, + "loss": 2.5969, + "theoretical_loss": 3.3788295170785396, + "tokens_seen": 2406416384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027545454545454547, + "loss": 2.8528, + "theoretical_loss": 3.378822121017235, + "tokens_seen": 2406481920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027543434343434343, + "loss": 2.5279, + "theoretical_loss": 3.378814725213741, + "tokens_seen": 2406547456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002754141414141414, + "loss": 2.5849, + "theoretical_loss": 3.378807329668041, + "tokens_seen": 2406612992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753939393939394, + "loss": 2.6766, + "theoretical_loss": 3.3787999343801194, + "tokens_seen": 2406678528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753737373737374, + "loss": 2.5753, + "theoretical_loss": 3.37879253934996, + "tokens_seen": 2406744064 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027535353535353537, + "loss": 2.836, + "theoretical_loss": 3.378785144577547, + "tokens_seen": 2406809600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753333333333333, + "loss": 2.7524, + "theoretical_loss": 3.3787777500628646, + "tokens_seen": 2406875136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0002753131313131313, + "loss": 2.8045, + "theoretical_loss": 3.378770355805896, + "tokens_seen": 2406940672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00027529292929292934, + "loss": 2.9984, + "theoretical_loss": 3.3787629618066255, + "tokens_seen": 2407006208 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 1356171, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.686603546142578, + "objective/train/theoretical_loss": 3.3787555680650376, + "objective/train/tokens_used": 765930976, + "theoretical_loss": 3.3787555680650376, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752727272727273, + "loss": 2.4677, + "theoretical_loss": 3.3787555680650376, + "tokens_seen": 2407071744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027525252525252526, + "loss": 2.3963, + "theoretical_loss": 3.3787481745811156, + "tokens_seen": 2407137280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002752323232323232, + "loss": 2.4261, + "theoretical_loss": 3.3787407813548445, + "tokens_seen": 2407202816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027521212121212117, + "loss": 2.6736, + "theoretical_loss": 3.378733388386207, + "tokens_seen": 2407268352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027519191919191924, + "loss": 2.515, + "theoretical_loss": 3.3787259956751883, + "tokens_seen": 2407333888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751717171717172, + "loss": 2.6927, + "theoretical_loss": 3.3787186032217718, + "tokens_seen": 2407399424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027515151515151515, + "loss": 2.6436, + "theoretical_loss": 3.378711211025941, + "tokens_seen": 2407464960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002751313131313131, + "loss": 2.4874, + "theoretical_loss": 3.3787038190876815, + "tokens_seen": 2407530496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027511111111111106, + "loss": 2.76, + "theoretical_loss": 3.3786964274069757, + "tokens_seen": 2407596032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027509090909090913, + "loss": 2.8685, + "theoretical_loss": 3.3786890359838084, + "tokens_seen": 2407661568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002750707070707071, + "loss": 2.3877, + "theoretical_loss": 3.3786816448181636, + "tokens_seen": 2407727104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027505050505050504, + "loss": 2.7844, + "theoretical_loss": 3.378674253910025, + "tokens_seen": 2407792640 + }, + { + "epoch": 0.46, + "learning_rate": 0.000275030303030303, + "loss": 2.8392, + "theoretical_loss": 3.378666863259377, + "tokens_seen": 2407858176 + }, + { + "epoch": 0.46, + "learning_rate": 0.000275010101010101, + "loss": 2.7896, + "theoretical_loss": 3.3786594728662034, + "tokens_seen": 2407923712 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274989898989899, + "loss": 2.8347, + "theoretical_loss": 3.378652082730488, + "tokens_seen": 2407989248 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274969696969697, + "loss": 2.6893, + "theoretical_loss": 3.378644692852215, + "tokens_seen": 2408054784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027494949494949494, + "loss": 2.585, + "theoretical_loss": 3.3786373032313692, + "tokens_seen": 2408120320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002749292929292929, + "loss": 2.4693, + "theoretical_loss": 3.3786299138679334, + "tokens_seen": 2408185856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027490909090909096, + "loss": 2.5145, + "theoretical_loss": 3.378622524761892, + "tokens_seen": 2408251392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002748888888888889, + "loss": 2.7176, + "theoretical_loss": 3.3786151359132295, + "tokens_seen": 2408316928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027486868686868687, + "loss": 2.6916, + "theoretical_loss": 3.37860774732193, + "tokens_seen": 2408382464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027484848484848483, + "loss": 2.5285, + "theoretical_loss": 3.3786003589879763, + "tokens_seen": 2408448000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027482828282828284, + "loss": 2.7363, + "theoretical_loss": 3.378592970911354, + "tokens_seen": 2408513536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027480808080808085, + "loss": 2.8201, + "theoretical_loss": 3.378585583092046, + "tokens_seen": 2408579072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747878787878788, + "loss": 2.6867, + "theoretical_loss": 3.3785781955300367, + "tokens_seen": 2408644608 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1356770, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.764434576034546, + "objective/train/theoretical_loss": 3.3785708082253105, + "objective/train/tokens_used": 767569376, + "theoretical_loss": 3.3785708082253105, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027476767676767676, + "loss": 2.6452, + "theoretical_loss": 3.3785708082253105, + "tokens_seen": 2408710144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002747474747474747, + "loss": 2.6929, + "theoretical_loss": 3.378563421177851, + "tokens_seen": 2408775680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027472727272727273, + "loss": 2.55, + "theoretical_loss": 3.378556034387642, + "tokens_seen": 2408841216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027470707070707074, + "loss": 2.7077, + "theoretical_loss": 3.378548647854669, + "tokens_seen": 2408906752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746868686868687, + "loss": 2.6395, + "theoretical_loss": 3.378541261578914, + "tokens_seen": 2408972288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027466666666666666, + "loss": 2.5824, + "theoretical_loss": 3.378533875560362, + "tokens_seen": 2409037824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027464646464646467, + "loss": 2.6483, + "theoretical_loss": 3.3785264897989977, + "tokens_seen": 2409103360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002746262626262626, + "loss": 2.6488, + "theoretical_loss": 3.378519104294804, + "tokens_seen": 2409168896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027460606060606064, + "loss": 2.7615, + "theoretical_loss": 3.378511719047766, + "tokens_seen": 2409234432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745858585858586, + "loss": 2.5451, + "theoretical_loss": 3.3785043340578667, + "tokens_seen": 2409299968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027456565656565655, + "loss": 2.5885, + "theoretical_loss": 3.3784969493250907, + "tokens_seen": 2409365504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027454545454545456, + "loss": 2.7113, + "theoretical_loss": 3.378489564849422, + "tokens_seen": 2409431040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002745252525252525, + "loss": 2.6616, + "theoretical_loss": 3.378482180630845, + "tokens_seen": 2409496576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027450505050505053, + "loss": 2.8175, + "theoretical_loss": 3.378474796669343, + "tokens_seen": 2409562112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744848484848485, + "loss": 2.9357, + "theoretical_loss": 3.378467412964901, + "tokens_seen": 2409627648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744646464646465, + "loss": 2.6062, + "theoretical_loss": 3.3784600295175022, + "tokens_seen": 2409693184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027444444444444445, + "loss": 2.8357, + "theoretical_loss": 3.3784526463271307, + "tokens_seen": 2409758720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744242424242424, + "loss": 2.472, + "theoretical_loss": 3.3784452633937714, + "tokens_seen": 2409824256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002744040404040404, + "loss": 2.7669, + "theoretical_loss": 3.378437880717408, + "tokens_seen": 2409889792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743838383838384, + "loss": 2.8154, + "theoretical_loss": 3.378430498298024, + "tokens_seen": 2409955328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743636363636364, + "loss": 2.6232, + "theoretical_loss": 3.378423116135604, + "tokens_seen": 2410020864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027434343434343435, + "loss": 2.8301, + "theoretical_loss": 3.3784157342301318, + "tokens_seen": 2410086400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743232323232323, + "loss": 2.4924, + "theoretical_loss": 3.378408352581592, + "tokens_seen": 2410151936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002743030303030303, + "loss": 2.7753, + "theoretical_loss": 3.3784009711899676, + "tokens_seen": 2410217472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742828282828283, + "loss": 2.8931, + "theoretical_loss": 3.3783935900552438, + "tokens_seen": 2410283008 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1358132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5906593799591064, + "objective/train/theoretical_loss": 3.378386209177404, + "objective/train/tokens_used": 769207776, + "theoretical_loss": 3.378386209177404, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742626262626263, + "loss": 2.6897, + "theoretical_loss": 3.378386209177404, + "tokens_seen": 2410348544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027424242424242424, + "loss": 2.7538, + "theoretical_loss": 3.378378828556433, + "tokens_seen": 2410414080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742222222222222, + "loss": 2.5502, + "theoretical_loss": 3.378371448192314, + "tokens_seen": 2410479616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002742020202020202, + "loss": 2.8781, + "theoretical_loss": 3.3783640680850313, + "tokens_seen": 2410545152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741818181818182, + "loss": 2.7303, + "theoretical_loss": 3.378356688234569, + "tokens_seen": 2410610688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741616161616162, + "loss": 2.5713, + "theoretical_loss": 3.378349308640912, + "tokens_seen": 2410676224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027414141414141413, + "loss": 2.8469, + "theoretical_loss": 3.3783419293040433, + "tokens_seen": 2410741760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002741212121212121, + "loss": 2.8572, + "theoretical_loss": 3.378334550223947, + "tokens_seen": 2410807296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027410101010101015, + "loss": 2.7314, + "theoretical_loss": 3.378327171400608, + "tokens_seen": 2410872832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002740808080808081, + "loss": 2.5106, + "theoretical_loss": 3.3783197928340103, + "tokens_seen": 2410938368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027406060606060607, + "loss": 2.6442, + "theoretical_loss": 3.378312414524137, + "tokens_seen": 2411003904 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274040404040404, + "loss": 2.8704, + "theoretical_loss": 3.378305036470973, + "tokens_seen": 2411069440 + }, + { + "epoch": 0.46, + "learning_rate": 0.000274020202020202, + "loss": 2.723, + "theoretical_loss": 3.378297658674503, + "tokens_seen": 2411134976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027400000000000005, + "loss": 3.0829, + "theoretical_loss": 3.3782902811347095, + "tokens_seen": 2411200512 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273979797979798, + "loss": 2.6501, + "theoretical_loss": 3.3782829038515776, + "tokens_seen": 2411266048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027395959595959596, + "loss": 2.6726, + "theoretical_loss": 3.378275526825091, + "tokens_seen": 2411331584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002739393939393939, + "loss": 2.5402, + "theoretical_loss": 3.3782681500552343, + "tokens_seen": 2411397120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027391919191919193, + "loss": 2.6948, + "theoretical_loss": 3.378260773541991, + "tokens_seen": 2411462656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027389898989898994, + "loss": 2.6149, + "theoretical_loss": 3.378253397285346, + "tokens_seen": 2411528192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738787878787879, + "loss": 2.7356, + "theoretical_loss": 3.3782460212852827, + "tokens_seen": 2411593728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027385858585858585, + "loss": 2.5288, + "theoretical_loss": 3.378238645541785, + "tokens_seen": 2411659264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738383838383838, + "loss": 2.4311, + "theoretical_loss": 3.3782312700548376, + "tokens_seen": 2411724800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002738181818181818, + "loss": 2.8925, + "theoretical_loss": 3.3782238948244245, + "tokens_seen": 2411790336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027379797979797983, + "loss": 3.0368, + "theoretical_loss": 3.37821651985053, + "tokens_seen": 2411855872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737777777777778, + "loss": 2.6842, + "theoretical_loss": 3.3782091451331375, + "tokens_seen": 2411921408 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1358876, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.526784896850586, + "objective/train/theoretical_loss": 3.3782017706722316, + "objective/train/tokens_used": 770846176, + "theoretical_loss": 3.3782017706722316, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027375757575757575, + "loss": 2.6036, + "theoretical_loss": 3.3782017706722316, + "tokens_seen": 2411986944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737373737373737, + "loss": 2.7753, + "theoretical_loss": 3.3781943964677965, + "tokens_seen": 2412052480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002737171717171717, + "loss": 2.7789, + "theoretical_loss": 3.3781870225198163, + "tokens_seen": 2412118016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736969696969697, + "loss": 2.6707, + "theoretical_loss": 3.378179648828275, + "tokens_seen": 2412183552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736767676767677, + "loss": 2.6484, + "theoretical_loss": 3.3781722753931565, + "tokens_seen": 2412249088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027365656565656564, + "loss": 2.7716, + "theoretical_loss": 3.378164902214445, + "tokens_seen": 2412314624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027363636363636365, + "loss": 2.7481, + "theoretical_loss": 3.3781575292921247, + "tokens_seen": 2412380160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002736161616161616, + "loss": 2.7064, + "theoretical_loss": 3.37815015662618, + "tokens_seen": 2412445696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735959595959596, + "loss": 2.792, + "theoretical_loss": 3.3781427842165948, + "tokens_seen": 2412511232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735757575757576, + "loss": 2.7218, + "theoretical_loss": 3.3781354120633527, + "tokens_seen": 2412576768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027355555555555553, + "loss": 2.6739, + "theoretical_loss": 3.378128040166439, + "tokens_seen": 2412642304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027353535353535354, + "loss": 2.6848, + "theoretical_loss": 3.3781206685258365, + "tokens_seen": 2412707840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002735151515151515, + "loss": 2.9586, + "theoretical_loss": 3.3781132971415304, + "tokens_seen": 2412773376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734949494949495, + "loss": 2.8475, + "theoretical_loss": 3.3781059260135042, + "tokens_seen": 2412838912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027347474747474747, + "loss": 2.6335, + "theoretical_loss": 3.3780985551417424, + "tokens_seen": 2412904448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734545454545455, + "loss": 2.5367, + "theoretical_loss": 3.3780911845262285, + "tokens_seen": 2412969984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027343434343434343, + "loss": 2.6929, + "theoretical_loss": 3.378083814166948, + "tokens_seen": 2413035520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002734141414141414, + "loss": 2.6461, + "theoretical_loss": 3.378076444063883, + "tokens_seen": 2413101056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733939393939394, + "loss": 2.5454, + "theoretical_loss": 3.378069074217019, + "tokens_seen": 2413166592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027337373737373736, + "loss": 2.6212, + "theoretical_loss": 3.3780617046263406, + "tokens_seen": 2413232128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027335353535353537, + "loss": 2.6136, + "theoretical_loss": 3.378054335291831, + "tokens_seen": 2413297664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027333333333333333, + "loss": 2.5599, + "theoretical_loss": 3.378046966213474, + "tokens_seen": 2413363200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002733131313131313, + "loss": 2.7636, + "theoretical_loss": 3.378039597391255, + "tokens_seen": 2413428736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732929292929293, + "loss": 2.6608, + "theoretical_loss": 3.378032228825157, + "tokens_seen": 2413494272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732727272727273, + "loss": 2.5598, + "theoretical_loss": 3.378024860515165, + "tokens_seen": 2413559808 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1360381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5831706523895264, + "objective/train/theoretical_loss": 3.3780174924612623, + "objective/train/tokens_used": 772484576, + "theoretical_loss": 3.3780174924612623, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027325252525252526, + "loss": 2.6596, + "theoretical_loss": 3.3780174924612623, + "tokens_seen": 2413625344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002732323232323232, + "loss": 2.8143, + "theoretical_loss": 3.3780101246634335, + "tokens_seen": 2413690880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027321212121212123, + "loss": 2.6653, + "theoretical_loss": 3.3780027571216626, + "tokens_seen": 2413756416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731919191919192, + "loss": 2.535, + "theoretical_loss": 3.3779953898359345, + "tokens_seen": 2413821952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731717171717172, + "loss": 2.535, + "theoretical_loss": 3.3779880228062322, + "tokens_seen": 2413887488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027315151515151516, + "loss": 2.7802, + "theoretical_loss": 3.3779806560325407, + "tokens_seen": 2413953024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731313131313131, + "loss": 2.556, + "theoretical_loss": 3.3779732895148435, + "tokens_seen": 2414018560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002731111111111111, + "loss": 2.5717, + "theoretical_loss": 3.3779659232531256, + "tokens_seen": 2414084096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027309090909090913, + "loss": 2.774, + "theoretical_loss": 3.3779585572473705, + "tokens_seen": 2414149632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002730707070707071, + "loss": 2.8008, + "theoretical_loss": 3.377951191497562, + "tokens_seen": 2414215168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027305050505050505, + "loss": 2.5732, + "theoretical_loss": 3.377943826003685, + "tokens_seen": 2414280704 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273030303030303, + "loss": 2.615, + "theoretical_loss": 3.377936460765724, + "tokens_seen": 2414346240 + }, + { + "epoch": 0.46, + "learning_rate": 0.000273010101010101, + "loss": 2.6698, + "theoretical_loss": 3.377929095783662, + "tokens_seen": 2414411776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027298989898989903, + "loss": 2.534, + "theoretical_loss": 3.3779217310574836, + "tokens_seen": 2414477312 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272969696969697, + "loss": 2.6351, + "theoretical_loss": 3.3779143665871736, + "tokens_seen": 2414542848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027294949494949494, + "loss": 2.647, + "theoretical_loss": 3.3779070023727154, + "tokens_seen": 2414608384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002729292929292929, + "loss": 2.576, + "theoretical_loss": 3.377899638414094, + "tokens_seen": 2414673920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027290909090909096, + "loss": 2.7491, + "theoretical_loss": 3.377892274711292, + "tokens_seen": 2414739456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728888888888889, + "loss": 2.6306, + "theoretical_loss": 3.3778849112642955, + "tokens_seen": 2414804992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728686868686869, + "loss": 2.5783, + "theoretical_loss": 3.377877548073087, + "tokens_seen": 2414870528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027284848484848483, + "loss": 2.7916, + "theoretical_loss": 3.377870185137652, + "tokens_seen": 2414936064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002728282828282828, + "loss": 2.5542, + "theoretical_loss": 3.377862822457974, + "tokens_seen": 2415001600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027280808080808086, + "loss": 2.7836, + "theoretical_loss": 3.3778554600340374, + "tokens_seen": 2415067136 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727878787878788, + "loss": 2.5598, + "theoretical_loss": 3.377848097865826, + "tokens_seen": 2415132672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027276767676767677, + "loss": 2.612, + "theoretical_loss": 3.3778407359533245, + "tokens_seen": 2415198208 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1361019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4516093730926514, + "objective/train/theoretical_loss": 3.3778333742965163, + "objective/train/tokens_used": 774122976, + "theoretical_loss": 3.3778333742965163, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002727474747474747, + "loss": 2.6057, + "theoretical_loss": 3.3778333742965163, + "tokens_seen": 2415263744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027272727272727274, + "loss": 2.7464, + "theoretical_loss": 3.3778260128953868, + "tokens_seen": 2415329280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027270707070707075, + "loss": 2.7255, + "theoretical_loss": 3.377818651749919, + "tokens_seen": 2415394816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726868686868687, + "loss": 2.5599, + "theoretical_loss": 3.3778112908600977, + "tokens_seen": 2415460352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027266666666666666, + "loss": 2.6934, + "theoretical_loss": 3.377803930225907, + "tokens_seen": 2415525888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002726464646464646, + "loss": 2.3837, + "theoretical_loss": 3.377796569847331, + "tokens_seen": 2415591424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027262626262626263, + "loss": 2.875, + "theoretical_loss": 3.377789209724354, + "tokens_seen": 2415656960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027260606060606064, + "loss": 2.568, + "theoretical_loss": 3.37778184985696, + "tokens_seen": 2415722496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725858585858586, + "loss": 2.7209, + "theoretical_loss": 3.3777744902451334, + "tokens_seen": 2415788032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027256565656565656, + "loss": 2.6057, + "theoretical_loss": 3.3777671308888584, + "tokens_seen": 2415853568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027254545454545457, + "loss": 2.5225, + "theoretical_loss": 3.377759771788119, + "tokens_seen": 2415919104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002725252525252525, + "loss": 2.6857, + "theoretical_loss": 3.3777524129428995, + "tokens_seen": 2415984640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027250505050505053, + "loss": 2.6235, + "theoretical_loss": 3.3777450543531837, + "tokens_seen": 2416050176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724848484848485, + "loss": 2.6755, + "theoretical_loss": 3.377737696018957, + "tokens_seen": 2416115712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027246464646464645, + "loss": 2.6791, + "theoretical_loss": 3.377730337940202, + "tokens_seen": 2416181248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027244444444444446, + "loss": 2.6252, + "theoretical_loss": 3.377722980116904, + "tokens_seen": 2416246784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724242424242424, + "loss": 2.6127, + "theoretical_loss": 3.3777156225490472, + "tokens_seen": 2416312320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002724040404040404, + "loss": 2.9111, + "theoretical_loss": 3.377708265236615, + "tokens_seen": 2416377856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723838383838384, + "loss": 2.6974, + "theoretical_loss": 3.3777009081795923, + "tokens_seen": 2416443392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723636363636364, + "loss": 2.6807, + "theoretical_loss": 3.377693551377963, + "tokens_seen": 2416508928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027234343434343435, + "loss": 2.7349, + "theoretical_loss": 3.377686194831712, + "tokens_seen": 2416574464 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723232323232323, + "loss": 2.5242, + "theoretical_loss": 3.377678838540822, + "tokens_seen": 2416640000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002723030303030303, + "loss": 2.5953, + "theoretical_loss": 3.3776714825052787, + "tokens_seen": 2416705536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722828282828283, + "loss": 2.8589, + "theoretical_loss": 3.3776641267250658, + "tokens_seen": 2416771072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722626262626263, + "loss": 2.758, + "theoretical_loss": 3.3776567712001673, + "tokens_seen": 2416836608 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1362090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4934792518615723, + "objective/train/theoretical_loss": 3.3776494159305677, + "objective/train/tokens_used": 775761376, + "theoretical_loss": 3.3776494159305677, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027224242424242424, + "loss": 2.4688, + "theoretical_loss": 3.3776494159305677, + "tokens_seen": 2416902144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722222222222222, + "loss": 2.5952, + "theoretical_loss": 3.377642060916251, + "tokens_seen": 2416967680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002722020202020202, + "loss": 2.6135, + "theoretical_loss": 3.3776347061572016, + "tokens_seen": 2417033216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721818181818182, + "loss": 2.6367, + "theoretical_loss": 3.3776273516534037, + "tokens_seen": 2417098752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721616161616162, + "loss": 2.4144, + "theoretical_loss": 3.377619997404841, + "tokens_seen": 2417164288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027214141414141414, + "loss": 2.6014, + "theoretical_loss": 3.377612643411499, + "tokens_seen": 2417229824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721212121212121, + "loss": 2.8482, + "theoretical_loss": 3.3776052896733604, + "tokens_seen": 2417295360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002721010101010101, + "loss": 2.6922, + "theoretical_loss": 3.3775979361904103, + "tokens_seen": 2417360896 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720808080808081, + "loss": 2.6697, + "theoretical_loss": 3.377590582962633, + "tokens_seen": 2417426432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002720606060606061, + "loss": 2.646, + "theoretical_loss": 3.3775832299900124, + "tokens_seen": 2417491968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027204040404040403, + "loss": 2.6179, + "theoretical_loss": 3.3775758772725326, + "tokens_seen": 2417557504 + }, + { + "epoch": 0.46, + "learning_rate": 0.000272020202020202, + "loss": 2.5671, + "theoretical_loss": 3.377568524810178, + "tokens_seen": 2417623040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027200000000000005, + "loss": 2.6967, + "theoretical_loss": 3.377561172602933, + "tokens_seen": 2417688576 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271979797979798, + "loss": 2.583, + "theoretical_loss": 3.377553820650782, + "tokens_seen": 2417754112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027195959595959597, + "loss": 2.7763, + "theoretical_loss": 3.3775464689537085, + "tokens_seen": 2417819648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719393939393939, + "loss": 2.6015, + "theoretical_loss": 3.3775391175116973, + "tokens_seen": 2417885184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002719191919191919, + "loss": 2.6174, + "theoretical_loss": 3.3775317663247324, + "tokens_seen": 2417950720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027189898989898994, + "loss": 2.6553, + "theoretical_loss": 3.3775244153927986, + "tokens_seen": 2418016256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718787878787879, + "loss": 2.784, + "theoretical_loss": 3.3775170647158794, + "tokens_seen": 2418081792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027185858585858586, + "loss": 2.474, + "theoretical_loss": 3.3775097142939594, + "tokens_seen": 2418147328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002718383838383838, + "loss": 2.609, + "theoretical_loss": 3.3775023641270225, + "tokens_seen": 2418212864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027181818181818177, + "loss": 2.8412, + "theoretical_loss": 3.3774950142150537, + "tokens_seen": 2418278400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027179797979797984, + "loss": 2.6037, + "theoretical_loss": 3.3774876645580365, + "tokens_seen": 2418343936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717777777777778, + "loss": 2.826, + "theoretical_loss": 3.3774803151559554, + "tokens_seen": 2418409472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027175757575757575, + "loss": 2.7238, + "theoretical_loss": 3.3774729660087948, + "tokens_seen": 2418475008 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1362527, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9331196546554565, + "objective/train/theoretical_loss": 3.3774656171165387, + "objective/train/tokens_used": 777399776, + "theoretical_loss": 3.3774656171165387, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717373737373737, + "loss": 2.3388, + "theoretical_loss": 3.3774656171165387, + "tokens_seen": 2418540544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002717171717171717, + "loss": 2.6091, + "theoretical_loss": 3.3774582684791716, + "tokens_seen": 2418606080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027169696969696973, + "loss": 2.3485, + "theoretical_loss": 3.3774509200966776, + "tokens_seen": 2418671616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716767676767677, + "loss": 2.7651, + "theoretical_loss": 3.377443571969041, + "tokens_seen": 2418737152 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027165656565656564, + "loss": 2.6362, + "theoretical_loss": 3.377436224096246, + "tokens_seen": 2418802688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716363636363636, + "loss": 2.5153, + "theoretical_loss": 3.3774288764782767, + "tokens_seen": 2418868224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002716161616161616, + "loss": 2.535, + "theoretical_loss": 3.377421529115118, + "tokens_seen": 2418933760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715959595959596, + "loss": 2.7652, + "theoretical_loss": 3.377414182006753, + "tokens_seen": 2418999296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002715757575757576, + "loss": 2.5551, + "theoretical_loss": 3.377406835153167, + "tokens_seen": 2419064832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027155555555555554, + "loss": 2.7551, + "theoretical_loss": 3.3773994885543446, + "tokens_seen": 2419130368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027153535353535355, + "loss": 2.7158, + "theoretical_loss": 3.377392142210269, + "tokens_seen": 2419195904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027151515151515156, + "loss": 2.7398, + "theoretical_loss": 3.3773847961209245, + "tokens_seen": 2419261440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002714949494949495, + "loss": 2.7767, + "theoretical_loss": 3.377377450286296, + "tokens_seen": 2419326976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027147474747474747, + "loss": 2.6076, + "theoretical_loss": 3.377370104706367, + "tokens_seen": 2419392512 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027145454545454543, + "loss": 2.5536, + "theoretical_loss": 3.377362759381123, + "tokens_seen": 2419458048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027143434343434344, + "loss": 2.5376, + "theoretical_loss": 3.3773554143105473, + "tokens_seen": 2419523584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027141414141414145, + "loss": 2.714, + "theoretical_loss": 3.3773480694946243, + "tokens_seen": 2419589120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713939393939394, + "loss": 2.7979, + "theoretical_loss": 3.3773407249333385, + "tokens_seen": 2419654656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027137373737373737, + "loss": 2.6154, + "theoretical_loss": 3.3773333806266743, + "tokens_seen": 2419720192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002713535353535354, + "loss": 2.6326, + "theoretical_loss": 3.3773260365746154, + "tokens_seen": 2419785728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027133333333333333, + "loss": 2.3522, + "theoretical_loss": 3.3773186927771466, + "tokens_seen": 2419851264 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027131313131313134, + "loss": 2.4848, + "theoretical_loss": 3.377311349234252, + "tokens_seen": 2419916800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712929292929293, + "loss": 2.695, + "theoretical_loss": 3.3773040059459154, + "tokens_seen": 2419982336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027127272727272726, + "loss": 2.8697, + "theoretical_loss": 3.377296662912122, + "tokens_seen": 2420047872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027125252525252527, + "loss": 2.6187, + "theoretical_loss": 3.3772893201328555, + "tokens_seen": 2420113408 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1363833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6757397651672363, + "objective/train/theoretical_loss": 3.3772819776081007, + "objective/train/tokens_used": 779038176, + "theoretical_loss": 3.3772819776081007, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002712323232323232, + "loss": 2.7318, + "theoretical_loss": 3.3772819776081007, + "tokens_seen": 2420178944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027121212121212124, + "loss": 2.388, + "theoretical_loss": 3.3772746353378413, + "tokens_seen": 2420244480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711919191919192, + "loss": 2.7063, + "theoretical_loss": 3.377267293322062, + "tokens_seen": 2420310016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711717171717172, + "loss": 2.8024, + "theoretical_loss": 3.3772599515607467, + "tokens_seen": 2420375552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027115151515151516, + "loss": 2.6185, + "theoretical_loss": 3.3772526100538798, + "tokens_seen": 2420441088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002711313131313131, + "loss": 2.7684, + "theoretical_loss": 3.3772452688014454, + "tokens_seen": 2420506624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027111111111111113, + "loss": 2.71, + "theoretical_loss": 3.377237927803429, + "tokens_seen": 2420572160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710909090909091, + "loss": 2.4926, + "theoretical_loss": 3.377230587059813, + "tokens_seen": 2420637696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002710707070707071, + "loss": 2.7167, + "theoretical_loss": 3.3772232465705834, + "tokens_seen": 2420703232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027105050505050505, + "loss": 2.6871, + "theoretical_loss": 3.3772159063357234, + "tokens_seen": 2420768768 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271030303030303, + "loss": 2.7229, + "theoretical_loss": 3.3772085663552174, + "tokens_seen": 2420834304 + }, + { + "epoch": 0.46, + "learning_rate": 0.000271010101010101, + "loss": 2.4393, + "theoretical_loss": 3.37720122662905, + "tokens_seen": 2420899840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027098989898989903, + "loss": 2.6318, + "theoretical_loss": 3.377193887157206, + "tokens_seen": 2420965376 + }, + { + "epoch": 0.46, + "learning_rate": 0.000270969696969697, + "loss": 2.4918, + "theoretical_loss": 3.3771865479396688, + "tokens_seen": 2421030912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027094949494949495, + "loss": 2.7641, + "theoretical_loss": 3.3771792089764237, + "tokens_seen": 2421096448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709292929292929, + "loss": 2.8292, + "theoretical_loss": 3.3771718702674534, + "tokens_seen": 2421161984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002709090909090909, + "loss": 2.8254, + "theoretical_loss": 3.3771645318127437, + "tokens_seen": 2421227520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708888888888889, + "loss": 2.5961, + "theoretical_loss": 3.3771571936122786, + "tokens_seen": 2421293056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708686868686869, + "loss": 2.6839, + "theoretical_loss": 3.3771498556660418, + "tokens_seen": 2421358592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027084848484848484, + "loss": 2.6253, + "theoretical_loss": 3.3771425179740184, + "tokens_seen": 2421424128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002708282828282828, + "loss": 2.7749, + "theoretical_loss": 3.3771351805361918, + "tokens_seen": 2421489664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027080808080808086, + "loss": 2.6147, + "theoretical_loss": 3.3771278433525476, + "tokens_seen": 2421555200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707878787878788, + "loss": 2.7901, + "theoretical_loss": 3.3771205064230685, + "tokens_seen": 2421620736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707676767676768, + "loss": 2.6646, + "theoretical_loss": 3.3771131697477403, + "tokens_seen": 2421686272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027074747474747473, + "loss": 2.6772, + "theoretical_loss": 3.3771058333265467, + "tokens_seen": 2421751808 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1364650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.376500368118286, + "objective/train/theoretical_loss": 3.3770984971594715, + "objective/train/tokens_used": 780676576, + "theoretical_loss": 3.3770984971594715, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002707272727272727, + "loss": 2.6158, + "theoretical_loss": 3.3770984971594715, + "tokens_seen": 2421817344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027070707070707075, + "loss": 2.685, + "theoretical_loss": 3.3770911612464998, + "tokens_seen": 2421882880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706868686868687, + "loss": 2.4644, + "theoretical_loss": 3.377083825587616, + "tokens_seen": 2421948416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027066666666666667, + "loss": 2.6805, + "theoretical_loss": 3.377076490182804, + "tokens_seen": 2422013952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002706464646464646, + "loss": 2.6588, + "theoretical_loss": 3.3770691550320477, + "tokens_seen": 2422079488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027062626262626264, + "loss": 2.6171, + "theoretical_loss": 3.3770618201353324, + "tokens_seen": 2422145024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027060606060606065, + "loss": 2.7573, + "theoretical_loss": 3.377054485492642, + "tokens_seen": 2422210560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705858585858586, + "loss": 2.8251, + "theoretical_loss": 3.3770471511039606, + "tokens_seen": 2422276096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027056565656565656, + "loss": 2.683, + "theoretical_loss": 3.3770398169692726, + "tokens_seen": 2422341632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002705454545454545, + "loss": 2.5839, + "theoretical_loss": 3.3770324830885627, + "tokens_seen": 2422407168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027052525252525253, + "loss": 2.8516, + "theoretical_loss": 3.377025149461815, + "tokens_seen": 2422472704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027050505050505054, + "loss": 2.7693, + "theoretical_loss": 3.3770178160890136, + "tokens_seen": 2422538240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704848484848485, + "loss": 2.7529, + "theoretical_loss": 3.3770104829701437, + "tokens_seen": 2422603776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027046464646464645, + "loss": 2.6746, + "theoretical_loss": 3.3770031501051885, + "tokens_seen": 2422669312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704444444444444, + "loss": 2.8046, + "theoretical_loss": 3.376995817494133, + "tokens_seen": 2422734848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002704242424242424, + "loss": 2.6917, + "theoretical_loss": 3.376988485136961, + "tokens_seen": 2422800384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027040404040404043, + "loss": 2.6356, + "theoretical_loss": 3.3769811530336575, + "tokens_seen": 2422865920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703838383838384, + "loss": 2.6095, + "theoretical_loss": 3.376973821184207, + "tokens_seen": 2422931456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027036363636363635, + "loss": 2.5866, + "theoretical_loss": 3.3769664895885927, + "tokens_seen": 2422996992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027034343434343436, + "loss": 2.7819, + "theoretical_loss": 3.3769591582468004, + "tokens_seen": 2423062528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703232323232323, + "loss": 2.6247, + "theoretical_loss": 3.376951827158813, + "tokens_seen": 2423128064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002703030303030303, + "loss": 2.5583, + "theoretical_loss": 3.376944496324616, + "tokens_seen": 2423193600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0002702828282828283, + "loss": 2.6041, + "theoretical_loss": 3.3769371657441933, + "tokens_seen": 2423259136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027026262626262624, + "loss": 2.8224, + "theoretical_loss": 3.376929835417529, + "tokens_seen": 2423324672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00027024242424242425, + "loss": 2.4647, + "theoretical_loss": 3.376922505344608, + "tokens_seen": 2423390208 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 1365365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9174885749816895, + "objective/train/theoretical_loss": 3.376915175525414, + "objective/train/tokens_used": 782314976, + "theoretical_loss": 3.376915175525414, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702222222222222, + "loss": 2.7723, + "theoretical_loss": 3.376915175525414, + "tokens_seen": 2423455744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002702020202020202, + "loss": 2.6215, + "theoretical_loss": 3.3769078459599324, + "tokens_seen": 2423521280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701818181818182, + "loss": 2.6692, + "theoretical_loss": 3.3769005166481465, + "tokens_seen": 2423586816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701616161616162, + "loss": 2.9025, + "theoretical_loss": 3.376893187590041, + "tokens_seen": 2423652352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027014141414141414, + "loss": 2.6509, + "theoretical_loss": 3.3768858587856005, + "tokens_seen": 2423717888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701212121212121, + "loss": 2.7163, + "theoretical_loss": 3.376878530234809, + "tokens_seen": 2423783424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002701010101010101, + "loss": 2.5677, + "theoretical_loss": 3.376871201937651, + "tokens_seen": 2423848960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027008080808080807, + "loss": 2.5819, + "theoretical_loss": 3.376863873894111, + "tokens_seen": 2423914496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002700606060606061, + "loss": 2.7024, + "theoretical_loss": 3.3768565461041735, + "tokens_seen": 2423980032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027004040404040404, + "loss": 2.4431, + "theoretical_loss": 3.376849218567822, + "tokens_seen": 2424045568 + }, + { + "epoch": 0.47, + "learning_rate": 0.000270020202020202, + "loss": 2.6959, + "theoretical_loss": 3.376841891285042, + "tokens_seen": 2424111104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00027, + "loss": 2.6332, + "theoretical_loss": 3.3768345642558173, + "tokens_seen": 2424176640 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269979797979798, + "loss": 2.8087, + "theoretical_loss": 3.3768272374801325, + "tokens_seen": 2424242176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026995959595959597, + "loss": 2.6685, + "theoretical_loss": 3.3768199109579715, + "tokens_seen": 2424307712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026993939393939393, + "loss": 2.4679, + "theoretical_loss": 3.376812584689319, + "tokens_seen": 2424373248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002699191919191919, + "loss": 2.7951, + "theoretical_loss": 3.3768052586741595, + "tokens_seen": 2424438784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698989898989899, + "loss": 2.6032, + "theoretical_loss": 3.3767979329124773, + "tokens_seen": 2424504320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698787878787879, + "loss": 2.674, + "theoretical_loss": 3.3767906074042564, + "tokens_seen": 2424569856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026985858585858586, + "loss": 2.6328, + "theoretical_loss": 3.376783282149482, + "tokens_seen": 2424635392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002698383838383838, + "loss": 2.5106, + "theoretical_loss": 3.376775957148138, + "tokens_seen": 2424700928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026981818181818183, + "loss": 2.6909, + "theoretical_loss": 3.376768632400208, + "tokens_seen": 2424766464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026979797979797984, + "loss": 2.6153, + "theoretical_loss": 3.376761307905678, + "tokens_seen": 2424832000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697777777777778, + "loss": 2.837, + "theoretical_loss": 3.376753983664531, + "tokens_seen": 2424897536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026975757575757576, + "loss": 2.4299, + "theoretical_loss": 3.376746659676752, + "tokens_seen": 2424963072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697373737373737, + "loss": 2.7183, + "theoretical_loss": 3.3767393359423257, + "tokens_seen": 2425028608 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1366807, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.489300489425659, + "objective/train/theoretical_loss": 3.3767320124612357, + "objective/train/tokens_used": 783953376, + "theoretical_loss": 3.3767320124612357, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002697171717171717, + "loss": 2.7076, + "theoretical_loss": 3.3767320124612357, + "tokens_seen": 2425094144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026969696969696974, + "loss": 2.5945, + "theoretical_loss": 3.376724689233467, + "tokens_seen": 2425159680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696767676767677, + "loss": 2.6901, + "theoretical_loss": 3.3767173662590038, + "tokens_seen": 2425225216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026965656565656565, + "loss": 2.9168, + "theoretical_loss": 3.3767100435378303, + "tokens_seen": 2425290752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002696363636363636, + "loss": 2.5441, + "theoretical_loss": 3.3767027210699307, + "tokens_seen": 2425356288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026961616161616167, + "loss": 2.5321, + "theoretical_loss": 3.3766953988552904, + "tokens_seen": 2425421824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026959595959595963, + "loss": 2.7277, + "theoretical_loss": 3.376688076893893, + "tokens_seen": 2425487360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695757575757576, + "loss": 2.8846, + "theoretical_loss": 3.376680755185723, + "tokens_seen": 2425552896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026955555555555554, + "loss": 2.6239, + "theoretical_loss": 3.376673433730765, + "tokens_seen": 2425618432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002695353535353535, + "loss": 2.7762, + "theoretical_loss": 3.376666112529003, + "tokens_seen": 2425683968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026951515151515156, + "loss": 2.626, + "theoretical_loss": 3.376658791580422, + "tokens_seen": 2425749504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694949494949495, + "loss": 2.7576, + "theoretical_loss": 3.376651470885006, + "tokens_seen": 2425815040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002694747474747475, + "loss": 2.6068, + "theoretical_loss": 3.3766441504427394, + "tokens_seen": 2425880576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026945454545454543, + "loss": 2.5638, + "theoretical_loss": 3.3766368302536067, + "tokens_seen": 2425946112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026943434343434345, + "loss": 2.6667, + "theoretical_loss": 3.376629510317592, + "tokens_seen": 2426011648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026941414141414146, + "loss": 2.5746, + "theoretical_loss": 3.3766221906346807, + "tokens_seen": 2426077184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002693939393939394, + "loss": 2.6942, + "theoretical_loss": 3.3766148712048563, + "tokens_seen": 2426142720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026937373737373737, + "loss": 2.7507, + "theoretical_loss": 3.376607552028103, + "tokens_seen": 2426208256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026935353535353533, + "loss": 2.5715, + "theoretical_loss": 3.376600233104406, + "tokens_seen": 2426273792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026933333333333334, + "loss": 2.5209, + "theoretical_loss": 3.376592914433749, + "tokens_seen": 2426339328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026931313131313135, + "loss": 2.6236, + "theoretical_loss": 3.3765855960161173, + "tokens_seen": 2426404864 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692929292929293, + "loss": 2.8895, + "theoretical_loss": 3.3765782778514946, + "tokens_seen": 2426470400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026927272727272726, + "loss": 2.4307, + "theoretical_loss": 3.3765709599398654, + "tokens_seen": 2426535936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002692525252525253, + "loss": 2.458, + "theoretical_loss": 3.3765636422812144, + "tokens_seen": 2426601472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026923232323232323, + "loss": 2.5734, + "theoretical_loss": 3.376556324875526, + "tokens_seen": 2426667008 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1367391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.400280475616455, + "objective/train/theoretical_loss": 3.376549007722784, + "objective/train/tokens_used": 785591776, + "theoretical_loss": 3.376549007722784, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026921212121212124, + "loss": 2.5266, + "theoretical_loss": 3.376549007722784, + "tokens_seen": 2426732544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691919191919192, + "loss": 2.6911, + "theoretical_loss": 3.3765416908229735, + "tokens_seen": 2426798080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026917171717171716, + "loss": 2.7773, + "theoretical_loss": 3.376534374176079, + "tokens_seen": 2426863616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026915151515151517, + "loss": 2.7667, + "theoretical_loss": 3.3765270577820847, + "tokens_seen": 2426929152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002691313131313131, + "loss": 2.7598, + "theoretical_loss": 3.3765197416409745, + "tokens_seen": 2426994688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026911111111111114, + "loss": 2.6401, + "theoretical_loss": 3.376512425752734, + "tokens_seen": 2427060224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690909090909091, + "loss": 2.6946, + "theoretical_loss": 3.376505110117346, + "tokens_seen": 2427125760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002690707070707071, + "loss": 2.7637, + "theoretical_loss": 3.3764977947347967, + "tokens_seen": 2427191296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026905050505050506, + "loss": 2.553, + "theoretical_loss": 3.37649047960507, + "tokens_seen": 2427256832 + }, + { + "epoch": 0.47, + "learning_rate": 0.000269030303030303, + "loss": 2.5897, + "theoretical_loss": 3.3764831647281492, + "tokens_seen": 2427322368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026901010101010103, + "loss": 2.6031, + "theoretical_loss": 3.37647585010402, + "tokens_seen": 2427387904 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268989898989899, + "loss": 2.5815, + "theoretical_loss": 3.3764685357326667, + "tokens_seen": 2427453440 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268969696969697, + "loss": 2.6985, + "theoretical_loss": 3.3764612216140732, + "tokens_seen": 2427518976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026894949494949495, + "loss": 2.6633, + "theoretical_loss": 3.376453907748224, + "tokens_seen": 2427584512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689292929292929, + "loss": 2.62, + "theoretical_loss": 3.3764465941351043, + "tokens_seen": 2427650048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002689090909090909, + "loss": 2.8813, + "theoretical_loss": 3.3764392807746977, + "tokens_seen": 2427715584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026888888888888893, + "loss": 2.7644, + "theoretical_loss": 3.376431967666989, + "tokens_seen": 2427781120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688686868686869, + "loss": 2.742, + "theoretical_loss": 3.3764246548119625, + "tokens_seen": 2427846656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026884848484848485, + "loss": 2.5906, + "theoretical_loss": 3.376417342209603, + "tokens_seen": 2427912192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688282828282828, + "loss": 2.6082, + "theoretical_loss": 3.3764100298598945, + "tokens_seen": 2427977728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002688080808080808, + "loss": 2.6635, + "theoretical_loss": 3.3764027177628217, + "tokens_seen": 2428043264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687878787878788, + "loss": 2.4502, + "theoretical_loss": 3.376395405918369, + "tokens_seen": 2428108800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687676767676768, + "loss": 2.6642, + "theoretical_loss": 3.376388094326521, + "tokens_seen": 2428174336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026874747474747474, + "loss": 2.5635, + "theoretical_loss": 3.376380782987262, + "tokens_seen": 2428239872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002687272727272727, + "loss": 2.7295, + "theoretical_loss": 3.3763734719005765, + "tokens_seen": 2428305408 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1368496, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.638556480407715, + "objective/train/theoretical_loss": 3.376366161066449, + "objective/train/tokens_used": 787230176, + "theoretical_loss": 3.376366161066449, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026870707070707076, + "loss": 2.6512, + "theoretical_loss": 3.376366161066449, + "tokens_seen": 2428370944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686868686868687, + "loss": 2.7965, + "theoretical_loss": 3.3763588504848636, + "tokens_seen": 2428436480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686666666666667, + "loss": 2.7021, + "theoretical_loss": 3.376351540155805, + "tokens_seen": 2428502016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026864646464646463, + "loss": 2.5549, + "theoretical_loss": 3.3763442300792583, + "tokens_seen": 2428567552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002686262626262626, + "loss": 2.8748, + "theoretical_loss": 3.376336920255207, + "tokens_seen": 2428633088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026860606060606065, + "loss": 2.5867, + "theoretical_loss": 3.376329610683636, + "tokens_seen": 2428698624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685858585858586, + "loss": 2.5602, + "theoretical_loss": 3.37632230136453, + "tokens_seen": 2428764160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026856565656565657, + "loss": 2.5167, + "theoretical_loss": 3.3763149922978726, + "tokens_seen": 2428829696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685454545454545, + "loss": 2.6221, + "theoretical_loss": 3.376307683483649, + "tokens_seen": 2428895232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002685252525252525, + "loss": 2.9752, + "theoretical_loss": 3.376300374921844, + "tokens_seen": 2428960768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026850505050505055, + "loss": 2.6785, + "theoretical_loss": 3.376293066612441, + "tokens_seen": 2429026304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684848484848485, + "loss": 2.8384, + "theoretical_loss": 3.3762857585554253, + "tokens_seen": 2429091840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026846464646464646, + "loss": 2.7773, + "theoretical_loss": 3.3762784507507813, + "tokens_seen": 2429157376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002684444444444444, + "loss": 2.5581, + "theoretical_loss": 3.376271143198493, + "tokens_seen": 2429222912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026842424242424243, + "loss": 2.6184, + "theoretical_loss": 3.3762638358985457, + "tokens_seen": 2429288448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026840404040404044, + "loss": 2.4371, + "theoretical_loss": 3.376256528850923, + "tokens_seen": 2429353984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683838383838384, + "loss": 2.6864, + "theoretical_loss": 3.37624922205561, + "tokens_seen": 2429419520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026836363636363635, + "loss": 2.3839, + "theoretical_loss": 3.3762419155125905, + "tokens_seen": 2429485056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683434343434343, + "loss": 2.6466, + "theoretical_loss": 3.37623460922185, + "tokens_seen": 2429550592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002683232323232323, + "loss": 2.7899, + "theoretical_loss": 3.3762273031833723, + "tokens_seen": 2429616128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026830303030303033, + "loss": 2.6127, + "theoretical_loss": 3.3762199973971416, + "tokens_seen": 2429681664 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682828282828283, + "loss": 2.5595, + "theoretical_loss": 3.3762126918631434, + "tokens_seen": 2429747200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026826262626262624, + "loss": 2.8326, + "theoretical_loss": 3.376205386581361, + "tokens_seen": 2429812736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026824242424242426, + "loss": 2.7035, + "theoretical_loss": 3.3761980815517796, + "tokens_seen": 2429878272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682222222222222, + "loss": 2.8422, + "theoretical_loss": 3.3761907767743837, + "tokens_seen": 2429943808 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1368772, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.847707986831665, + "objective/train/theoretical_loss": 3.3761834722491577, + "objective/train/tokens_used": 788868576, + "theoretical_loss": 3.3761834722491577, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002682020202020202, + "loss": 2.8037, + "theoretical_loss": 3.3761834722491577, + "tokens_seen": 2430009344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681818181818182, + "loss": 2.8409, + "theoretical_loss": 3.376176167976086, + "tokens_seen": 2430074880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026816161616161614, + "loss": 2.7995, + "theoretical_loss": 3.376168863955153, + "tokens_seen": 2430140416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026814141414141415, + "loss": 2.7766, + "theoretical_loss": 3.3761615601863433, + "tokens_seen": 2430205952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681212121212121, + "loss": 2.4698, + "theoretical_loss": 3.3761542566696416, + "tokens_seen": 2430271488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002681010101010101, + "loss": 2.8403, + "theoretical_loss": 3.376146953405032, + "tokens_seen": 2430337024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680808080808081, + "loss": 2.4636, + "theoretical_loss": 3.3761396503924996, + "tokens_seen": 2430402560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002680606060606061, + "loss": 2.43, + "theoretical_loss": 3.376132347632028, + "tokens_seen": 2430468096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026804040404040404, + "loss": 2.6748, + "theoretical_loss": 3.3761250451236027, + "tokens_seen": 2430533632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026802020202020205, + "loss": 2.5966, + "theoretical_loss": 3.376117742867208, + "tokens_seen": 2430599168 + }, + { + "epoch": 0.47, + "learning_rate": 0.000268, + "loss": 2.7389, + "theoretical_loss": 3.3761104408628277, + "tokens_seen": 2430664704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026797979797979797, + "loss": 2.4434, + "theoretical_loss": 3.376103139110447, + "tokens_seen": 2430730240 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267959595959596, + "loss": 2.581, + "theoretical_loss": 3.37609583761005, + "tokens_seen": 2430795776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026793939393939393, + "loss": 2.6526, + "theoretical_loss": 3.376088536361621, + "tokens_seen": 2430861312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026791919191919194, + "loss": 2.7522, + "theoretical_loss": 3.3760812353651457, + "tokens_seen": 2430926848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678989898989899, + "loss": 2.7179, + "theoretical_loss": 3.3760739346206075, + "tokens_seen": 2430992384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678787878787879, + "loss": 2.7161, + "theoretical_loss": 3.376066634127991, + "tokens_seen": 2431057920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026785858585858587, + "loss": 2.6842, + "theoretical_loss": 3.376059333887281, + "tokens_seen": 2431123456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002678383838383838, + "loss": 2.6135, + "theoretical_loss": 3.3760520338984623, + "tokens_seen": 2431188992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026781818181818184, + "loss": 2.8653, + "theoretical_loss": 3.3760447341615185, + "tokens_seen": 2431254528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677979797979798, + "loss": 2.6669, + "theoretical_loss": 3.376037434676435, + "tokens_seen": 2431320064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677777777777778, + "loss": 2.6223, + "theoretical_loss": 3.3760301354431963, + "tokens_seen": 2431385600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026775757575757576, + "loss": 2.5606, + "theoretical_loss": 3.376022836461786, + "tokens_seen": 2431451136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002677373737373737, + "loss": 2.6102, + "theoretical_loss": 3.37601553773219, + "tokens_seen": 2431516672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026771717171717173, + "loss": 2.7301, + "theoretical_loss": 3.3760082392543915, + "tokens_seen": 2431582208 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1370076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6118152141571045, + "objective/train/theoretical_loss": 3.3760009410283756, + "objective/train/tokens_used": 790506976, + "theoretical_loss": 3.3760009410283756, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026769696969696974, + "loss": 2.6178, + "theoretical_loss": 3.3760009410283756, + "tokens_seen": 2431647744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676767676767677, + "loss": 2.6881, + "theoretical_loss": 3.3759936430541275, + "tokens_seen": 2431713280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026765656565656566, + "loss": 2.4784, + "theoretical_loss": 3.3759863453316306, + "tokens_seen": 2431778816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676363636363636, + "loss": 2.6749, + "theoretical_loss": 3.37597904786087, + "tokens_seen": 2431844352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002676161616161616, + "loss": 2.626, + "theoretical_loss": 3.37597175064183, + "tokens_seen": 2431909888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026759595959595963, + "loss": 2.6757, + "theoretical_loss": 3.375964453674495, + "tokens_seen": 2431975424 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675757575757576, + "loss": 2.5581, + "theoretical_loss": 3.3759571569588505, + "tokens_seen": 2432040960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026755555555555555, + "loss": 2.7045, + "theoretical_loss": 3.37594986049488, + "tokens_seen": 2432106496 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002675353535353535, + "loss": 2.5895, + "theoretical_loss": 3.3759425642825684, + "tokens_seen": 2432172032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026751515151515157, + "loss": 2.7229, + "theoretical_loss": 3.3759352683219004, + "tokens_seen": 2432237568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674949494949495, + "loss": 2.378, + "theoretical_loss": 3.37592797261286, + "tokens_seen": 2432303104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674747474747475, + "loss": 2.8792, + "theoretical_loss": 3.3759206771554324, + "tokens_seen": 2432368640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026745454545454544, + "loss": 2.8684, + "theoretical_loss": 3.3759133819496014, + "tokens_seen": 2432434176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002674343434343434, + "loss": 2.8224, + "theoretical_loss": 3.3759060869953528, + "tokens_seen": 2432499712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026741414141414146, + "loss": 2.5084, + "theoretical_loss": 3.3758987922926695, + "tokens_seen": 2432565248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673939393939394, + "loss": 2.5827, + "theoretical_loss": 3.3758914978415375, + "tokens_seen": 2432630784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002673737373737374, + "loss": 2.9089, + "theoretical_loss": 3.3758842036419408, + "tokens_seen": 2432696320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026735353535353533, + "loss": 2.7469, + "theoretical_loss": 3.3758769096938632, + "tokens_seen": 2432761856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026733333333333334, + "loss": 2.4318, + "theoretical_loss": 3.3758696159972903, + "tokens_seen": 2432827392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026731313131313136, + "loss": 2.6749, + "theoretical_loss": 3.3758623225522064, + "tokens_seen": 2432892928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672929292929293, + "loss": 2.5905, + "theoretical_loss": 3.375855029358596, + "tokens_seen": 2432958464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026727272727272727, + "loss": 2.3743, + "theoretical_loss": 3.3758477364164436, + "tokens_seen": 2433024000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002672525252525252, + "loss": 2.4981, + "theoretical_loss": 3.3758404437257337, + "tokens_seen": 2433089536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026723232323232324, + "loss": 2.6983, + "theoretical_loss": 3.3758331512864506, + "tokens_seen": 2433155072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026721212121212125, + "loss": 2.4659, + "theoretical_loss": 3.3758258590985797, + "tokens_seen": 2433220608 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1370835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7022616863250732, + "objective/train/theoretical_loss": 3.3758185671621046, + "objective/train/tokens_used": 792145376, + "theoretical_loss": 3.3758185671621046, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671919191919192, + "loss": 2.4675, + "theoretical_loss": 3.3758185671621046, + "tokens_seen": 2433286144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026717171717171716, + "loss": 2.7134, + "theoretical_loss": 3.3758112754770107, + "tokens_seen": 2433351680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002671515151515151, + "loss": 2.4614, + "theoretical_loss": 3.375803984043282, + "tokens_seen": 2433417216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026713131313131313, + "loss": 2.768, + "theoretical_loss": 3.375796692860903, + "tokens_seen": 2433482752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026711111111111114, + "loss": 2.5057, + "theoretical_loss": 3.3757894019298584, + "tokens_seen": 2433548288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002670909090909091, + "loss": 2.8262, + "theoretical_loss": 3.375782111250133, + "tokens_seen": 2433613824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026707070707070705, + "loss": 2.5684, + "theoretical_loss": 3.3757748208217118, + "tokens_seen": 2433679360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026705050505050507, + "loss": 2.5996, + "theoretical_loss": 3.3757675306445782, + "tokens_seen": 2433744896 + }, + { + "epoch": 0.47, + "learning_rate": 0.000267030303030303, + "loss": 2.8667, + "theoretical_loss": 3.3757602407187175, + "tokens_seen": 2433810432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026701010101010103, + "loss": 2.6141, + "theoretical_loss": 3.3757529510441144, + "tokens_seen": 2433875968 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266989898989899, + "loss": 2.5718, + "theoretical_loss": 3.375745661620753, + "tokens_seen": 2433941504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026696969696969695, + "loss": 2.7555, + "theoretical_loss": 3.375738372448618, + "tokens_seen": 2434007040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026694949494949496, + "loss": 2.6357, + "theoretical_loss": 3.375731083527694, + "tokens_seen": 2434072576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669292929292929, + "loss": 2.6386, + "theoretical_loss": 3.375723794857966, + "tokens_seen": 2434138112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002669090909090909, + "loss": 2.533, + "theoretical_loss": 3.375716506439418, + "tokens_seen": 2434203648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668888888888889, + "loss": 2.5688, + "theoretical_loss": 3.375709218272035, + "tokens_seen": 2434269184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668686868686869, + "loss": 2.6806, + "theoretical_loss": 3.3757019303558016, + "tokens_seen": 2434334720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026684848484848485, + "loss": 2.8696, + "theoretical_loss": 3.3756946426907017, + "tokens_seen": 2434400256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668282828282828, + "loss": 2.4331, + "theoretical_loss": 3.375687355276721, + "tokens_seen": 2434465792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002668080808080808, + "loss": 2.4444, + "theoretical_loss": 3.375680068113843, + "tokens_seen": 2434531328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667878787878788, + "loss": 2.9067, + "theoretical_loss": 3.375672781202053, + "tokens_seen": 2434596864 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667676767676768, + "loss": 2.5112, + "theoretical_loss": 3.375665494541335, + "tokens_seen": 2434662400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026674747474747474, + "loss": 2.6178, + "theoretical_loss": 3.375658208131674, + "tokens_seen": 2434727936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667272727272727, + "loss": 2.6463, + "theoretical_loss": 3.3756509219730546, + "tokens_seen": 2434793472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002667070707070707, + "loss": 2.5035, + "theoretical_loss": 3.3756436360654614, + "tokens_seen": 2434859008 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1372111, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.614074945449829, + "objective/train/theoretical_loss": 3.375636350408879, + "objective/train/tokens_used": 793783776, + "theoretical_loss": 3.375636350408879, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666868686868687, + "loss": 2.7014, + "theoretical_loss": 3.375636350408879, + "tokens_seen": 2434924544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666666666666667, + "loss": 2.7359, + "theoretical_loss": 3.3756290650032916, + "tokens_seen": 2434990080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026664646464646464, + "loss": 2.6714, + "theoretical_loss": 3.3756217798486845, + "tokens_seen": 2435055616 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666262626262626, + "loss": 2.6073, + "theoretical_loss": 3.3756144949450415, + "tokens_seen": 2435121152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002666060606060606, + "loss": 2.6923, + "theoretical_loss": 3.375607210292348, + "tokens_seen": 2435186688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665858585858586, + "loss": 2.6753, + "theoretical_loss": 3.375599925890588, + "tokens_seen": 2435252224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026656565656565657, + "loss": 2.4836, + "theoretical_loss": 3.3755926417397464, + "tokens_seen": 2435317760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026654545454545453, + "loss": 2.4139, + "theoretical_loss": 3.3755853578398076, + "tokens_seen": 2435383296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002665252525252525, + "loss": 2.5256, + "theoretical_loss": 3.375578074190756, + "tokens_seen": 2435448832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026650505050505055, + "loss": 2.5242, + "theoretical_loss": 3.375570790792577, + "tokens_seen": 2435514368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664848484848485, + "loss": 2.751, + "theoretical_loss": 3.375563507645255, + "tokens_seen": 2435579904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026646464646464646, + "loss": 2.683, + "theoretical_loss": 3.375556224748774, + "tokens_seen": 2435645440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664444444444444, + "loss": 2.8772, + "theoretical_loss": 3.375548942103119, + "tokens_seen": 2435710976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002664242424242424, + "loss": 2.527, + "theoretical_loss": 3.3755416597082744, + "tokens_seen": 2435776512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026640404040404044, + "loss": 2.5502, + "theoretical_loss": 3.3755343775642253, + "tokens_seen": 2435842048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663838383838384, + "loss": 2.7017, + "theoretical_loss": 3.3755270956709555, + "tokens_seen": 2435907584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026636363636363636, + "loss": 2.6485, + "theoretical_loss": 3.3755198140284506, + "tokens_seen": 2435973120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663434343434343, + "loss": 2.4962, + "theoretical_loss": 3.3755125326366944, + "tokens_seen": 2436038656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002663232323232324, + "loss": 2.7054, + "theoretical_loss": 3.3755052514956723, + "tokens_seen": 2436104192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026630303030303034, + "loss": 2.6929, + "theoretical_loss": 3.3754979706053683, + "tokens_seen": 2436169728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662828282828283, + "loss": 2.5719, + "theoretical_loss": 3.3754906899657673, + "tokens_seen": 2436235264 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026626262626262625, + "loss": 2.7251, + "theoretical_loss": 3.3754834095768533, + "tokens_seen": 2436300800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002662424242424242, + "loss": 2.7352, + "theoretical_loss": 3.375476129438612, + "tokens_seen": 2436366336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026622222222222227, + "loss": 2.7711, + "theoretical_loss": 3.3754688495510274, + "tokens_seen": 2436431872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026620202020202023, + "loss": 2.6489, + "theoretical_loss": 3.3754615699140844, + "tokens_seen": 2436497408 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1372668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8316433429718018, + "objective/train/theoretical_loss": 3.3754542905277667, + "objective/train/tokens_used": 795422176, + "theoretical_loss": 3.3754542905277667, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661818181818182, + "loss": 2.686, + "theoretical_loss": 3.3754542905277667, + "tokens_seen": 2436562944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026616161616161614, + "loss": 2.6666, + "theoretical_loss": 3.3754470113920605, + "tokens_seen": 2436628480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026614141414141415, + "loss": 2.5717, + "theoretical_loss": 3.3754397325069494, + "tokens_seen": 2436694016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026612121212121217, + "loss": 2.5032, + "theoretical_loss": 3.375432453872418, + "tokens_seen": 2436759552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002661010101010101, + "loss": 2.6572, + "theoretical_loss": 3.3754251754884512, + "tokens_seen": 2436825088 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002660808080808081, + "loss": 2.5147, + "theoretical_loss": 3.3754178973550335, + "tokens_seen": 2436890624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026606060606060604, + "loss": 2.6366, + "theoretical_loss": 3.37541061947215, + "tokens_seen": 2436956160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026604040404040405, + "loss": 2.6031, + "theoretical_loss": 3.375403341839785, + "tokens_seen": 2437021696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026602020202020206, + "loss": 2.6577, + "theoretical_loss": 3.375396064457923, + "tokens_seen": 2437087232 + }, + { + "epoch": 0.47, + "learning_rate": 0.000266, + "loss": 2.7528, + "theoretical_loss": 3.3753887873265485, + "tokens_seen": 2437152768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026597979797979797, + "loss": 2.735, + "theoretical_loss": 3.3753815104456466, + "tokens_seen": 2437218304 + }, + { + "epoch": 0.47, + "learning_rate": 0.000265959595959596, + "loss": 2.4172, + "theoretical_loss": 3.3753742338152017, + "tokens_seen": 2437283840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026593939393939394, + "loss": 2.5809, + "theoretical_loss": 3.3753669574351983, + "tokens_seen": 2437349376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026591919191919195, + "loss": 2.5613, + "theoretical_loss": 3.3753596813056213, + "tokens_seen": 2437414912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658989898989899, + "loss": 2.6226, + "theoretical_loss": 3.375352405426456, + "tokens_seen": 2437480448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026587878787878786, + "loss": 2.6026, + "theoretical_loss": 3.3753451297976858, + "tokens_seen": 2437545984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002658585858585859, + "loss": 2.7801, + "theoretical_loss": 3.375337854419296, + "tokens_seen": 2437611520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026583838383838383, + "loss": 2.4686, + "theoretical_loss": 3.375330579291271, + "tokens_seen": 2437677056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026581818181818184, + "loss": 2.3819, + "theoretical_loss": 3.3753233044135955, + "tokens_seen": 2437742592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657979797979798, + "loss": 2.6188, + "theoretical_loss": 3.375316029786254, + "tokens_seen": 2437808128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657777777777778, + "loss": 2.681, + "theoretical_loss": 3.375308755409232, + "tokens_seen": 2437873664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026575757575757577, + "loss": 2.6494, + "theoretical_loss": 3.3753014812825133, + "tokens_seen": 2437939200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002657373737373737, + "loss": 2.5262, + "theoretical_loss": 3.3752942074060828, + "tokens_seen": 2438004736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026571717171717174, + "loss": 2.6191, + "theoretical_loss": 3.3752869337799254, + "tokens_seen": 2438070272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656969696969697, + "loss": 2.6981, + "theoretical_loss": 3.3752796604040256, + "tokens_seen": 2438135808 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1374127, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6285626888275146, + "objective/train/theoretical_loss": 3.375272387278368, + "objective/train/tokens_used": 797060576, + "theoretical_loss": 3.375272387278368, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656767676767677, + "loss": 2.5575, + "theoretical_loss": 3.375272387278368, + "tokens_seen": 2438201344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026565656565656566, + "loss": 2.4138, + "theoretical_loss": 3.3752651144029366, + "tokens_seen": 2438266880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002656363636363636, + "loss": 2.6926, + "theoretical_loss": 3.3752578417777177, + "tokens_seen": 2438332416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026561616161616163, + "loss": 2.5198, + "theoretical_loss": 3.375250569402694, + "tokens_seen": 2438397952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026559595959595964, + "loss": 2.6001, + "theoretical_loss": 3.3752432972778523, + "tokens_seen": 2438463488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655757575757576, + "loss": 2.6874, + "theoretical_loss": 3.3752360254031757, + "tokens_seen": 2438529024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026555555555555555, + "loss": 2.6905, + "theoretical_loss": 3.375228753778649, + "tokens_seen": 2438594560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655353535353535, + "loss": 2.6227, + "theoretical_loss": 3.3752214824042572, + "tokens_seen": 2438660096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002655151515151515, + "loss": 2.8413, + "theoretical_loss": 3.375214211279985, + "tokens_seen": 2438725632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026549494949494953, + "loss": 2.5149, + "theoretical_loss": 3.3752069404058176, + "tokens_seen": 2438791168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654747474747475, + "loss": 2.7769, + "theoretical_loss": 3.375199669781739, + "tokens_seen": 2438856704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026545454545454545, + "loss": 2.8397, + "theoretical_loss": 3.3751923994077337, + "tokens_seen": 2438922240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002654343434343434, + "loss": 2.5924, + "theoretical_loss": 3.3751851292837864, + "tokens_seen": 2438987776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026541414141414147, + "loss": 2.6145, + "theoretical_loss": 3.3751778594098822, + "tokens_seen": 2439053312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653939393939394, + "loss": 2.829, + "theoretical_loss": 3.375170589786006, + "tokens_seen": 2439118848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653737373737374, + "loss": 2.8528, + "theoretical_loss": 3.375163320412142, + "tokens_seen": 2439184384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026535353535353534, + "loss": 2.466, + "theoretical_loss": 3.3751560512882746, + "tokens_seen": 2439249920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002653333333333333, + "loss": 2.5624, + "theoretical_loss": 3.3751487824143895, + "tokens_seen": 2439315456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026531313131313136, + "loss": 2.7537, + "theoretical_loss": 3.3751415137904703, + "tokens_seen": 2439380992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652929292929293, + "loss": 2.4824, + "theoretical_loss": 3.3751342454165023, + "tokens_seen": 2439446528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652727272727273, + "loss": 2.6175, + "theoretical_loss": 3.3751269772924704, + "tokens_seen": 2439512064 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026525252525252523, + "loss": 2.6148, + "theoretical_loss": 3.3751197094183585, + "tokens_seen": 2439577600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002652323232323232, + "loss": 2.6776, + "theoretical_loss": 3.375112441794152, + "tokens_seen": 2439643136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026521212121212125, + "loss": 2.734, + "theoretical_loss": 3.3751051744198355, + "tokens_seen": 2439708672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0002651919191919192, + "loss": 2.6683, + "theoretical_loss": 3.375097907295393, + "tokens_seen": 2439774208 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 1374810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.197831153869629, + "objective/train/theoretical_loss": 3.37509064042081, + "objective/train/tokens_used": 798698976, + "theoretical_loss": 3.37509064042081, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00026517171717171717, + "loss": 2.4762, + "theoretical_loss": 3.37509064042081, + "tokens_seen": 2439839744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002651515151515151, + "loss": 2.7099, + "theoretical_loss": 3.375083373796071, + "tokens_seen": 2439905280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026513131313131314, + "loss": 2.7057, + "theoretical_loss": 3.3750761074211604, + "tokens_seen": 2439970816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026511111111111115, + "loss": 2.6094, + "theoretical_loss": 3.3750688412960637, + "tokens_seen": 2440036352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002650909090909091, + "loss": 2.6002, + "theoretical_loss": 3.3750615754207645, + "tokens_seen": 2440101888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026507070707070706, + "loss": 2.7744, + "theoretical_loss": 3.375054309795248, + "tokens_seen": 2440167424 + }, + { + "epoch": 0.48, + "learning_rate": 0.000265050505050505, + "loss": 2.7427, + "theoretical_loss": 3.375047044419499, + "tokens_seen": 2440232960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026503030303030303, + "loss": 2.6152, + "theoretical_loss": 3.3750397792935023, + "tokens_seen": 2440298496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026501010101010104, + "loss": 2.7586, + "theoretical_loss": 3.375032514417242, + "tokens_seen": 2440364032 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264989898989899, + "loss": 2.648, + "theoretical_loss": 3.375025249790704, + "tokens_seen": 2440429568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026496969696969695, + "loss": 2.7007, + "theoretical_loss": 3.375017985413872, + "tokens_seen": 2440495104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026494949494949496, + "loss": 2.452, + "theoretical_loss": 3.375010721286731, + "tokens_seen": 2440560640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002649292929292929, + "loss": 2.5549, + "theoretical_loss": 3.3750034574092656, + "tokens_seen": 2440626176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026490909090909093, + "loss": 2.5153, + "theoretical_loss": 3.3749961937814605, + "tokens_seen": 2440691712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648888888888889, + "loss": 2.527, + "theoretical_loss": 3.3749889304033007, + "tokens_seen": 2440757248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026486868686868685, + "loss": 2.7849, + "theoretical_loss": 3.3749816672747706, + "tokens_seen": 2440822784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026484848484848486, + "loss": 2.757, + "theoretical_loss": 3.3749744043958554, + "tokens_seen": 2440888320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648282828282828, + "loss": 2.7385, + "theoretical_loss": 3.374967141766539, + "tokens_seen": 2440953856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002648080808080808, + "loss": 2.8366, + "theoretical_loss": 3.374959879386807, + "tokens_seen": 2441019392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647878787878788, + "loss": 2.5328, + "theoretical_loss": 3.3749526172566435, + "tokens_seen": 2441084928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647676767676768, + "loss": 2.5468, + "theoretical_loss": 3.3749453553760334, + "tokens_seen": 2441150464 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026474747474747475, + "loss": 2.8365, + "theoretical_loss": 3.374938093744962, + "tokens_seen": 2441216000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647272727272727, + "loss": 2.7959, + "theoretical_loss": 3.374930832363413, + "tokens_seen": 2441281536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002647070707070707, + "loss": 2.4396, + "theoretical_loss": 3.3749235712313714, + "tokens_seen": 2441347072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646868686868687, + "loss": 2.7288, + "theoretical_loss": 3.3749163103488224, + "tokens_seen": 2441412608 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1375482, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.251352310180664, + "objective/train/theoretical_loss": 3.374909049715751, + "objective/train/tokens_used": 800337376, + "theoretical_loss": 3.374909049715751, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646666666666667, + "loss": 2.6938, + "theoretical_loss": 3.374909049715751, + "tokens_seen": 2441478144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026464646464646464, + "loss": 2.6219, + "theoretical_loss": 3.3749017893321405, + "tokens_seen": 2441543680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026462626262626265, + "loss": 2.4999, + "theoretical_loss": 3.3748945291979773, + "tokens_seen": 2441609216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002646060606060606, + "loss": 2.3704, + "theoretical_loss": 3.374887269313245, + "tokens_seen": 2441674752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645858585858586, + "loss": 2.5775, + "theoretical_loss": 3.374880009677929, + "tokens_seen": 2441740288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645656565656566, + "loss": 2.759, + "theoretical_loss": 3.3748727502920133, + "tokens_seen": 2441805824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026454545454545453, + "loss": 2.9025, + "theoretical_loss": 3.3748654911554836, + "tokens_seen": 2441871360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026452525252525255, + "loss": 2.745, + "theoretical_loss": 3.374858232268324, + "tokens_seen": 2441936896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002645050505050505, + "loss": 2.5834, + "theoretical_loss": 3.3748509736305192, + "tokens_seen": 2442002432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002644848484848485, + "loss": 2.556, + "theoretical_loss": 3.3748437152420543, + "tokens_seen": 2442067968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026446464646464647, + "loss": 2.7336, + "theoretical_loss": 3.374836457102914, + "tokens_seen": 2442133504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026444444444444443, + "loss": 2.6785, + "theoretical_loss": 3.3748291992130826, + "tokens_seen": 2442199040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026442424242424244, + "loss": 2.6817, + "theoretical_loss": 3.374821941572545, + "tokens_seen": 2442264576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026440404040404045, + "loss": 2.6878, + "theoretical_loss": 3.3748146841812865, + "tokens_seen": 2442330112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643838383838384, + "loss": 2.7571, + "theoretical_loss": 3.3748074270392916, + "tokens_seen": 2442395648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026436363636363636, + "loss": 2.7643, + "theoretical_loss": 3.3748001701465444, + "tokens_seen": 2442461184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002643434343434343, + "loss": 2.7752, + "theoretical_loss": 3.374792913503031, + "tokens_seen": 2442526720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026432323232323233, + "loss": 2.6156, + "theoretical_loss": 3.3747856571087342, + "tokens_seen": 2442592256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026430303030303034, + "loss": 2.5661, + "theoretical_loss": 3.3747784009636406, + "tokens_seen": 2442657792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642828282828283, + "loss": 2.7237, + "theoretical_loss": 3.3747711450677342, + "tokens_seen": 2442723328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026426262626262626, + "loss": 2.8578, + "theoretical_loss": 3.3747638894209997, + "tokens_seen": 2442788864 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642424242424242, + "loss": 2.7373, + "theoretical_loss": 3.374756634023422, + "tokens_seen": 2442854400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002642222222222223, + "loss": 2.7473, + "theoretical_loss": 3.374749378874985, + "tokens_seen": 2442919936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026420202020202023, + "loss": 2.7117, + "theoretical_loss": 3.3747421239756754, + "tokens_seen": 2442985472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641818181818182, + "loss": 2.538, + "theoretical_loss": 3.374734869325476, + "tokens_seen": 2443051008 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4994184970855713, + "objective/train/theoretical_loss": 3.374727614924373, + "objective/train/tokens_used": 801975776, + "theoretical_loss": 3.374727614924373, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026416161616161615, + "loss": 2.6352, + "theoretical_loss": 3.374727614924373, + "tokens_seen": 2443116544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002641414141414141, + "loss": 2.5784, + "theoretical_loss": 3.3747203607723506, + "tokens_seen": 2443182080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026412121212121217, + "loss": 2.6467, + "theoretical_loss": 3.374713106869393, + "tokens_seen": 2443247616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026410101010101013, + "loss": 2.4954, + "theoretical_loss": 3.3747058532154854, + "tokens_seen": 2443313152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002640808080808081, + "loss": 2.6647, + "theoretical_loss": 3.374698599810613, + "tokens_seen": 2443378688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026406060606060604, + "loss": 2.6428, + "theoretical_loss": 3.3746913466547603, + "tokens_seen": 2443444224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026404040404040405, + "loss": 2.8303, + "theoretical_loss": 3.374684093747912, + "tokens_seen": 2443509760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026402020202020206, + "loss": 2.7631, + "theoretical_loss": 3.3746768410900527, + "tokens_seen": 2443575296 + }, + { + "epoch": 0.48, + "learning_rate": 0.000264, + "loss": 2.7202, + "theoretical_loss": 3.3746695886811677, + "tokens_seen": 2443640832 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263979797979798, + "loss": 2.704, + "theoretical_loss": 3.374662336521241, + "tokens_seen": 2443706368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026395959595959593, + "loss": 2.6498, + "theoretical_loss": 3.374655084610258, + "tokens_seen": 2443771904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026393939393939395, + "loss": 2.8145, + "theoretical_loss": 3.3746478329482033, + "tokens_seen": 2443837440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026391919191919196, + "loss": 2.9156, + "theoretical_loss": 3.3746405815350613, + "tokens_seen": 2443902976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638989898989899, + "loss": 2.6018, + "theoretical_loss": 3.3746333303708176, + "tokens_seen": 2443968512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026387878787878787, + "loss": 2.7803, + "theoretical_loss": 3.3746260794554566, + "tokens_seen": 2444034048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002638585858585858, + "loss": 2.8377, + "theoretical_loss": 3.3746188287889627, + "tokens_seen": 2444099584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026383838383838384, + "loss": 2.815, + "theoretical_loss": 3.3746115783713213, + "tokens_seen": 2444165120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026381818181818185, + "loss": 2.6358, + "theoretical_loss": 3.3746043282025164, + "tokens_seen": 2444230656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637979797979798, + "loss": 2.8963, + "theoretical_loss": 3.3745970782825334, + "tokens_seen": 2444296192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026377777777777776, + "loss": 2.7634, + "theoretical_loss": 3.3745898286113576, + "tokens_seen": 2444361728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002637575757575758, + "loss": 2.9524, + "theoretical_loss": 3.3745825791889725, + "tokens_seen": 2444427264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026373737373737373, + "loss": 2.736, + "theoretical_loss": 3.3745753300153636, + "tokens_seen": 2444492800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026371717171717174, + "loss": 2.5302, + "theoretical_loss": 3.3745680810905156, + "tokens_seen": 2444558336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636969696969697, + "loss": 2.6615, + "theoretical_loss": 3.3745608324144136, + "tokens_seen": 2444623872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026367676767676766, + "loss": 2.6066, + "theoretical_loss": 3.374553583987042, + "tokens_seen": 2444689408 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1376516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6544651985168457, + "objective/train/theoretical_loss": 3.374546335808386, + "objective/train/tokens_used": 803614176, + "theoretical_loss": 3.374546335808386, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026365656565656567, + "loss": 2.6818, + "theoretical_loss": 3.374546335808386, + "tokens_seen": 2444754944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002636363636363636, + "loss": 2.7242, + "theoretical_loss": 3.3745390878784294, + "tokens_seen": 2444820480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026361616161616163, + "loss": 2.8005, + "theoretical_loss": 3.3745318401971582, + "tokens_seen": 2444886016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635959595959596, + "loss": 2.8178, + "theoretical_loss": 3.3745245927645566, + "tokens_seen": 2444951552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635757575757576, + "loss": 2.7058, + "theoretical_loss": 3.37451734558061, + "tokens_seen": 2445017088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026355555555555556, + "loss": 2.7798, + "theoretical_loss": 3.3745100986453025, + "tokens_seen": 2445082624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635353535353535, + "loss": 2.7403, + "theoretical_loss": 3.374502851958619, + "tokens_seen": 2445148160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002635151515151515, + "loss": 2.7218, + "theoretical_loss": 3.3744956055205444, + "tokens_seen": 2445213696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634949494949495, + "loss": 2.7998, + "theoretical_loss": 3.3744883593310635, + "tokens_seen": 2445279232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634747474747475, + "loss": 2.8145, + "theoretical_loss": 3.3744811133901615, + "tokens_seen": 2445344768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026345454545454545, + "loss": 2.6881, + "theoretical_loss": 3.3744738676978225, + "tokens_seen": 2445410304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634343434343434, + "loss": 2.6843, + "theoretical_loss": 3.374466622254032, + "tokens_seen": 2445475840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002634141414141414, + "loss": 2.5507, + "theoretical_loss": 3.3744593770587743, + "tokens_seen": 2445541376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026339393939393943, + "loss": 2.4827, + "theoretical_loss": 3.3744521321120344, + "tokens_seen": 2445606912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633737373737374, + "loss": 2.7611, + "theoretical_loss": 3.374444887413797, + "tokens_seen": 2445672448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026335353535353534, + "loss": 2.6534, + "theoretical_loss": 3.3744376429640477, + "tokens_seen": 2445737984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633333333333333, + "loss": 2.7409, + "theoretical_loss": 3.37443039876277, + "tokens_seen": 2445803520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002633131313131313, + "loss": 3.0244, + "theoretical_loss": 3.3744231548099495, + "tokens_seen": 2445869056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632929292929293, + "loss": 2.8442, + "theoretical_loss": 3.3744159111055714, + "tokens_seen": 2445934592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632727272727273, + "loss": 2.7396, + "theoretical_loss": 3.3744086676496194, + "tokens_seen": 2446000128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026325252525252524, + "loss": 2.6554, + "theoretical_loss": 3.3744014244420795, + "tokens_seen": 2446065664 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002632323232323232, + "loss": 2.5858, + "theoretical_loss": 3.3743941814829355, + "tokens_seen": 2446131200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026321212121212126, + "loss": 2.7081, + "theoretical_loss": 3.374386938772173, + "tokens_seen": 2446196736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631919191919192, + "loss": 2.6956, + "theoretical_loss": 3.3743796963097763, + "tokens_seen": 2446262272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631717171717172, + "loss": 2.5185, + "theoretical_loss": 3.374372454095731, + "tokens_seen": 2446327808 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1377582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4908623695373535, + "objective/train/theoretical_loss": 3.3743652121300207, + "objective/train/tokens_used": 805252576, + "theoretical_loss": 3.3743652121300207, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026315151515151513, + "loss": 2.5474, + "theoretical_loss": 3.3743652121300207, + "tokens_seen": 2446393344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002631313131313131, + "loss": 2.7351, + "theoretical_loss": 3.3743579704126314, + "tokens_seen": 2446458880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026311111111111115, + "loss": 2.8404, + "theoretical_loss": 3.374350728943547, + "tokens_seen": 2446524416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002630909090909091, + "loss": 2.6257, + "theoretical_loss": 3.3743434877227534, + "tokens_seen": 2446589952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026307070707070707, + "loss": 2.7464, + "theoretical_loss": 3.3743362467502345, + "tokens_seen": 2446655488 + }, + { + "epoch": 0.48, + "learning_rate": 0.000263050505050505, + "loss": 2.5903, + "theoretical_loss": 3.3743290060259756, + "tokens_seen": 2446721024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026303030303030303, + "loss": 2.727, + "theoretical_loss": 3.3743217655499613, + "tokens_seen": 2446786560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026301010101010104, + "loss": 2.8117, + "theoretical_loss": 3.374314525322177, + "tokens_seen": 2446852096 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262989898989899, + "loss": 2.9643, + "theoretical_loss": 3.3743072853426064, + "tokens_seen": 2446917632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026296969696969696, + "loss": 2.9336, + "theoretical_loss": 3.3743000456112355, + "tokens_seen": 2446983168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002629494949494949, + "loss": 2.8953, + "theoretical_loss": 3.3742928061280484, + "tokens_seen": 2447048704 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262929292929293, + "loss": 2.8571, + "theoretical_loss": 3.3742855668930307, + "tokens_seen": 2447114240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026290909090909094, + "loss": 2.6748, + "theoretical_loss": 3.374278327906166, + "tokens_seen": 2447179776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628888888888889, + "loss": 2.8584, + "theoretical_loss": 3.374271089167441, + "tokens_seen": 2447245312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026286868686868685, + "loss": 2.7695, + "theoretical_loss": 3.3742638506768383, + "tokens_seen": 2447310848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026284848484848486, + "loss": 2.7226, + "theoretical_loss": 3.3742566124343445, + "tokens_seen": 2447376384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002628282828282829, + "loss": 2.5825, + "theoretical_loss": 3.3742493744399438, + "tokens_seen": 2447441920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026280808080808083, + "loss": 2.7459, + "theoretical_loss": 3.3742421366936215, + "tokens_seen": 2447507456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627878787878788, + "loss": 2.7468, + "theoretical_loss": 3.3742348991953617, + "tokens_seen": 2447572992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026276767676767674, + "loss": 2.5776, + "theoretical_loss": 3.3742276619451497, + "tokens_seen": 2447638528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026274747474747475, + "loss": 2.8217, + "theoretical_loss": 3.3742204249429704, + "tokens_seen": 2447704064 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026272727272727277, + "loss": 2.8285, + "theoretical_loss": 3.374213188188808, + "tokens_seen": 2447769600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002627070707070707, + "loss": 2.5063, + "theoretical_loss": 3.374205951682649, + "tokens_seen": 2447835136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626868686868687, + "loss": 2.9214, + "theoretical_loss": 3.3741987154244764, + "tokens_seen": 2447900672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626666666666667, + "loss": 2.9096, + "theoretical_loss": 3.374191479414276, + "tokens_seen": 2447966208 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1378304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.426485300064087, + "objective/train/theoretical_loss": 3.3741842436520324, + "objective/train/tokens_used": 806890976, + "theoretical_loss": 3.3741842436520324, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026264646464646465, + "loss": 2.8157, + "theoretical_loss": 3.3741842436520324, + "tokens_seen": 2448031744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026262626262626266, + "loss": 2.5923, + "theoretical_loss": 3.374177008137731, + "tokens_seen": 2448097280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002626060606060606, + "loss": 2.7587, + "theoretical_loss": 3.3741697728713556, + "tokens_seen": 2448162816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026258585858585857, + "loss": 2.5472, + "theoretical_loss": 3.374162537852892, + "tokens_seen": 2448228352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625656565656566, + "loss": 2.7153, + "theoretical_loss": 3.374155303082325, + "tokens_seen": 2448293888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026254545454545454, + "loss": 2.7574, + "theoretical_loss": 3.374148068559639, + "tokens_seen": 2448359424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026252525252525255, + "loss": 2.8224, + "theoretical_loss": 3.374140834284819, + "tokens_seen": 2448424960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002625050505050505, + "loss": 2.7967, + "theoretical_loss": 3.37413360025785, + "tokens_seen": 2448490496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624848484848485, + "loss": 2.8549, + "theoretical_loss": 3.3741263664787176, + "tokens_seen": 2448556032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624646464646465, + "loss": 2.8201, + "theoretical_loss": 3.3741191329474054, + "tokens_seen": 2448621568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026244444444444443, + "loss": 2.7808, + "theoretical_loss": 3.3741118996638986, + "tokens_seen": 2448687104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026242424242424244, + "loss": 2.5562, + "theoretical_loss": 3.3741046666281824, + "tokens_seen": 2448752640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002624040404040404, + "loss": 2.7818, + "theoretical_loss": 3.374097433840242, + "tokens_seen": 2448818176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623838383838384, + "loss": 2.7154, + "theoretical_loss": 3.3740902013000613, + "tokens_seen": 2448883712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026236363636363637, + "loss": 2.4812, + "theoretical_loss": 3.374082969007626, + "tokens_seen": 2448949248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002623434343434343, + "loss": 2.7968, + "theoretical_loss": 3.374075736962921, + "tokens_seen": 2449014784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026232323232323234, + "loss": 2.7124, + "theoretical_loss": 3.3740685051659307, + "tokens_seen": 2449080320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026230303030303035, + "loss": 2.8296, + "theoretical_loss": 3.3740612736166398, + "tokens_seen": 2449145856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622828282828283, + "loss": 2.8743, + "theoretical_loss": 3.3740540423150343, + "tokens_seen": 2449211392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026226262626262626, + "loss": 2.8082, + "theoretical_loss": 3.3740468112610977, + "tokens_seen": 2449276928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002622424242424242, + "loss": 2.5777, + "theoretical_loss": 3.374039580454816, + "tokens_seen": 2449342464 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026222222222222223, + "loss": 2.6087, + "theoretical_loss": 3.374032349896174, + "tokens_seen": 2449408000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026220202020202024, + "loss": 2.977, + "theoretical_loss": 3.3740251195851556, + "tokens_seen": 2449473536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621818181818182, + "loss": 2.7366, + "theoretical_loss": 3.3740178895217463, + "tokens_seen": 2449539072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026216161616161615, + "loss": 2.9014, + "theoretical_loss": 3.3740106597059314, + "tokens_seen": 2449604608 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1378955, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7929234504699707, + "objective/train/theoretical_loss": 3.3740034301376953, + "objective/train/tokens_used": 808529376, + "theoretical_loss": 3.3740034301376953, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621414141414141, + "loss": 2.6305, + "theoretical_loss": 3.3740034301376953, + "tokens_seen": 2449670144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002621212121212122, + "loss": 2.7822, + "theoretical_loss": 3.3739962008170234, + "tokens_seen": 2449735680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026210101010101013, + "loss": 2.7639, + "theoretical_loss": 3.3739889717439, + "tokens_seen": 2449801216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002620808080808081, + "loss": 2.5319, + "theoretical_loss": 3.37398174291831, + "tokens_seen": 2449866752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026206060606060605, + "loss": 2.644, + "theoretical_loss": 3.3739745143402384, + "tokens_seen": 2449932288 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262040404040404, + "loss": 2.5639, + "theoretical_loss": 3.373967286009671, + "tokens_seen": 2449997824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026202020202020207, + "loss": 2.6818, + "theoretical_loss": 3.3739600579265914, + "tokens_seen": 2450063360 + }, + { + "epoch": 0.48, + "learning_rate": 0.000262, + "loss": 2.5983, + "theoretical_loss": 3.3739528300909853, + "tokens_seen": 2450128896 + }, + { + "epoch": 0.48, + "learning_rate": 0.000261979797979798, + "loss": 2.7577, + "theoretical_loss": 3.3739456025028374, + "tokens_seen": 2450194432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026195959595959594, + "loss": 2.7349, + "theoretical_loss": 3.3739383751621324, + "tokens_seen": 2450259968 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002619393939393939, + "loss": 2.7296, + "theoretical_loss": 3.3739311480688556, + "tokens_seen": 2450325504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026191919191919196, + "loss": 2.3109, + "theoretical_loss": 3.3739239212229912, + "tokens_seen": 2450391040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618989898989899, + "loss": 2.7661, + "theoretical_loss": 3.373916694624525, + "tokens_seen": 2450456576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002618787878787879, + "loss": 2.4551, + "theoretical_loss": 3.3739094682734416, + "tokens_seen": 2450522112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026185858585858583, + "loss": 2.6676, + "theoretical_loss": 3.373902242169726, + "tokens_seen": 2450587648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026183838383838384, + "loss": 2.5146, + "theoretical_loss": 3.3738950163133623, + "tokens_seen": 2450653184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026181818181818185, + "loss": 2.738, + "theoretical_loss": 3.3738877907043365, + "tokens_seen": 2450718720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617979797979798, + "loss": 2.5104, + "theoretical_loss": 3.3738805653426334, + "tokens_seen": 2450784256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026177777777777777, + "loss": 2.5478, + "theoretical_loss": 3.3738733402282373, + "tokens_seen": 2450849792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002617575757575757, + "loss": 2.5984, + "theoretical_loss": 3.3738661153611336, + "tokens_seen": 2450915328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026173737373737374, + "loss": 2.6464, + "theoretical_loss": 3.3738588907413067, + "tokens_seen": 2450980864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026171717171717175, + "loss": 2.6351, + "theoretical_loss": 3.373851666368742, + "tokens_seen": 2451046400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002616969696969697, + "loss": 2.9205, + "theoretical_loss": 3.373844442243425, + "tokens_seen": 2451111936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026167676767676766, + "loss": 2.8467, + "theoretical_loss": 3.373837218365339, + "tokens_seen": 2451177472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026165656565656567, + "loss": 2.7593, + "theoretical_loss": 3.3738299947344705, + "tokens_seen": 2451243008 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1380357, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.30672287940979, + "objective/train/theoretical_loss": 3.3738227713508038, + "objective/train/tokens_used": 810167776, + "theoretical_loss": 3.3738227713508038, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026163636363636363, + "loss": 2.4734, + "theoretical_loss": 3.3738227713508038, + "tokens_seen": 2451308544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026161616161616164, + "loss": 2.5805, + "theoretical_loss": 3.3738155482143233, + "tokens_seen": 2451374080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615959595959596, + "loss": 2.7616, + "theoretical_loss": 3.373808325325015, + "tokens_seen": 2451439616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026157575757575755, + "loss": 2.4545, + "theoretical_loss": 3.3738011026828634, + "tokens_seen": 2451505152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026155555555555556, + "loss": 2.6362, + "theoretical_loss": 3.373793880287853, + "tokens_seen": 2451570688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002615353535353535, + "loss": 2.6704, + "theoretical_loss": 3.373786658139969, + "tokens_seen": 2451636224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026151515151515153, + "loss": 2.8315, + "theoretical_loss": 3.3737794362391966, + "tokens_seen": 2451701760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614949494949495, + "loss": 2.644, + "theoretical_loss": 3.3737722145855207, + "tokens_seen": 2451767296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614747474747475, + "loss": 2.5636, + "theoretical_loss": 3.3737649931789258, + "tokens_seen": 2451832832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026145454545454546, + "loss": 2.8013, + "theoretical_loss": 3.373757772019397, + "tokens_seen": 2451898368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614343434343434, + "loss": 2.788, + "theoretical_loss": 3.37375055110692, + "tokens_seen": 2451963904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002614141414141414, + "loss": 2.6461, + "theoretical_loss": 3.373743330441479, + "tokens_seen": 2452029440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613939393939394, + "loss": 2.6602, + "theoretical_loss": 3.3737361100230587, + "tokens_seen": 2452094976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613737373737374, + "loss": 2.8888, + "theoretical_loss": 3.3737288898516447, + "tokens_seen": 2452160512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026135353535353535, + "loss": 3.0098, + "theoretical_loss": 3.3737216699272214, + "tokens_seen": 2452226048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613333333333333, + "loss": 2.6016, + "theoretical_loss": 3.3737144502497745, + "tokens_seen": 2452291584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002613131313131313, + "loss": 2.8556, + "theoretical_loss": 3.373707230819288, + "tokens_seen": 2452357120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026129292929292933, + "loss": 2.6773, + "theoretical_loss": 3.3737000116357474, + "tokens_seen": 2452422656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612727272727273, + "loss": 2.7507, + "theoretical_loss": 3.3736927926991376, + "tokens_seen": 2452488192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026125252525252524, + "loss": 2.3707, + "theoretical_loss": 3.373685574009444, + "tokens_seen": 2452553728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612323232323232, + "loss": 2.8399, + "theoretical_loss": 3.3736783555666507, + "tokens_seen": 2452619264 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002612121212121212, + "loss": 2.5239, + "theoretical_loss": 3.373671137370743, + "tokens_seen": 2452684800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611919191919192, + "loss": 2.5891, + "theoretical_loss": 3.373663919421706, + "tokens_seen": 2452750336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002611717171717172, + "loss": 2.642, + "theoretical_loss": 3.3736567017195247, + "tokens_seen": 2452815872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026115151515151514, + "loss": 2.6805, + "theoretical_loss": 3.3736494842641838, + "tokens_seen": 2452881408 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1380887, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2253129482269287, + "objective/train/theoretical_loss": 3.3736422670556685, + "objective/train/tokens_used": 811806176, + "theoretical_loss": 3.3736422670556685, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026113131313131315, + "loss": 2.3173, + "theoretical_loss": 3.3736422670556685, + "tokens_seen": 2452946944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026111111111111116, + "loss": 2.4786, + "theoretical_loss": 3.373635050093964, + "tokens_seen": 2453012480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002610909090909091, + "loss": 2.5761, + "theoretical_loss": 3.373627833379054, + "tokens_seen": 2453078016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026107070707070707, + "loss": 2.7929, + "theoretical_loss": 3.373620616910925, + "tokens_seen": 2453143552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026105050505050503, + "loss": 2.5641, + "theoretical_loss": 3.3736134006895613, + "tokens_seen": 2453209088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026103030303030304, + "loss": 2.6181, + "theoretical_loss": 3.373606184714948, + "tokens_seen": 2453274624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026101010101010105, + "loss": 2.7329, + "theoretical_loss": 3.37359896898707, + "tokens_seen": 2453340160 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260989898989899, + "loss": 2.8296, + "theoretical_loss": 3.373591753505912, + "tokens_seen": 2453405696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026096969696969696, + "loss": 2.7251, + "theoretical_loss": 3.3735845382714595, + "tokens_seen": 2453471232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002609494949494949, + "loss": 2.5733, + "theoretical_loss": 3.373577323283697, + "tokens_seen": 2453536768 + }, + { + "epoch": 0.48, + "learning_rate": 0.000260929292929293, + "loss": 2.668, + "theoretical_loss": 3.3735701085426104, + "tokens_seen": 2453602304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026090909090909094, + "loss": 2.5378, + "theoretical_loss": 3.3735628940481837, + "tokens_seen": 2453667840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608888888888889, + "loss": 2.7715, + "theoretical_loss": 3.3735556798004014, + "tokens_seen": 2453733376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026086868686868686, + "loss": 2.6143, + "theoretical_loss": 3.3735484657992503, + "tokens_seen": 2453798912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608484848484848, + "loss": 2.7068, + "theoretical_loss": 3.3735412520447134, + "tokens_seen": 2453864448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002608282828282829, + "loss": 2.7391, + "theoretical_loss": 3.373534038536777, + "tokens_seen": 2453929984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026080808080808084, + "loss": 2.6192, + "theoretical_loss": 3.3735268252754262, + "tokens_seen": 2453995520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002607878787878788, + "loss": 2.7688, + "theoretical_loss": 3.373519612260645, + "tokens_seen": 2454061056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026076767676767675, + "loss": 2.7667, + "theoretical_loss": 3.373512399492419, + "tokens_seen": 2454126592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026074747474747476, + "loss": 2.8114, + "theoretical_loss": 3.373505186970733, + "tokens_seen": 2454192128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026072727272727277, + "loss": 2.7208, + "theoretical_loss": 3.3734979746955718, + "tokens_seen": 2454257664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026070707070707073, + "loss": 2.7376, + "theoretical_loss": 3.373490762666921, + "tokens_seen": 2454323200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606868686868687, + "loss": 2.6795, + "theoretical_loss": 3.373483550884765, + "tokens_seen": 2454388736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026066666666666664, + "loss": 2.3828, + "theoretical_loss": 3.3734763393490894, + "tokens_seen": 2454454272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026064646464646465, + "loss": 2.7064, + "theoretical_loss": 3.3734691280598783, + "tokens_seen": 2454519808 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1382209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.435671806335449, + "objective/train/theoretical_loss": 3.3734619170171176, + "objective/train/tokens_used": 813444576, + "theoretical_loss": 3.3734619170171176, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026062626262626266, + "loss": 2.7652, + "theoretical_loss": 3.3734619170171176, + "tokens_seen": 2454585344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002606060606060606, + "loss": 2.6492, + "theoretical_loss": 3.373454706220792, + "tokens_seen": 2454650880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605858585858586, + "loss": 2.8513, + "theoretical_loss": 3.3734474956708858, + "tokens_seen": 2454716416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026056565656565653, + "loss": 2.5122, + "theoretical_loss": 3.373440285367385, + "tokens_seen": 2454781952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026054545454545455, + "loss": 2.7872, + "theoretical_loss": 3.3734330753102744, + "tokens_seen": 2454847488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026052525252525256, + "loss": 2.6191, + "theoretical_loss": 3.3734258654995384, + "tokens_seen": 2454913024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002605050505050505, + "loss": 2.4805, + "theoretical_loss": 3.3734186559351627, + "tokens_seen": 2454978560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026048484848484847, + "loss": 2.7638, + "theoretical_loss": 3.373411446617132, + "tokens_seen": 2455044096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604646464646465, + "loss": 2.4113, + "theoretical_loss": 3.373404237545431, + "tokens_seen": 2455109632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026044444444444444, + "loss": 2.6485, + "theoretical_loss": 3.3733970287200457, + "tokens_seen": 2455175168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026042424242424245, + "loss": 2.8612, + "theoretical_loss": 3.37338982014096, + "tokens_seen": 2455240704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002604040404040404, + "loss": 2.7088, + "theoretical_loss": 3.3733826118081596, + "tokens_seen": 2455306240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026038383838383836, + "loss": 2.6053, + "theoretical_loss": 3.3733754037216293, + "tokens_seen": 2455371776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603636363636364, + "loss": 2.6841, + "theoretical_loss": 3.373368195881354, + "tokens_seen": 2455437312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026034343434343433, + "loss": 2.6539, + "theoretical_loss": 3.3733609882873186, + "tokens_seen": 2455502848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026032323232323234, + "loss": 2.6503, + "theoretical_loss": 3.3733537809395084, + "tokens_seen": 2455568384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002603030303030303, + "loss": 2.5369, + "theoretical_loss": 3.3733465738379085, + "tokens_seen": 2455633920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602828282828283, + "loss": 2.6317, + "theoretical_loss": 3.3733393669825036, + "tokens_seen": 2455699456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026026262626262627, + "loss": 2.4912, + "theoretical_loss": 3.3733321603732787, + "tokens_seen": 2455764992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602424242424242, + "loss": 2.5574, + "theoretical_loss": 3.3733249540102195, + "tokens_seen": 2455830528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026022222222222223, + "loss": 2.5554, + "theoretical_loss": 3.37331774789331, + "tokens_seen": 2455896064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002602020202020202, + "loss": 2.6735, + "theoretical_loss": 3.373310542022536, + "tokens_seen": 2455961600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002601818181818182, + "loss": 2.7537, + "theoretical_loss": 3.3733033363978824, + "tokens_seen": 2456027136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026016161616161616, + "loss": 2.6941, + "theoretical_loss": 3.373296131019334, + "tokens_seen": 2456092672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0002601414141414141, + "loss": 2.4869, + "theoretical_loss": 3.3732889258868757, + "tokens_seen": 2456158208 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 1382688, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1942532062530518, + "objective/train/theoretical_loss": 3.373281721000493, + "objective/train/tokens_used": 815082976, + "theoretical_loss": 3.373281721000493, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00026012121212121213, + "loss": 2.6719, + "theoretical_loss": 3.373281721000493, + "tokens_seen": 2456223744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026010101010101014, + "loss": 2.6785, + "theoretical_loss": 3.3732745163601705, + "tokens_seen": 2456289280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002600808080808081, + "loss": 2.8604, + "theoretical_loss": 3.373267311965894, + "tokens_seen": 2456354816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026006060606060605, + "loss": 2.6477, + "theoretical_loss": 3.373260107817647, + "tokens_seen": 2456420352 + }, + { + "epoch": 0.49, + "learning_rate": 0.000260040404040404, + "loss": 2.5036, + "theoretical_loss": 3.373252903915416, + "tokens_seen": 2456485888 + }, + { + "epoch": 0.49, + "learning_rate": 0.000260020202020202, + "loss": 2.6253, + "theoretical_loss": 3.3732457002591856, + "tokens_seen": 2456551424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00026000000000000003, + "loss": 2.7159, + "theoretical_loss": 3.3732384968489404, + "tokens_seen": 2456616960 + }, + { + "epoch": 0.49, + "learning_rate": 0.000259979797979798, + "loss": 2.4875, + "theoretical_loss": 3.3732312936846665, + "tokens_seen": 2456682496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025995959595959595, + "loss": 2.7041, + "theoretical_loss": 3.3732240907663473, + "tokens_seen": 2456748032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002599393939393939, + "loss": 2.7388, + "theoretical_loss": 3.3732168880939692, + "tokens_seen": 2456813568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025991919191919197, + "loss": 2.8114, + "theoretical_loss": 3.373209685667517, + "tokens_seen": 2456879104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598989898989899, + "loss": 2.6451, + "theoretical_loss": 3.3732024834869754, + "tokens_seen": 2456944640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598787878787879, + "loss": 2.6221, + "theoretical_loss": 3.37319528155233, + "tokens_seen": 2457010176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025985858585858584, + "loss": 2.7001, + "theoretical_loss": 3.3731880798635645, + "tokens_seen": 2457075712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002598383838383838, + "loss": 2.8865, + "theoretical_loss": 3.3731808784206656, + "tokens_seen": 2457141248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025981818181818186, + "loss": 2.7527, + "theoretical_loss": 3.3731736772236176, + "tokens_seen": 2457206784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597979797979798, + "loss": 2.7051, + "theoretical_loss": 3.3731664762724054, + "tokens_seen": 2457272320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002597777777777778, + "loss": 2.7675, + "theoretical_loss": 3.3731592755670143, + "tokens_seen": 2457337856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025975757575757573, + "loss": 2.6563, + "theoretical_loss": 3.3731520751074293, + "tokens_seen": 2457403392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025973737373737374, + "loss": 2.5794, + "theoretical_loss": 3.3731448748936357, + "tokens_seen": 2457468928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025971717171717175, + "loss": 2.7574, + "theoretical_loss": 3.3731376749256183, + "tokens_seen": 2457534464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596969696969697, + "loss": 2.5546, + "theoretical_loss": 3.3731304752033617, + "tokens_seen": 2457600000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025967676767676767, + "loss": 2.8253, + "theoretical_loss": 3.3731232757268517, + "tokens_seen": 2457665536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002596565656565656, + "loss": 2.6761, + "theoretical_loss": 3.373116076496073, + "tokens_seen": 2457731072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025963636363636363, + "loss": 2.757, + "theoretical_loss": 3.373108877511011, + "tokens_seen": 2457796608 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1383732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6407430171966553, + "objective/train/theoretical_loss": 3.3731016787716506, + "objective/train/tokens_used": 816721376, + "theoretical_loss": 3.3731016787716506, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025961616161616165, + "loss": 2.5556, + "theoretical_loss": 3.3731016787716506, + "tokens_seen": 2457862144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595959595959596, + "loss": 2.7655, + "theoretical_loss": 3.3730944802779765, + "tokens_seen": 2457927680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025957575757575756, + "loss": 2.8096, + "theoretical_loss": 3.373087282029974, + "tokens_seen": 2457993216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025955555555555557, + "loss": 2.4266, + "theoretical_loss": 3.3730800840276283, + "tokens_seen": 2458058752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002595353535353535, + "loss": 2.6438, + "theoretical_loss": 3.3730728862709243, + "tokens_seen": 2458124288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025951515151515154, + "loss": 2.6787, + "theoretical_loss": 3.373065688759847, + "tokens_seen": 2458189824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594949494949495, + "loss": 2.4855, + "theoretical_loss": 3.373058491494382, + "tokens_seen": 2458255360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025947474747474745, + "loss": 2.5282, + "theoretical_loss": 3.3730512944745135, + "tokens_seen": 2458320896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025945454545454546, + "loss": 2.792, + "theoretical_loss": 3.3730440977002276, + "tokens_seen": 2458386432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002594343434343435, + "loss": 2.7043, + "theoretical_loss": 3.3730369011715085, + "tokens_seen": 2458451968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025941414141414143, + "loss": 2.6292, + "theoretical_loss": 3.3730297048883413, + "tokens_seen": 2458517504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593939393939394, + "loss": 2.77, + "theoretical_loss": 3.373022508850712, + "tokens_seen": 2458583040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593737373737374, + "loss": 2.6212, + "theoretical_loss": 3.3730153130586045, + "tokens_seen": 2458648576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025935353535353536, + "loss": 2.8191, + "theoretical_loss": 3.373008117512005, + "tokens_seen": 2458714112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025933333333333337, + "loss": 2.6787, + "theoretical_loss": 3.373000922210897, + "tokens_seen": 2458779648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002593131313131313, + "loss": 2.7548, + "theoretical_loss": 3.3729937271552677, + "tokens_seen": 2458845184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592929292929293, + "loss": 2.5843, + "theoretical_loss": 3.3729865323451005, + "tokens_seen": 2458910720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592727272727273, + "loss": 2.6677, + "theoretical_loss": 3.3729793377803814, + "tokens_seen": 2458976256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025925252525252525, + "loss": 2.7194, + "theoretical_loss": 3.372972143461095, + "tokens_seen": 2459041792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025923232323232326, + "loss": 2.5791, + "theoretical_loss": 3.3729649493872262, + "tokens_seen": 2459107328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002592121212121212, + "loss": 2.4597, + "theoretical_loss": 3.372957755558761, + "tokens_seen": 2459172864 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025919191919191923, + "loss": 2.5472, + "theoretical_loss": 3.3729505619756837, + "tokens_seen": 2459238400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591717171717172, + "loss": 2.8237, + "theoretical_loss": 3.372943368637979, + "tokens_seen": 2459303936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025915151515151514, + "loss": 2.4929, + "theoretical_loss": 3.3729361755456333, + "tokens_seen": 2459369472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025913131313131315, + "loss": 2.7698, + "theoretical_loss": 3.372928982698631, + "tokens_seen": 2459435008 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1384412, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4120562076568604, + "objective/train/theoretical_loss": 3.372921790096957, + "objective/train/tokens_used": 818359776, + "theoretical_loss": 3.372921790096957, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002591111111111111, + "loss": 2.653, + "theoretical_loss": 3.372921790096957, + "tokens_seen": 2459500544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590909090909091, + "loss": 2.7176, + "theoretical_loss": 3.3729145977405963, + "tokens_seen": 2459566080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002590707070707071, + "loss": 2.5882, + "theoretical_loss": 3.372907405629535, + "tokens_seen": 2459631616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025905050505050503, + "loss": 2.6167, + "theoretical_loss": 3.3729002137637565, + "tokens_seen": 2459697152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025903030303030304, + "loss": 2.6304, + "theoretical_loss": 3.3728930221432476, + "tokens_seen": 2459762688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025901010101010106, + "loss": 2.6163, + "theoretical_loss": 3.3728858307679923, + "tokens_seen": 2459828224 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258989898989899, + "loss": 2.6575, + "theoretical_loss": 3.3728786396379764, + "tokens_seen": 2459893760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025896969696969697, + "loss": 2.7139, + "theoretical_loss": 3.3728714487531843, + "tokens_seen": 2459959296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002589494949494949, + "loss": 2.7133, + "theoretical_loss": 3.372864258113602, + "tokens_seen": 2460024832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025892929292929294, + "loss": 2.6539, + "theoretical_loss": 3.3728570677192136, + "tokens_seen": 2460090368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025890909090909095, + "loss": 2.5557, + "theoretical_loss": 3.3728498775700047, + "tokens_seen": 2460155904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588888888888889, + "loss": 2.9434, + "theoretical_loss": 3.3728426876659605, + "tokens_seen": 2460221440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025886868686868686, + "loss": 2.5462, + "theoretical_loss": 3.372835498007066, + "tokens_seen": 2460286976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002588484848484848, + "loss": 2.5719, + "theoretical_loss": 3.3728283085933066, + "tokens_seen": 2460352512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025882828282828283, + "loss": 2.4683, + "theoretical_loss": 3.3728211194246667, + "tokens_seen": 2460418048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025880808080808084, + "loss": 2.7931, + "theoretical_loss": 3.3728139305011324, + "tokens_seen": 2460483584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587878787878788, + "loss": 2.4508, + "theoretical_loss": 3.3728067418226875, + "tokens_seen": 2460549120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025876767676767675, + "loss": 2.6021, + "theoretical_loss": 3.372799553389319, + "tokens_seen": 2460614656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587474747474747, + "loss": 2.6265, + "theoretical_loss": 3.37279236520101, + "tokens_seen": 2460680192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002587272727272728, + "loss": 2.5692, + "theoretical_loss": 3.3727851772577466, + "tokens_seen": 2460745728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025870707070707073, + "loss": 2.6171, + "theoretical_loss": 3.372777989559514, + "tokens_seen": 2460811264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586868686868687, + "loss": 2.911, + "theoretical_loss": 3.372770802106297, + "tokens_seen": 2460876800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025866666666666665, + "loss": 2.6211, + "theoretical_loss": 3.3727636148980813, + "tokens_seen": 2460942336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586464646464646, + "loss": 2.8254, + "theoretical_loss": 3.3727564279348514, + "tokens_seen": 2461007872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025862626262626267, + "loss": 2.462, + "theoretical_loss": 3.3727492412165927, + "tokens_seen": 2461073408 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1385122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.077772855758667, + "objective/train/theoretical_loss": 3.37274205474329, + "objective/train/tokens_used": 819998176, + "theoretical_loss": 3.37274205474329, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002586060606060606, + "loss": 2.7462, + "theoretical_loss": 3.37274205474329, + "tokens_seen": 2461138944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585858585858586, + "loss": 2.6135, + "theoretical_loss": 3.3727348685149288, + "tokens_seen": 2461204480 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025856565656565654, + "loss": 2.5446, + "theoretical_loss": 3.3727276825314942, + "tokens_seen": 2461270016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025854545454545455, + "loss": 2.5701, + "theoretical_loss": 3.3727204967929714, + "tokens_seen": 2461335552 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025852525252525256, + "loss": 2.666, + "theoretical_loss": 3.372713311299345, + "tokens_seen": 2461401088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002585050505050505, + "loss": 2.7233, + "theoretical_loss": 3.3727061260506006, + "tokens_seen": 2461466624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584848484848485, + "loss": 2.7061, + "theoretical_loss": 3.3726989410467234, + "tokens_seen": 2461532160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025846464646464643, + "loss": 2.7656, + "theoretical_loss": 3.372691756287699, + "tokens_seen": 2461597696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025844444444444444, + "loss": 2.5441, + "theoretical_loss": 3.3726845717735108, + "tokens_seen": 2461663232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025842424242424246, + "loss": 2.554, + "theoretical_loss": 3.3726773875041456, + "tokens_seen": 2461728768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002584040404040404, + "loss": 2.9101, + "theoretical_loss": 3.372670203479588, + "tokens_seen": 2461794304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025838383838383837, + "loss": 2.5832, + "theoretical_loss": 3.3726630196998233, + "tokens_seen": 2461859840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583636363636364, + "loss": 2.6129, + "theoretical_loss": 3.372655836164836, + "tokens_seen": 2461925376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025834343434343434, + "loss": 2.5739, + "theoretical_loss": 3.372648652874612, + "tokens_seen": 2461990912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025832323232323235, + "loss": 2.7911, + "theoretical_loss": 3.3726414698291363, + "tokens_seen": 2462056448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002583030303030303, + "loss": 2.5293, + "theoretical_loss": 3.372634287028394, + "tokens_seen": 2462121984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025828282828282826, + "loss": 2.5772, + "theoretical_loss": 3.37262710447237, + "tokens_seen": 2462187520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025826262626262627, + "loss": 2.6701, + "theoretical_loss": 3.3726199221610496, + "tokens_seen": 2462253056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025824242424242423, + "loss": 2.5746, + "theoretical_loss": 3.3726127400944184, + "tokens_seen": 2462318592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025822222222222224, + "loss": 2.4774, + "theoretical_loss": 3.3726055582724603, + "tokens_seen": 2462384128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002582020202020202, + "loss": 2.8126, + "theoretical_loss": 3.372598376695162, + "tokens_seen": 2462449664 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581818181818182, + "loss": 2.8808, + "theoretical_loss": 3.3725911953625074, + "tokens_seen": 2462515200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025816161616161617, + "loss": 2.6186, + "theoretical_loss": 3.3725840142744827, + "tokens_seen": 2462580736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581414141414141, + "loss": 2.5306, + "theoretical_loss": 3.372576833431072, + "tokens_seen": 2462646272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025812121212121213, + "loss": 2.7897, + "theoretical_loss": 3.3725696528322615, + "tokens_seen": 2462711808 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1386383, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.628753185272217, + "objective/train/theoretical_loss": 3.3725624724780356, + "objective/train/tokens_used": 821636576, + "theoretical_loss": 3.3725624724780356, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002581010101010101, + "loss": 2.7718, + "theoretical_loss": 3.3725624724780356, + "tokens_seen": 2462777344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002580808080808081, + "loss": 2.6391, + "theoretical_loss": 3.37255529236838, + "tokens_seen": 2462842880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025806060606060606, + "loss": 2.6747, + "theoretical_loss": 3.3725481125032792, + "tokens_seen": 2462908416 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258040404040404, + "loss": 2.6212, + "theoretical_loss": 3.372540932882719, + "tokens_seen": 2462973952 + }, + { + "epoch": 0.49, + "learning_rate": 0.000258020202020202, + "loss": 2.794, + "theoretical_loss": 3.3725337535066844, + "tokens_seen": 2463039488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025800000000000004, + "loss": 2.681, + "theoretical_loss": 3.37252657437516, + "tokens_seen": 2463105024 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257979797979798, + "loss": 2.5585, + "theoretical_loss": 3.3725193954881316, + "tokens_seen": 2463170560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025795959595959595, + "loss": 2.759, + "theoretical_loss": 3.3725122168455846, + "tokens_seen": 2463236096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579393939393939, + "loss": 2.5883, + "theoretical_loss": 3.3725050384475033, + "tokens_seen": 2463301632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002579191919191919, + "loss": 2.788, + "theoretical_loss": 3.372497860293874, + "tokens_seen": 2463367168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025789898989898993, + "loss": 2.5521, + "theoretical_loss": 3.3724906823846807, + "tokens_seen": 2463432704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578787878787879, + "loss": 2.6324, + "theoretical_loss": 3.3724835047199093, + "tokens_seen": 2463498240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025785858585858584, + "loss": 2.6856, + "theoretical_loss": 3.3724763272995446, + "tokens_seen": 2463563776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002578383838383838, + "loss": 2.6805, + "theoretical_loss": 3.372469150123572, + "tokens_seen": 2463629312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025781818181818187, + "loss": 2.5171, + "theoretical_loss": 3.3724619731919767, + "tokens_seen": 2463694848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577979797979798, + "loss": 2.5444, + "theoretical_loss": 3.372454796504744, + "tokens_seen": 2463760384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002577777777777778, + "loss": 2.5605, + "theoretical_loss": 3.3724476200618585, + "tokens_seen": 2463825920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025775757575757574, + "loss": 2.6016, + "theoretical_loss": 3.372440443863306, + "tokens_seen": 2463891456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025773737373737375, + "loss": 2.4739, + "theoretical_loss": 3.3724332679090714, + "tokens_seen": 2463956992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025771717171717176, + "loss": 2.6702, + "theoretical_loss": 3.37242609219914, + "tokens_seen": 2464022528 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576969696969697, + "loss": 3.0067, + "theoretical_loss": 3.372418916733497, + "tokens_seen": 2464088064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025767676767676767, + "loss": 2.6364, + "theoretical_loss": 3.3724117415121273, + "tokens_seen": 2464153600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025765656565656563, + "loss": 2.6665, + "theoretical_loss": 3.372404566535016, + "tokens_seen": 2464219136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002576363636363637, + "loss": 2.5636, + "theoretical_loss": 3.3723973918021493, + "tokens_seen": 2464284672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025761616161616165, + "loss": 2.6554, + "theoretical_loss": 3.3723902173135114, + "tokens_seen": 2464350208 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1387153, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0869603157043457, + "objective/train/theoretical_loss": 3.3723830430690875, + "objective/train/tokens_used": 823274976, + "theoretical_loss": 3.3723830430690875, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575959595959596, + "loss": 2.8407, + "theoretical_loss": 3.3723830430690875, + "tokens_seen": 2464415744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025757575757575756, + "loss": 2.6364, + "theoretical_loss": 3.3723758690688634, + "tokens_seen": 2464481280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575555555555555, + "loss": 2.792, + "theoretical_loss": 3.372368695312824, + "tokens_seen": 2464546816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002575353535353536, + "loss": 2.6318, + "theoretical_loss": 3.372361521800954, + "tokens_seen": 2464612352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025751515151515154, + "loss": 2.5343, + "theoretical_loss": 3.3723543485332392, + "tokens_seen": 2464677888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574949494949495, + "loss": 2.5983, + "theoretical_loss": 3.372347175509665, + "tokens_seen": 2464743424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025747474747474746, + "loss": 2.9117, + "theoretical_loss": 3.3723400027302164, + "tokens_seen": 2464808960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574545454545454, + "loss": 2.8373, + "theoretical_loss": 3.372332830194878, + "tokens_seen": 2464874496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002574343434343435, + "loss": 2.7307, + "theoretical_loss": 3.3723256579036356, + "tokens_seen": 2464940032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025741414141414144, + "loss": 2.6291, + "theoretical_loss": 3.372318485856474, + "tokens_seen": 2465005568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002573939393939394, + "loss": 2.6291, + "theoretical_loss": 3.3723113140533787, + "tokens_seen": 2465071104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025737373737373735, + "loss": 2.534, + "theoretical_loss": 3.372304142494335, + "tokens_seen": 2465136640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025735353535353536, + "loss": 2.5889, + "theoretical_loss": 3.372296971179328, + "tokens_seen": 2465202176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025733333333333337, + "loss": 2.726, + "theoretical_loss": 3.372289800108343, + "tokens_seen": 2465267712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025731313131313133, + "loss": 2.8959, + "theoretical_loss": 3.372282629281365, + "tokens_seen": 2465333248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572929292929293, + "loss": 2.7902, + "theoretical_loss": 3.372275458698379, + "tokens_seen": 2465398784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025727272727272724, + "loss": 2.7014, + "theoretical_loss": 3.372268288359371, + "tokens_seen": 2465464320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025725252525252525, + "loss": 2.7106, + "theoretical_loss": 3.3722611182643254, + "tokens_seen": 2465529856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025723232323232326, + "loss": 2.6261, + "theoretical_loss": 3.3722539484132277, + "tokens_seen": 2465595392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002572121212121212, + "loss": 2.8195, + "theoretical_loss": 3.3722467788060633, + "tokens_seen": 2465660928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571919191919192, + "loss": 2.5435, + "theoretical_loss": 3.3722396094428175, + "tokens_seen": 2465726464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571717171717172, + "loss": 2.7456, + "theoretical_loss": 3.372232440323475, + "tokens_seen": 2465792000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025715151515151515, + "loss": 2.5673, + "theoretical_loss": 3.372225271448021, + "tokens_seen": 2465857536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025713131313131316, + "loss": 2.5726, + "theoretical_loss": 3.3722181028164417, + "tokens_seen": 2465923072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002571111111111111, + "loss": 2.629, + "theoretical_loss": 3.3722109344287214, + "tokens_seen": 2465988608 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1388132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7322168350219727, + "objective/train/theoretical_loss": 3.3722037662848456, + "objective/train/tokens_used": 824913376, + "theoretical_loss": 3.3722037662848456, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025709090909090907, + "loss": 2.7237, + "theoretical_loss": 3.3722037662848456, + "tokens_seen": 2466054144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002570707070707071, + "loss": 2.756, + "theoretical_loss": 3.3721965983847992, + "tokens_seen": 2466119680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025705050505050504, + "loss": 2.6395, + "theoretical_loss": 3.372189430728568, + "tokens_seen": 2466185216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025703030303030305, + "loss": 2.6712, + "theoretical_loss": 3.3721822633161374, + "tokens_seen": 2466250752 + }, + { + "epoch": 0.49, + "learning_rate": 0.000257010101010101, + "loss": 2.6978, + "theoretical_loss": 3.3721750961474917, + "tokens_seen": 2466316288 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256989898989899, + "loss": 2.5697, + "theoretical_loss": 3.3721679292226163, + "tokens_seen": 2466381824 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256969696969697, + "loss": 2.5505, + "theoretical_loss": 3.3721607625414975, + "tokens_seen": 2466447360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025694949494949493, + "loss": 2.6512, + "theoretical_loss": 3.3721535961041194, + "tokens_seen": 2466512896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025692929292929294, + "loss": 2.6516, + "theoretical_loss": 3.3721464299104675, + "tokens_seen": 2466578432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002569090909090909, + "loss": 2.6914, + "theoretical_loss": 3.3721392639605274, + "tokens_seen": 2466643968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568888888888889, + "loss": 2.5844, + "theoretical_loss": 3.372132098254284, + "tokens_seen": 2466709504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025686868686868687, + "loss": 2.7308, + "theoretical_loss": 3.3721249327917224, + "tokens_seen": 2466775040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002568484848484848, + "loss": 2.8052, + "theoretical_loss": 3.3721177675728287, + "tokens_seen": 2466840576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025682828282828284, + "loss": 2.5765, + "theoretical_loss": 3.372110602597587, + "tokens_seen": 2466906112 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025680808080808085, + "loss": 2.5673, + "theoretical_loss": 3.372103437865983, + "tokens_seen": 2466971648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567878787878788, + "loss": 2.4827, + "theoretical_loss": 3.372096273378002, + "tokens_seen": 2467037184 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025676767676767676, + "loss": 2.677, + "theoretical_loss": 3.3720891091336296, + "tokens_seen": 2467102720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002567474747474747, + "loss": 2.4876, + "theoretical_loss": 3.3720819451328508, + "tokens_seen": 2467168256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025672727272727273, + "loss": 2.7392, + "theoretical_loss": 3.37207478137565, + "tokens_seen": 2467233792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025670707070707074, + "loss": 2.8118, + "theoretical_loss": 3.372067617862014, + "tokens_seen": 2467299328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566868686868687, + "loss": 2.8042, + "theoretical_loss": 3.3720604545919266, + "tokens_seen": 2467364864 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025666666666666665, + "loss": 2.7927, + "theoretical_loss": 3.372053291565374, + "tokens_seen": 2467430400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566464646464646, + "loss": 2.5149, + "theoretical_loss": 3.3720461287823413, + "tokens_seen": 2467495936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002566262626262627, + "loss": 2.5423, + "theoretical_loss": 3.372038966242813, + "tokens_seen": 2467561472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025660606060606063, + "loss": 2.7534, + "theoretical_loss": 3.3720318039467756, + "tokens_seen": 2467627008 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1388592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5209381580352783, + "objective/train/theoretical_loss": 3.372024641894213, + "objective/train/tokens_used": 826551776, + "theoretical_loss": 3.372024641894213, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565858585858586, + "loss": 2.5018, + "theoretical_loss": 3.372024641894213, + "tokens_seen": 2467692544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025656565656565655, + "loss": 2.7264, + "theoretical_loss": 3.3720174800851117, + "tokens_seen": 2467758080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565454545454545, + "loss": 2.7895, + "theoretical_loss": 3.3720103185194565, + "tokens_seen": 2467823616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025652525252525257, + "loss": 2.6436, + "theoretical_loss": 3.3720031571972324, + "tokens_seen": 2467889152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002565050505050505, + "loss": 2.4623, + "theoretical_loss": 3.3719959961184247, + "tokens_seen": 2467954688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564848484848485, + "loss": 2.5081, + "theoretical_loss": 3.371988835283019, + "tokens_seen": 2468020224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025646464646464644, + "loss": 2.6999, + "theoretical_loss": 3.3719816746910003, + "tokens_seen": 2468085760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025644444444444445, + "loss": 2.5563, + "theoretical_loss": 3.371974514342354, + "tokens_seen": 2468151296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025642424242424246, + "loss": 2.826, + "theoretical_loss": 3.371967354237065, + "tokens_seen": 2468216832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002564040404040404, + "loss": 2.9285, + "theoretical_loss": 3.371960194375119, + "tokens_seen": 2468282368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563838383838384, + "loss": 2.6247, + "theoretical_loss": 3.371953034756501, + "tokens_seen": 2468347904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025636363636363633, + "loss": 2.7877, + "theoretical_loss": 3.371945875381197, + "tokens_seen": 2468413440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025634343434343434, + "loss": 2.6358, + "theoretical_loss": 3.371938716249191, + "tokens_seen": 2468478976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025632323232323235, + "loss": 2.4933, + "theoretical_loss": 3.37193155736047, + "tokens_seen": 2468544512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002563030303030303, + "loss": 2.6373, + "theoretical_loss": 3.3719243987150174, + "tokens_seen": 2468610048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025628282828282827, + "loss": 2.7029, + "theoretical_loss": 3.3719172403128193, + "tokens_seen": 2468675584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562626262626263, + "loss": 2.7936, + "theoretical_loss": 3.371910082153861, + "tokens_seen": 2468741120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025624242424242424, + "loss": 2.7653, + "theoretical_loss": 3.3719029242381278, + "tokens_seen": 2468806656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025622222222222225, + "loss": 2.7801, + "theoretical_loss": 3.3718957665656046, + "tokens_seen": 2468872192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002562020202020202, + "loss": 2.8052, + "theoretical_loss": 3.3718886091362776, + "tokens_seen": 2468937728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025618181818181816, + "loss": 2.8247, + "theoretical_loss": 3.3718814519501312, + "tokens_seen": 2469003264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025616161616161617, + "loss": 2.7322, + "theoretical_loss": 3.371874295007151, + "tokens_seen": 2469068800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025614141414141413, + "loss": 2.7459, + "theoretical_loss": 3.371867138307322, + "tokens_seen": 2469134336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025612121212121214, + "loss": 2.6096, + "theoretical_loss": 3.3718599818506303, + "tokens_seen": 2469199872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002561010101010101, + "loss": 2.7195, + "theoretical_loss": 3.3718528256370606, + "tokens_seen": 2469265408 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1390136, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.264204263687134, + "objective/train/theoretical_loss": 3.3718456696665977, + "objective/train/tokens_used": 828190176, + "theoretical_loss": 3.3718456696665977, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002560808080808081, + "loss": 2.5066, + "theoretical_loss": 3.3718456696665977, + "tokens_seen": 2469330944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025606060606060606, + "loss": 2.7784, + "theoretical_loss": 3.371838513939228, + "tokens_seen": 2469396480 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256040404040404, + "loss": 2.6871, + "theoretical_loss": 3.3718313584549358, + "tokens_seen": 2469462016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025602020202020203, + "loss": 2.8632, + "theoretical_loss": 3.371824203213707, + "tokens_seen": 2469527552 + }, + { + "epoch": 0.49, + "learning_rate": 0.000256, + "loss": 2.6165, + "theoretical_loss": 3.3718170482155267, + "tokens_seen": 2469593088 + }, + { + "epoch": 0.49, + "learning_rate": 0.000255979797979798, + "loss": 2.7242, + "theoretical_loss": 3.3718098934603797, + "tokens_seen": 2469658624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025595959595959596, + "loss": 2.6787, + "theoretical_loss": 3.3718027389482526, + "tokens_seen": 2469724160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025593939393939397, + "loss": 2.5911, + "theoretical_loss": 3.371795584679129, + "tokens_seen": 2469789696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002559191919191919, + "loss": 2.4766, + "theoretical_loss": 3.3717884306529955, + "tokens_seen": 2469855232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025589898989898994, + "loss": 2.5565, + "theoretical_loss": 3.371781276869837, + "tokens_seen": 2469920768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558787878787879, + "loss": 2.6828, + "theoretical_loss": 3.371774123329639, + "tokens_seen": 2469986304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025585858585858585, + "loss": 2.3683, + "theoretical_loss": 3.3717669700323865, + "tokens_seen": 2470051840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025583838383838386, + "loss": 2.8365, + "theoretical_loss": 3.3717598169780647, + "tokens_seen": 2470117376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002558181818181818, + "loss": 2.775, + "theoretical_loss": 3.3717526641666593, + "tokens_seen": 2470182912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025579797979797983, + "loss": 2.786, + "theoretical_loss": 3.371745511598155, + "tokens_seen": 2470248448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002557777777777778, + "loss": 2.8145, + "theoretical_loss": 3.3717383592725376, + "tokens_seen": 2470313984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025575757575757574, + "loss": 2.5746, + "theoretical_loss": 3.371731207189793, + "tokens_seen": 2470379520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025573737373737375, + "loss": 2.9685, + "theoretical_loss": 3.371724055349905, + "tokens_seen": 2470445056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025571717171717176, + "loss": 2.8629, + "theoretical_loss": 3.3717169037528603, + "tokens_seen": 2470510592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556969696969697, + "loss": 2.7294, + "theoretical_loss": 3.371709752398643, + "tokens_seen": 2470576128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002556767676767677, + "loss": 2.7665, + "theoretical_loss": 3.3717026012872395, + "tokens_seen": 2470641664 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025565656565656563, + "loss": 2.7294, + "theoretical_loss": 3.3716954504186347, + "tokens_seen": 2470707200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025563636363636365, + "loss": 2.8117, + "theoretical_loss": 3.3716882997928135, + "tokens_seen": 2470772736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025561616161616166, + "loss": 2.722, + "theoretical_loss": 3.371681149409762, + "tokens_seen": 2470838272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002555959595959596, + "loss": 2.6333, + "theoretical_loss": 3.371673999269465, + "tokens_seen": 2470903808 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1390574, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2846310138702393, + "objective/train/theoretical_loss": 3.371666849371908, + "objective/train/tokens_used": 829828576, + "theoretical_loss": 3.371666849371908, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025557575757575757, + "loss": 2.646, + "theoretical_loss": 3.371666849371908, + "tokens_seen": 2470969344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025555555555555553, + "loss": 2.6614, + "theoretical_loss": 3.3716596997170765, + "tokens_seen": 2471034880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025553535353535354, + "loss": 2.6798, + "theoretical_loss": 3.371652550304955, + "tokens_seen": 2471100416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025551515151515155, + "loss": 2.5511, + "theoretical_loss": 3.3716454011355297, + "tokens_seen": 2471165952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554949494949495, + "loss": 2.8376, + "theoretical_loss": 3.3716382522087858, + "tokens_seen": 2471231488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025547474747474746, + "loss": 2.5525, + "theoretical_loss": 3.371631103524708, + "tokens_seen": 2471297024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554545454545454, + "loss": 2.7358, + "theoretical_loss": 3.3716239550832827, + "tokens_seen": 2471362560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002554343434343435, + "loss": 2.7443, + "theoretical_loss": 3.3716168068844943, + "tokens_seen": 2471428096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025541414141414144, + "loss": 2.7173, + "theoretical_loss": 3.3716096589283286, + "tokens_seen": 2471493632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553939393939394, + "loss": 2.6789, + "theoretical_loss": 3.3716025112147707, + "tokens_seen": 2471559168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025537373737373736, + "loss": 2.7259, + "theoretical_loss": 3.371595363743806, + "tokens_seen": 2471624704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553535353535353, + "loss": 2.6454, + "theoretical_loss": 3.37158821651542, + "tokens_seen": 2471690240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002553333333333334, + "loss": 2.8601, + "theoretical_loss": 3.3715810695295976, + "tokens_seen": 2471755776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025531313131313133, + "loss": 2.8095, + "theoretical_loss": 3.3715739227863244, + "tokens_seen": 2471821312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002552929292929293, + "loss": 2.5714, + "theoretical_loss": 3.371566776285586, + "tokens_seen": 2471886848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025527272727272725, + "loss": 2.7646, + "theoretical_loss": 3.3715596300273676, + "tokens_seen": 2471952384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025525252525252526, + "loss": 2.5855, + "theoretical_loss": 3.371552484011654, + "tokens_seen": 2472017920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025523232323232327, + "loss": 2.8766, + "theoretical_loss": 3.3715453382384317, + "tokens_seen": 2472083456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025521212121212123, + "loss": 2.5844, + "theoretical_loss": 3.3715381927076846, + "tokens_seen": 2472148992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551919191919192, + "loss": 2.9305, + "theoretical_loss": 3.371531047419399, + "tokens_seen": 2472214528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025517171717171714, + "loss": 2.6522, + "theoretical_loss": 3.3715239023735606, + "tokens_seen": 2472280064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025515151515151515, + "loss": 2.9152, + "theoretical_loss": 3.3715167575701535, + "tokens_seen": 2472345600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00025513131313131316, + "loss": 2.6464, + "theoretical_loss": 3.371509613009164, + "tokens_seen": 2472411136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002551111111111111, + "loss": 2.8255, + "theoretical_loss": 3.371502468690577, + "tokens_seen": 2472476672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002550909090909091, + "loss": 2.5487, + "theoretical_loss": 3.371495324614378, + "tokens_seen": 2472542208 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 1391138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.112156867980957, + "objective/train/theoretical_loss": 3.3714881807805526, + "objective/train/tokens_used": 831466976, + "theoretical_loss": 3.3714881807805526, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0002550707070707071, + "loss": 2.6194, + "theoretical_loss": 3.3714881807805526, + "tokens_seen": 2472607744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025505050505050504, + "loss": 2.6535, + "theoretical_loss": 3.3714810371890858, + "tokens_seen": 2472673280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025503030303030306, + "loss": 2.5126, + "theoretical_loss": 3.371473893839963, + "tokens_seen": 2472738816 + }, + { + "epoch": 0.5, + "learning_rate": 0.000255010101010101, + "loss": 2.7421, + "theoretical_loss": 3.3714667507331697, + "tokens_seen": 2472804352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025498989898989897, + "loss": 2.6705, + "theoretical_loss": 3.371459607868691, + "tokens_seen": 2472869888 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254969696969697, + "loss": 2.876, + "theoretical_loss": 3.3714524652465125, + "tokens_seen": 2472935424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025494949494949494, + "loss": 2.5172, + "theoretical_loss": 3.3714453228666197, + "tokens_seen": 2473000960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025492929292929295, + "loss": 2.5477, + "theoretical_loss": 3.3714381807289975, + "tokens_seen": 2473066496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002549090909090909, + "loss": 2.7505, + "theoretical_loss": 3.3714310388336317, + "tokens_seen": 2473132032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548888888888889, + "loss": 2.8172, + "theoretical_loss": 3.3714238971805077, + "tokens_seen": 2473197568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548686868686869, + "loss": 2.8204, + "theoretical_loss": 3.3714167557696104, + "tokens_seen": 2473263104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025484848484848483, + "loss": 2.6947, + "theoretical_loss": 3.3714096146009256, + "tokens_seen": 2473328640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025482828282828284, + "loss": 2.7944, + "theoretical_loss": 3.371402473674438, + "tokens_seen": 2473394176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002548080808080808, + "loss": 2.8059, + "theoretical_loss": 3.3713953329901343, + "tokens_seen": 2473459712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547878787878788, + "loss": 2.3957, + "theoretical_loss": 3.3713881925479985, + "tokens_seen": 2473525248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025476767676767677, + "loss": 3.0489, + "theoretical_loss": 3.3713810523480165, + "tokens_seen": 2473590784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002547474747474747, + "loss": 2.6489, + "theoretical_loss": 3.371373912390174, + "tokens_seen": 2473656320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025472727272727273, + "loss": 2.9519, + "theoretical_loss": 3.3713667726744556, + "tokens_seen": 2473721856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025470707070707075, + "loss": 2.5138, + "theoretical_loss": 3.3713596332008473, + "tokens_seen": 2473787392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546868686868687, + "loss": 2.6391, + "theoretical_loss": 3.3713524939693347, + "tokens_seen": 2473852928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025466666666666666, + "loss": 2.445, + "theoretical_loss": 3.371345354979902, + "tokens_seen": 2473918464 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546464646464646, + "loss": 2.6741, + "theoretical_loss": 3.371338216232536, + "tokens_seen": 2473984000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002546262626262626, + "loss": 2.6768, + "theoretical_loss": 3.371331077727221, + "tokens_seen": 2474049536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025460606060606064, + "loss": 2.7498, + "theoretical_loss": 3.371323939463943, + "tokens_seen": 2474115072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545858585858586, + "loss": 2.5208, + "theoretical_loss": 3.371316801442687, + "tokens_seen": 2474180608 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1392460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.945885419845581, + "objective/train/theoretical_loss": 3.371309663663439, + "objective/train/tokens_used": 833105376, + "theoretical_loss": 3.371309663663439, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025456565656565655, + "loss": 2.76, + "theoretical_loss": 3.371309663663439, + "tokens_seen": 2474246144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545454545454545, + "loss": 2.8986, + "theoretical_loss": 3.371302526126184, + "tokens_seen": 2474311680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002545252525252526, + "loss": 2.6896, + "theoretical_loss": 3.371295388830907, + "tokens_seen": 2474377216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025450505050505053, + "loss": 2.8037, + "theoretical_loss": 3.3712882517775937, + "tokens_seen": 2474442752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544848484848485, + "loss": 2.6207, + "theoretical_loss": 3.3712811149662296, + "tokens_seen": 2474508288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025446464646464644, + "loss": 2.5777, + "theoretical_loss": 3.3712739783968004, + "tokens_seen": 2474573824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544444444444444, + "loss": 2.6309, + "theoretical_loss": 3.371266842069291, + "tokens_seen": 2474639360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025442424242424247, + "loss": 2.9023, + "theoretical_loss": 3.3712597059836864, + "tokens_seen": 2474704896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002544040404040404, + "loss": 2.6027, + "theoretical_loss": 3.371252570139973, + "tokens_seen": 2474770432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543838383838384, + "loss": 2.5107, + "theoretical_loss": 3.3712454345381353, + "tokens_seen": 2474835968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025436363636363634, + "loss": 2.6429, + "theoretical_loss": 3.3712382991781595, + "tokens_seen": 2474901504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025434343434343435, + "loss": 2.7808, + "theoretical_loss": 3.3712311640600303, + "tokens_seen": 2474967040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025432323232323236, + "loss": 2.765, + "theoretical_loss": 3.3712240291837334, + "tokens_seen": 2475032576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002543030303030303, + "loss": 2.7259, + "theoretical_loss": 3.3712168945492547, + "tokens_seen": 2475098112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025428282828282827, + "loss": 2.8596, + "theoretical_loss": 3.3712097601565785, + "tokens_seen": 2475163648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025426262626262623, + "loss": 2.7, + "theoretical_loss": 3.371202626005691, + "tokens_seen": 2475229184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542424242424243, + "loss": 2.7804, + "theoretical_loss": 3.3711954920965774, + "tokens_seen": 2475294720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025422222222222225, + "loss": 2.8024, + "theoretical_loss": 3.3711883584292233, + "tokens_seen": 2475360256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002542020202020202, + "loss": 2.5451, + "theoretical_loss": 3.3711812250036135, + "tokens_seen": 2475425792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025418181818181817, + "loss": 2.7211, + "theoretical_loss": 3.371174091819734, + "tokens_seen": 2475491328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541616161616161, + "loss": 2.7558, + "theoretical_loss": 3.37116695887757, + "tokens_seen": 2475556864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541414141414142, + "loss": 2.5621, + "theoretical_loss": 3.371159826177107, + "tokens_seen": 2475622400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025412121212121214, + "loss": 2.6117, + "theoretical_loss": 3.3711526937183303, + "tokens_seen": 2475687936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002541010101010101, + "loss": 2.7166, + "theoretical_loss": 3.371145561501226, + "tokens_seen": 2475753472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025408080808080806, + "loss": 2.5153, + "theoretical_loss": 3.371138429525778, + "tokens_seen": 2475819008 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1393060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.525599241256714, + "objective/train/theoretical_loss": 3.371131297791973, + "objective/train/tokens_used": 834743776, + "theoretical_loss": 3.371131297791973, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025406060606060607, + "loss": 2.6186, + "theoretical_loss": 3.371131297791973, + "tokens_seen": 2475884544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002540404040404041, + "loss": 2.6518, + "theoretical_loss": 3.3711241662997957, + "tokens_seen": 2475950080 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025402020202020204, + "loss": 2.6874, + "theoretical_loss": 3.371117035049232, + "tokens_seen": 2476015616 + }, + { + "epoch": 0.5, + "learning_rate": 0.000254, + "loss": 2.6933, + "theoretical_loss": 3.3711099040402672, + "tokens_seen": 2476081152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025397979797979795, + "loss": 2.7264, + "theoretical_loss": 3.3711027732728867, + "tokens_seen": 2476146688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025395959595959596, + "loss": 2.5902, + "theoretical_loss": 3.3710956427470755, + "tokens_seen": 2476212224 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253939393939394, + "loss": 2.8174, + "theoretical_loss": 3.37108851246282, + "tokens_seen": 2476277760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025391919191919193, + "loss": 2.8551, + "theoretical_loss": 3.3710813824201047, + "tokens_seen": 2476343296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538989898989899, + "loss": 2.9332, + "theoretical_loss": 3.3710742526189152, + "tokens_seen": 2476408832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538787878787879, + "loss": 2.7178, + "theoretical_loss": 3.3710671230592375, + "tokens_seen": 2476474368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025385858585858585, + "loss": 2.4841, + "theoretical_loss": 3.3710599937410564, + "tokens_seen": 2476539904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025383838383838387, + "loss": 2.7899, + "theoretical_loss": 3.3710528646643576, + "tokens_seen": 2476605440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002538181818181818, + "loss": 2.6747, + "theoretical_loss": 3.3710457358291266, + "tokens_seen": 2476670976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537979797979798, + "loss": 2.7941, + "theoretical_loss": 3.3710386072353486, + "tokens_seen": 2476736512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537777777777778, + "loss": 2.6417, + "theoretical_loss": 3.371031478883009, + "tokens_seen": 2476802048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025375757575757575, + "loss": 2.8256, + "theoretical_loss": 3.371024350772094, + "tokens_seen": 2476867584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025373737373737376, + "loss": 2.5394, + "theoretical_loss": 3.3710172229025877, + "tokens_seen": 2476933120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002537171717171717, + "loss": 2.861, + "theoretical_loss": 3.3710100952744764, + "tokens_seen": 2476998656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536969696969697, + "loss": 2.3896, + "theoretical_loss": 3.371002967887746, + "tokens_seen": 2477064192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536767676767677, + "loss": 2.5779, + "theoretical_loss": 3.3709958407423803, + "tokens_seen": 2477129728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025365656565656564, + "loss": 2.5206, + "theoretical_loss": 3.3709887138383663, + "tokens_seen": 2477195264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025363636363636365, + "loss": 2.5769, + "theoretical_loss": 3.370981587175689, + "tokens_seen": 2477260800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002536161616161616, + "loss": 2.5455, + "theoretical_loss": 3.3709744607543337, + "tokens_seen": 2477326336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535959595959596, + "loss": 2.7518, + "theoretical_loss": 3.370967334574286, + "tokens_seen": 2477391872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002535757575757576, + "loss": 2.8394, + "theoretical_loss": 3.3709602086355313, + "tokens_seen": 2477457408 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1394031, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.960780143737793, + "objective/train/theoretical_loss": 3.3709530829380543, + "objective/train/tokens_used": 836382176, + "theoretical_loss": 3.3709530829380543, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025355555555555553, + "loss": 2.6722, + "theoretical_loss": 3.3709530829380543, + "tokens_seen": 2477522944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025353535353535354, + "loss": 2.8136, + "theoretical_loss": 3.370945957481842, + "tokens_seen": 2477588480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025351515151515155, + "loss": 2.6897, + "theoretical_loss": 3.3709388322668783, + "tokens_seen": 2477654016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534949494949495, + "loss": 2.7637, + "theoretical_loss": 3.37093170729315, + "tokens_seen": 2477719552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025347474747474747, + "loss": 2.6985, + "theoretical_loss": 3.3709245825606415, + "tokens_seen": 2477785088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002534545454545454, + "loss": 2.6282, + "theoretical_loss": 3.370917458069339, + "tokens_seen": 2477850624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025343434343434344, + "loss": 2.6089, + "theoretical_loss": 3.370910333819227, + "tokens_seen": 2477916160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025341414141414145, + "loss": 2.8118, + "theoretical_loss": 3.370903209810292, + "tokens_seen": 2477981696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533939393939394, + "loss": 2.6033, + "theoretical_loss": 3.370896086042519, + "tokens_seen": 2478047232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025337373737373736, + "loss": 2.464, + "theoretical_loss": 3.3708889625158935, + "tokens_seen": 2478112768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533535353535353, + "loss": 2.6626, + "theoretical_loss": 3.3708818392304005, + "tokens_seen": 2478178304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002533333333333334, + "loss": 2.7205, + "theoretical_loss": 3.3708747161860266, + "tokens_seen": 2478243840 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025331313131313134, + "loss": 2.6801, + "theoretical_loss": 3.370867593382756, + "tokens_seen": 2478309376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532929292929293, + "loss": 2.6577, + "theoretical_loss": 3.370860470820575, + "tokens_seen": 2478374912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025327272727272725, + "loss": 2.596, + "theoretical_loss": 3.3708533484994687, + "tokens_seen": 2478440448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532525252525252, + "loss": 2.761, + "theoretical_loss": 3.3708462264194226, + "tokens_seen": 2478505984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002532323232323233, + "loss": 2.7913, + "theoretical_loss": 3.3708391045804222, + "tokens_seen": 2478571520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025321212121212123, + "loss": 2.6767, + "theoretical_loss": 3.370831982982453, + "tokens_seen": 2478637056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531919191919192, + "loss": 2.7933, + "theoretical_loss": 3.3708248616255005, + "tokens_seen": 2478702592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025317171717171715, + "loss": 2.7327, + "theoretical_loss": 3.3708177405095503, + "tokens_seen": 2478768128 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025315151515151516, + "loss": 2.7457, + "theoretical_loss": 3.3708106196345877, + "tokens_seen": 2478833664 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025313131313131317, + "loss": 2.6712, + "theoretical_loss": 3.370803499000598, + "tokens_seen": 2478899200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002531111111111111, + "loss": 2.5842, + "theoretical_loss": 3.370796378607567, + "tokens_seen": 2478964736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002530909090909091, + "loss": 2.6218, + "theoretical_loss": 3.37078925845548, + "tokens_seen": 2479030272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025307070707070704, + "loss": 2.8101, + "theoretical_loss": 3.3707821385443224, + "tokens_seen": 2479095808 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1394307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1336605548858643, + "objective/train/theoretical_loss": 3.3707750188740797, + "objective/train/tokens_used": 838020576, + "theoretical_loss": 3.3707750188740797, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025305050505050505, + "loss": 2.9381, + "theoretical_loss": 3.3707750188740797, + "tokens_seen": 2479161344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025303030303030306, + "loss": 2.7223, + "theoretical_loss": 3.3707678994447376, + "tokens_seen": 2479226880 + }, + { + "epoch": 0.5, + "learning_rate": 0.000253010101010101, + "loss": 2.7772, + "theoretical_loss": 3.3707607802562816, + "tokens_seen": 2479292416 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252989898989899, + "loss": 2.4381, + "theoretical_loss": 3.370753661308697, + "tokens_seen": 2479357952 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252969696969697, + "loss": 2.5598, + "theoretical_loss": 3.370746542601969, + "tokens_seen": 2479423488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025294949494949494, + "loss": 2.4734, + "theoretical_loss": 3.3707394241360835, + "tokens_seen": 2479489024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025292929292929295, + "loss": 2.8477, + "theoretical_loss": 3.370732305911026, + "tokens_seen": 2479554560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002529090909090909, + "loss": 2.8141, + "theoretical_loss": 3.370725187926782, + "tokens_seen": 2479620096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025288888888888887, + "loss": 2.6405, + "theoretical_loss": 3.3707180701833366, + "tokens_seen": 2479685632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528686868686869, + "loss": 2.64, + "theoretical_loss": 3.3707109526806756, + "tokens_seen": 2479751168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025284848484848484, + "loss": 2.805, + "theoretical_loss": 3.3707038354187846, + "tokens_seen": 2479816704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025282828282828285, + "loss": 2.8968, + "theoretical_loss": 3.3706967183976486, + "tokens_seen": 2479882240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002528080808080808, + "loss": 2.8122, + "theoretical_loss": 3.3706896016172543, + "tokens_seen": 2479947776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527878787878788, + "loss": 2.5673, + "theoretical_loss": 3.370682485077585, + "tokens_seen": 2480013312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025276767676767677, + "loss": 2.6611, + "theoretical_loss": 3.3706753687786284, + "tokens_seen": 2480078848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025274747474747473, + "loss": 2.7215, + "theoretical_loss": 3.3706682527203693, + "tokens_seen": 2480144384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025272727272727274, + "loss": 2.7938, + "theoretical_loss": 3.3706611369027923, + "tokens_seen": 2480209920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002527070707070707, + "loss": 2.6558, + "theoretical_loss": 3.370654021325884, + "tokens_seen": 2480275456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526868686868687, + "loss": 2.6432, + "theoretical_loss": 3.37064690598963, + "tokens_seen": 2480340992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025266666666666666, + "loss": 2.8375, + "theoretical_loss": 3.3706397908940144, + "tokens_seen": 2480406528 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002526464646464646, + "loss": 2.8243, + "theoretical_loss": 3.3706326760390244, + "tokens_seen": 2480472064 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025262626262626263, + "loss": 2.8715, + "theoretical_loss": 3.370625561424644, + "tokens_seen": 2480537600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025260606060606064, + "loss": 2.5464, + "theoretical_loss": 3.37061844705086, + "tokens_seen": 2480603136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525858585858586, + "loss": 2.6587, + "theoretical_loss": 3.3706113329176572, + "tokens_seen": 2480668672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025256565656565656, + "loss": 2.7969, + "theoretical_loss": 3.370604219025021, + "tokens_seen": 2480734208 + }, + { + "debugging/Self-BLEU-5": 0.29522518101396983, + "debugging/distinct-1-grams": 0.7404446023849008, + "debugging/distinct-2-grams": 0.9621106381669762, + "debugging/entropy-1-grams": 4.826739643784389, + "debugging/entropy-2-grams": 5.449830011825935, + "debugging/length": 541.0, + "debugging/num_segments": 3, + "epoch": 0.5, + "objective/train/docs_used": 1395647, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.489612102508545, + "objective/train/theoretical_loss": 3.3705971053729376, + "objective/train/tokens_used": 839658976, + "theoretical_loss": 3.3705971053729376, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025254545454545457, + "loss": 2.8663, + "theoretical_loss": 3.3705971053729376, + "tokens_seen": 2480799744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002525252525252525, + "loss": 2.7436, + "theoretical_loss": 3.3705899919613915, + "tokens_seen": 2480865280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025250505050505054, + "loss": 2.4628, + "theoretical_loss": 3.370582878790369, + "tokens_seen": 2480930816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002524848484848485, + "loss": 2.6383, + "theoretical_loss": 3.370575765859856, + "tokens_seen": 2480996352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025246464646464645, + "loss": 2.6462, + "theoretical_loss": 3.370568653169837, + "tokens_seen": 2481061888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025244444444444446, + "loss": 2.7254, + "theoretical_loss": 3.3705615407202973, + "tokens_seen": 2481127424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025242424242424247, + "loss": 2.7625, + "theoretical_loss": 3.370554428511224, + "tokens_seen": 2481192960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025240404040404043, + "loss": 2.7153, + "theoretical_loss": 3.370547316542601, + "tokens_seen": 2481258496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523838383838384, + "loss": 2.7739, + "theoretical_loss": 3.3705402048144153, + "tokens_seen": 2481324032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025236363636363634, + "loss": 2.6994, + "theoretical_loss": 3.370533093326651, + "tokens_seen": 2481389568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025234343434343435, + "loss": 2.5332, + "theoretical_loss": 3.370525982079294, + "tokens_seen": 2481455104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025232323232323236, + "loss": 2.6188, + "theoretical_loss": 3.3705188710723304, + "tokens_seen": 2481520640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002523030303030303, + "loss": 2.6388, + "theoretical_loss": 3.3705117603057455, + "tokens_seen": 2481586176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522828282828283, + "loss": 2.8156, + "theoretical_loss": 3.370504649779525, + "tokens_seen": 2481651712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025226262626262624, + "loss": 2.9061, + "theoretical_loss": 3.3704975394936536, + "tokens_seen": 2481717248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025224242424242425, + "loss": 2.5036, + "theoretical_loss": 3.3704904294481173, + "tokens_seen": 2481782784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025222222222222226, + "loss": 2.6198, + "theoretical_loss": 3.3704833196429016, + "tokens_seen": 2481848320 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002522020202020202, + "loss": 2.5787, + "theoretical_loss": 3.370476210077993, + "tokens_seen": 2481913856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025218181818181817, + "loss": 2.7652, + "theoretical_loss": 3.3704691007533754, + "tokens_seen": 2481979392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025216161616161613, + "loss": 2.6446, + "theoretical_loss": 3.370461991669035, + "tokens_seen": 2482044928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521414141414142, + "loss": 2.69, + "theoretical_loss": 3.370454882824958, + "tokens_seen": 2482110464 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025212121212121215, + "loss": 2.5785, + "theoretical_loss": 3.370447774221129, + "tokens_seen": 2482176000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002521010101010101, + "loss": 2.5663, + "theoretical_loss": 3.370440665857534, + "tokens_seen": 2482241536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025208080808080806, + "loss": 2.6334, + "theoretical_loss": 3.3704335577341586, + "tokens_seen": 2482307072 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252060606060606, + "loss": 2.5708, + "theoretical_loss": 3.3704264498509877, + "tokens_seen": 2482372608 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1396325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8709092140197754, + "objective/train/theoretical_loss": 3.3704193422080078, + "objective/train/tokens_used": 841297376, + "theoretical_loss": 3.3704193422080078, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002520404040404041, + "loss": 2.6254, + "theoretical_loss": 3.3704193422080078, + "tokens_seen": 2482438144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025202020202020204, + "loss": 2.3845, + "theoretical_loss": 3.3704122348052037, + "tokens_seen": 2482503680 + }, + { + "epoch": 0.5, + "learning_rate": 0.000252, + "loss": 2.7855, + "theoretical_loss": 3.3704051276425613, + "tokens_seen": 2482569216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025197979797979796, + "loss": 2.6292, + "theoretical_loss": 3.3703980207200663, + "tokens_seen": 2482634752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025195959595959597, + "loss": 2.5485, + "theoretical_loss": 3.3703909140377037, + "tokens_seen": 2482700288 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251939393939394, + "loss": 2.5528, + "theoretical_loss": 3.3703838075954593, + "tokens_seen": 2482765824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025191919191919194, + "loss": 2.679, + "theoretical_loss": 3.370376701393319, + "tokens_seen": 2482831360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002518989898989899, + "loss": 2.7721, + "theoretical_loss": 3.3703695954312676, + "tokens_seen": 2482896896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025187878787878785, + "loss": 2.6115, + "theoretical_loss": 3.3703624897092914, + "tokens_seen": 2482962432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025185858585858586, + "loss": 2.6405, + "theoretical_loss": 3.3703553842273757, + "tokens_seen": 2483027968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025183838383838387, + "loss": 2.8127, + "theoretical_loss": 3.370348278985506, + "tokens_seen": 2483093504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025181818181818183, + "loss": 2.6294, + "theoretical_loss": 3.3703411739836677, + "tokens_seen": 2483159040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517979797979798, + "loss": 2.6323, + "theoretical_loss": 3.3703340692218466, + "tokens_seen": 2483224576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517777777777778, + "loss": 2.9136, + "theoretical_loss": 3.3703269647000282, + "tokens_seen": 2483290112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025175757575757575, + "loss": 2.5351, + "theoretical_loss": 3.3703198604181983, + "tokens_seen": 2483355648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025173737373737376, + "loss": 2.6209, + "theoretical_loss": 3.3703127563763418, + "tokens_seen": 2483421184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002517171717171717, + "loss": 2.7328, + "theoretical_loss": 3.370305652574445, + "tokens_seen": 2483486720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516969696969697, + "loss": 2.7113, + "theoretical_loss": 3.370298549012493, + "tokens_seen": 2483552256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516767676767677, + "loss": 2.8797, + "theoretical_loss": 3.370291445690471, + "tokens_seen": 2483617792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025165656565656565, + "loss": 2.5625, + "theoretical_loss": 3.370284342608366, + "tokens_seen": 2483683328 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025163636363636366, + "loss": 2.6306, + "theoretical_loss": 3.370277239766162, + "tokens_seen": 2483748864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002516161616161616, + "loss": 2.6375, + "theoretical_loss": 3.3702701371638453, + "tokens_seen": 2483814400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515959595959596, + "loss": 2.552, + "theoretical_loss": 3.3702630348014013, + "tokens_seen": 2483879936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515757575757576, + "loss": 2.6123, + "theoretical_loss": 3.3702559326788157, + "tokens_seen": 2483945472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025155555555555554, + "loss": 2.4383, + "theoretical_loss": 3.370248830796074, + "tokens_seen": 2484011008 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1397604, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.552063226699829, + "objective/train/theoretical_loss": 3.3702417291531614, + "objective/train/tokens_used": 842935776, + "theoretical_loss": 3.3702417291531614, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025153535353535355, + "loss": 2.8212, + "theoretical_loss": 3.3702417291531614, + "tokens_seen": 2484076544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002515151515151515, + "loss": 2.8348, + "theoretical_loss": 3.3702346277500643, + "tokens_seen": 2484142080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514949494949495, + "loss": 2.8051, + "theoretical_loss": 3.370227526586768, + "tokens_seen": 2484207616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514747474747475, + "loss": 2.5824, + "theoretical_loss": 3.3702204256632573, + "tokens_seen": 2484273152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002514747474747475, + "loss": 2.6755, + "theoretical_loss": 3.3702133249795185, + "tokens_seen": 2484338688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025145454545454543, + "loss": 2.7035, + "theoretical_loss": 3.3702062245355373, + "tokens_seen": 2484404224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025143434343434344, + "loss": 2.6021, + "theoretical_loss": 3.370199124331299, + "tokens_seen": 2484469760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025141414141414145, + "loss": 2.7577, + "theoretical_loss": 3.3701920243667893, + "tokens_seen": 2484535296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513939393939394, + "loss": 2.5391, + "theoretical_loss": 3.3701849246419937, + "tokens_seen": 2484600832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025137373737373737, + "loss": 2.4656, + "theoretical_loss": 3.3701778251568975, + "tokens_seen": 2484666368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002513535353535353, + "loss": 2.4069, + "theoretical_loss": 3.370170725911487, + "tokens_seen": 2484731904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025133333333333333, + "loss": 2.638, + "theoretical_loss": 3.3701636269057467, + "tokens_seen": 2484797440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025131313131313135, + "loss": 2.7451, + "theoretical_loss": 3.3701565281396633, + "tokens_seen": 2484862976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512929292929293, + "loss": 2.7216, + "theoretical_loss": 3.370149429613222, + "tokens_seen": 2484928512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025127272727272726, + "loss": 2.7645, + "theoretical_loss": 3.370142331326408, + "tokens_seen": 2484994048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512525252525252, + "loss": 2.9805, + "theoretical_loss": 3.3701352332792074, + "tokens_seen": 2485059584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002512323232323233, + "loss": 2.6351, + "theoretical_loss": 3.3701281354716057, + "tokens_seen": 2485125120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025121212121212124, + "loss": 3.0558, + "theoretical_loss": 3.370121037903588, + "tokens_seen": 2485190656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511919191919192, + "loss": 2.9248, + "theoretical_loss": 3.370113940575141, + "tokens_seen": 2485256192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025117171717171715, + "loss": 2.6822, + "theoretical_loss": 3.3701068434862487, + "tokens_seen": 2485321728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511515151515151, + "loss": 2.7058, + "theoretical_loss": 3.3700997466368983, + "tokens_seen": 2485387264 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002511313131313132, + "loss": 2.6654, + "theoretical_loss": 3.3700926500270745, + "tokens_seen": 2485452800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025111111111111113, + "loss": 2.7634, + "theoretical_loss": 3.3700855536567627, + "tokens_seen": 2485518336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002510909090909091, + "loss": 2.6132, + "theoretical_loss": 3.3700784575259495, + "tokens_seen": 2485583872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025107070707070704, + "loss": 2.6706, + "theoretical_loss": 3.3700713616346194, + "tokens_seen": 2485649408 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1398115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8068766593933105, + "objective/train/theoretical_loss": 3.3700642659827587, + "objective/train/tokens_used": 844574176, + "theoretical_loss": 3.3700642659827587, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025105050505050506, + "loss": 2.8285, + "theoretical_loss": 3.3700642659827587, + "tokens_seen": 2485714944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025103030303030307, + "loss": 2.8112, + "theoretical_loss": 3.370057170570353, + "tokens_seen": 2485780480 + }, + { + "epoch": 0.5, + "learning_rate": 0.000251010101010101, + "loss": 2.6916, + "theoretical_loss": 3.3700500753973874, + "tokens_seen": 2485846016 + }, + { + "epoch": 0.5, + "learning_rate": 0.000250989898989899, + "loss": 2.7881, + "theoretical_loss": 3.370042980463848, + "tokens_seen": 2485911552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025096969696969694, + "loss": 2.5565, + "theoretical_loss": 3.3700358857697204, + "tokens_seen": 2485977088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025094949494949495, + "loss": 2.6168, + "theoretical_loss": 3.3700287913149896, + "tokens_seen": 2486042624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025092929292929296, + "loss": 2.5873, + "theoretical_loss": 3.3700216970996415, + "tokens_seen": 2486108160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002509090909090909, + "loss": 2.6569, + "theoretical_loss": 3.3700146031236624, + "tokens_seen": 2486173696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508888888888889, + "loss": 2.7418, + "theoretical_loss": 3.370007509387037, + "tokens_seen": 2486239232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025086868686868683, + "loss": 2.6081, + "theoretical_loss": 3.3700004158897516, + "tokens_seen": 2486304768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508484848484849, + "loss": 2.601, + "theoretical_loss": 3.3699933226317915, + "tokens_seen": 2486370304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025082828282828285, + "loss": 2.9466, + "theoretical_loss": 3.3699862296131418, + "tokens_seen": 2486435840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002508080808080808, + "loss": 2.7199, + "theoretical_loss": 3.3699791368337895, + "tokens_seen": 2486501376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025078787878787877, + "loss": 2.4924, + "theoretical_loss": 3.3699720442937187, + "tokens_seen": 2486566912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507676767676768, + "loss": 2.5915, + "theoretical_loss": 3.369964951992916, + "tokens_seen": 2486632448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507474747474748, + "loss": 2.7782, + "theoretical_loss": 3.3699578599313664, + "tokens_seen": 2486697984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025072727272727275, + "loss": 2.7935, + "theoretical_loss": 3.3699507681090557, + "tokens_seen": 2486763520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002507070707070707, + "loss": 2.8086, + "theoretical_loss": 3.36994367652597, + "tokens_seen": 2486829056 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025068686868686866, + "loss": 2.7996, + "theoretical_loss": 3.369936585182095, + "tokens_seen": 2486894592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025066666666666667, + "loss": 2.5882, + "theoretical_loss": 3.3699294940774154, + "tokens_seen": 2486960128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506464646464647, + "loss": 2.7029, + "theoretical_loss": 3.369922403211917, + "tokens_seen": 2487025664 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025062626262626264, + "loss": 2.7572, + "theoretical_loss": 3.369915312585586, + "tokens_seen": 2487091200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002506060606060606, + "loss": 2.8164, + "theoretical_loss": 3.3699082221984082, + "tokens_seen": 2487156736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505858585858586, + "loss": 2.5273, + "theoretical_loss": 3.3699011320503685, + "tokens_seen": 2487222272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025056565656565656, + "loss": 2.8659, + "theoretical_loss": 3.369894042141453, + "tokens_seen": 2487287808 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1398737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.104274272918701, + "objective/train/theoretical_loss": 3.369886952471647, + "objective/train/tokens_used": 846212576, + "theoretical_loss": 3.369886952471647, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505454545454546, + "loss": 2.7511, + "theoretical_loss": 3.369886952471647, + "tokens_seen": 2487353344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025052525252525253, + "loss": 2.7345, + "theoretical_loss": 3.3698798630409366, + "tokens_seen": 2487418880 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002505050505050505, + "loss": 2.575, + "theoretical_loss": 3.369872773849307, + "tokens_seen": 2487484416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504848484848485, + "loss": 2.4675, + "theoretical_loss": 3.3698656848967437, + "tokens_seen": 2487549952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025046464646464646, + "loss": 2.4035, + "theoretical_loss": 3.3698585961832332, + "tokens_seen": 2487615488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025044444444444447, + "loss": 2.7996, + "theoretical_loss": 3.36985150770876, + "tokens_seen": 2487681024 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002504242424242424, + "loss": 2.4979, + "theoretical_loss": 3.369844419473311, + "tokens_seen": 2487746560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025040404040404043, + "loss": 2.6051, + "theoretical_loss": 3.369837331476871, + "tokens_seen": 2487812096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503838383838384, + "loss": 2.7788, + "theoretical_loss": 3.3698302437194254, + "tokens_seen": 2487877632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025036363636363635, + "loss": 2.7086, + "theoretical_loss": 3.369823156200961, + "tokens_seen": 2487943168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025034343434343436, + "loss": 2.5893, + "theoretical_loss": 3.369816068921462, + "tokens_seen": 2488008704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503232323232323, + "loss": 2.6125, + "theoretical_loss": 3.369808981880915, + "tokens_seen": 2488074240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002503030303030303, + "loss": 2.7532, + "theoretical_loss": 3.3698018950793056, + "tokens_seen": 2488139776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502828282828283, + "loss": 2.605, + "theoretical_loss": 3.369794808516619, + "tokens_seen": 2488205312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025026262626262624, + "loss": 2.6247, + "theoretical_loss": 3.369787722192841, + "tokens_seen": 2488270848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025024242424242425, + "loss": 2.6183, + "theoretical_loss": 3.369780636107958, + "tokens_seen": 2488336384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025022222222222226, + "loss": 2.5234, + "theoretical_loss": 3.3697735502619546, + "tokens_seen": 2488401920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002502020202020202, + "loss": 2.5538, + "theoretical_loss": 3.369766464654817, + "tokens_seen": 2488467456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501818181818182, + "loss": 2.8815, + "theoretical_loss": 3.369759379286531, + "tokens_seen": 2488532992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025016161616161613, + "loss": 2.6823, + "theoretical_loss": 3.3697522941570814, + "tokens_seen": 2488598528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025014141414141414, + "loss": 2.5508, + "theoretical_loss": 3.369745209266455, + "tokens_seen": 2488664064 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025012121212121216, + "loss": 2.6801, + "theoretical_loss": 3.3697381246146363, + "tokens_seen": 2488729600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0002501010101010101, + "loss": 2.7711, + "theoretical_loss": 3.369731040201612, + "tokens_seen": 2488795136 + }, + { + "epoch": 0.5, + "learning_rate": 0.00025008080808080807, + "loss": 2.7317, + "theoretical_loss": 3.369723956027367, + "tokens_seen": 2488860672 + }, + { + "epoch": 0.5, + "learning_rate": 0.000250060606060606, + "loss": 2.7005, + "theoretical_loss": 3.369716872091888, + "tokens_seen": 2488926208 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 1399945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.502380132675171, + "objective/train/theoretical_loss": 3.3697097883951597, + "objective/train/tokens_used": 847850976, + "theoretical_loss": 3.3697097883951597, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002500404040404041, + "loss": 2.4684, + "theoretical_loss": 3.3697097883951597, + "tokens_seen": 2488991744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025002020202020205, + "loss": 2.6675, + "theoretical_loss": 3.3697027049371675, + "tokens_seen": 2489057280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00025, + "loss": 2.5078, + "theoretical_loss": 3.369695621717898, + "tokens_seen": 2489122816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024997979797979796, + "loss": 2.7762, + "theoretical_loss": 3.369688538737337, + "tokens_seen": 2489188352 + }, + { + "epoch": 0.51, + "learning_rate": 0.000249959595959596, + "loss": 2.7122, + "theoretical_loss": 3.369681455995469, + "tokens_seen": 2489253888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024993939393939393, + "loss": 2.5194, + "theoretical_loss": 3.3696743734922805, + "tokens_seen": 2489319424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024991919191919194, + "loss": 2.6868, + "theoretical_loss": 3.369667291227757, + "tokens_seen": 2489384960 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498989898989899, + "loss": 2.6136, + "theoretical_loss": 3.369660209201884, + "tokens_seen": 2489450496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024987878787878785, + "loss": 2.6027, + "theoretical_loss": 3.3696531274146477, + "tokens_seen": 2489516032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024985858585858587, + "loss": 2.7541, + "theoretical_loss": 3.369646045866033, + "tokens_seen": 2489581568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002498383838383838, + "loss": 2.7099, + "theoretical_loss": 3.3696389645560263, + "tokens_seen": 2489647104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024981818181818183, + "loss": 2.8117, + "theoretical_loss": 3.369631883484613, + "tokens_seen": 2489712640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497979797979798, + "loss": 2.8371, + "theoretical_loss": 3.3696248026517788, + "tokens_seen": 2489778176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024977777777777775, + "loss": 2.9717, + "theoretical_loss": 3.3696177220575096, + "tokens_seen": 2489843712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024975757575757576, + "loss": 2.8243, + "theoretical_loss": 3.36961064170179, + "tokens_seen": 2489909248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497373737373737, + "loss": 2.6394, + "theoretical_loss": 3.369603561584607, + "tokens_seen": 2489974784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002497171717171717, + "loss": 2.7539, + "theoretical_loss": 3.369596481705946, + "tokens_seen": 2490040320 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496969696969697, + "loss": 2.656, + "theoretical_loss": 3.3695894020657926, + "tokens_seen": 2490105856 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496767676767677, + "loss": 2.6401, + "theoretical_loss": 3.3695823226641317, + "tokens_seen": 2490171392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024965656565656565, + "loss": 2.5983, + "theoretical_loss": 3.3695752435009503, + "tokens_seen": 2490236928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496363636363636, + "loss": 3.0283, + "theoretical_loss": 3.369568164576233, + "tokens_seen": 2490302464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002496161616161616, + "loss": 2.9296, + "theoretical_loss": 3.3695610858899663, + "tokens_seen": 2490368000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495959595959596, + "loss": 2.8254, + "theoretical_loss": 3.3695540074421353, + "tokens_seen": 2490433536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495757575757576, + "loss": 2.7005, + "theoretical_loss": 3.369546929232726, + "tokens_seen": 2490499072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024955555555555554, + "loss": 2.4921, + "theoretical_loss": 3.369539851261724, + "tokens_seen": 2490564608 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1400696, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7776665687561035, + "objective/train/theoretical_loss": 3.369532773529115, + "objective/train/tokens_used": 849489376, + "theoretical_loss": 3.369532773529115, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024953535353535355, + "loss": 2.8002, + "theoretical_loss": 3.369532773529115, + "tokens_seen": 2490630144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002495151515151515, + "loss": 2.6144, + "theoretical_loss": 3.369525696034885, + "tokens_seen": 2490695680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494949494949495, + "loss": 2.4381, + "theoretical_loss": 3.3695186187790194, + "tokens_seen": 2490761216 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494747474747475, + "loss": 2.5728, + "theoretical_loss": 3.3695115417615034, + "tokens_seen": 2490826752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024945454545454544, + "loss": 2.7445, + "theoretical_loss": 3.369504464982324, + "tokens_seen": 2490892288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024943434343434345, + "loss": 2.7509, + "theoretical_loss": 3.3694973884414656, + "tokens_seen": 2490957824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002494141414141414, + "loss": 2.4403, + "theoretical_loss": 3.3694903121389146, + "tokens_seen": 2491023360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493939393939394, + "loss": 2.9707, + "theoretical_loss": 3.3694832360746565, + "tokens_seen": 2491088896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024937373737373737, + "loss": 2.8323, + "theoretical_loss": 3.369476160248677, + "tokens_seen": 2491154432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002493535353535354, + "loss": 2.7173, + "theoretical_loss": 3.369469084660962, + "tokens_seen": 2491219968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024933333333333334, + "loss": 2.8229, + "theoretical_loss": 3.369462009311497, + "tokens_seen": 2491285504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024931313131313135, + "loss": 2.6219, + "theoretical_loss": 3.3694549342002675, + "tokens_seen": 2491351040 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492929292929293, + "loss": 2.4854, + "theoretical_loss": 3.36944785932726, + "tokens_seen": 2491416576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024927272727272727, + "loss": 2.4852, + "theoretical_loss": 3.3694407846924594, + "tokens_seen": 2491482112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002492525252525253, + "loss": 2.5952, + "theoretical_loss": 3.369433710295852, + "tokens_seen": 2491547648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024923232323232323, + "loss": 2.6795, + "theoretical_loss": 3.3694266361374225, + "tokens_seen": 2491613184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024921212121212124, + "loss": 2.7402, + "theoretical_loss": 3.3694195622171583, + "tokens_seen": 2491678720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491919191919192, + "loss": 2.4222, + "theoretical_loss": 3.3694124885350436, + "tokens_seen": 2491744256 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491717171717172, + "loss": 2.7466, + "theoretical_loss": 3.369405415091065, + "tokens_seen": 2491809792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024915151515151517, + "loss": 2.5755, + "theoretical_loss": 3.3693983418852076, + "tokens_seen": 2491875328 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002491313131313131, + "loss": 2.6319, + "theoretical_loss": 3.3693912689174574, + "tokens_seen": 2491940864 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024911111111111114, + "loss": 2.4697, + "theoretical_loss": 3.3693841961878004, + "tokens_seen": 2492006400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490909090909091, + "loss": 2.5696, + "theoretical_loss": 3.3693771236962218, + "tokens_seen": 2492071936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002490707070707071, + "loss": 2.659, + "theoretical_loss": 3.369370051442708, + "tokens_seen": 2492137472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024905050505050506, + "loss": 2.6054, + "theoretical_loss": 3.369362979427244, + "tokens_seen": 2492203008 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1402287, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5457451343536377, + "objective/train/theoretical_loss": 3.369355907649816, + "objective/train/tokens_used": 851127776, + "theoretical_loss": 3.369355907649816, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 0.000249030303030303, + "loss": 2.8176, + "theoretical_loss": 3.369355907649816, + "tokens_seen": 2492268544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024901010101010103, + "loss": 2.8113, + "theoretical_loss": 3.3693488361104094, + "tokens_seen": 2492334080 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248989898989899, + "loss": 2.3995, + "theoretical_loss": 3.3693417648090103, + "tokens_seen": 2492399616 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248969696969697, + "loss": 2.553, + "theoretical_loss": 3.3693346937456043, + "tokens_seen": 2492465152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024894949494949495, + "loss": 2.6471, + "theoretical_loss": 3.369327622920177, + "tokens_seen": 2492530688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489292929292929, + "loss": 2.467, + "theoretical_loss": 3.369320552332714, + "tokens_seen": 2492596224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002489090909090909, + "loss": 2.5759, + "theoretical_loss": 3.3693134819832014, + "tokens_seen": 2492661760 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488888888888889, + "loss": 2.5294, + "theoretical_loss": 3.3693064118716247, + "tokens_seen": 2492727296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488686868686869, + "loss": 2.6284, + "theoretical_loss": 3.36929934199797, + "tokens_seen": 2492792832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024884848484848485, + "loss": 2.8887, + "theoretical_loss": 3.3692922723622223, + "tokens_seen": 2492858368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488282828282828, + "loss": 2.6152, + "theoretical_loss": 3.3692852029643685, + "tokens_seen": 2492923904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002488080808080808, + "loss": 2.732, + "theoretical_loss": 3.369278133804393, + "tokens_seen": 2492989440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024878787878787877, + "loss": 2.5747, + "theoretical_loss": 3.3692710648822826, + "tokens_seen": 2493054976 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487676767676768, + "loss": 2.7934, + "theoretical_loss": 3.369263996198022, + "tokens_seen": 2493120512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024874747474747474, + "loss": 2.4929, + "theoretical_loss": 3.3692569277515982, + "tokens_seen": 2493186048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487272727272727, + "loss": 2.5122, + "theoretical_loss": 3.3692498595429963, + "tokens_seen": 2493251584 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002487070707070707, + "loss": 2.5454, + "theoretical_loss": 3.369242791572202, + "tokens_seen": 2493317120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024868686868686866, + "loss": 2.4317, + "theoretical_loss": 3.369235723839201, + "tokens_seen": 2493382656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486666666666667, + "loss": 2.6154, + "theoretical_loss": 3.369228656343979, + "tokens_seen": 2493448192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024864646464646463, + "loss": 2.6502, + "theoretical_loss": 3.3692215890865223, + "tokens_seen": 2493513728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024862626262626264, + "loss": 2.66, + "theoretical_loss": 3.369214522066816, + "tokens_seen": 2493579264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002486060606060606, + "loss": 2.8713, + "theoretical_loss": 3.3692074552848457, + "tokens_seen": 2493644800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024858585858585856, + "loss": 2.9102, + "theoretical_loss": 3.369200388740598, + "tokens_seen": 2493710336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024856565656565657, + "loss": 2.6287, + "theoretical_loss": 3.3691933224340587, + "tokens_seen": 2493775872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485454545454545, + "loss": 2.651, + "theoretical_loss": 3.3691862563652126, + "tokens_seen": 2493841408 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1402946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.767141342163086, + "objective/train/theoretical_loss": 3.369179190534046, + "objective/train/tokens_used": 852766176, + "theoretical_loss": 3.369179190534046, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024852525252525254, + "loss": 2.8939, + "theoretical_loss": 3.369179190534046, + "tokens_seen": 2493906944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002485050505050505, + "loss": 2.5649, + "theoretical_loss": 3.3691721249405444, + "tokens_seen": 2493972480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484848484848485, + "loss": 2.6528, + "theoretical_loss": 3.3691650595846943, + "tokens_seen": 2494038016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024846464646464646, + "loss": 2.67, + "theoretical_loss": 3.3691579944664802, + "tokens_seen": 2494103552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024844444444444447, + "loss": 2.627, + "theoretical_loss": 3.3691509295858895, + "tokens_seen": 2494169088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024842424242424243, + "loss": 2.7311, + "theoretical_loss": 3.3691438649429064, + "tokens_seen": 2494234624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002484040404040404, + "loss": 2.4338, + "theoretical_loss": 3.3691368005375173, + "tokens_seen": 2494300160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483838383838384, + "loss": 2.8225, + "theoretical_loss": 3.3691297363697084, + "tokens_seen": 2494365696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024836363636363635, + "loss": 2.7583, + "theoretical_loss": 3.369122672439465, + "tokens_seen": 2494431232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024834343434343436, + "loss": 2.5668, + "theoretical_loss": 3.3691156087467724, + "tokens_seen": 2494496768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002483232323232323, + "loss": 2.6231, + "theoretical_loss": 3.3691085452916174, + "tokens_seen": 2494562304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024830303030303033, + "loss": 2.7638, + "theoretical_loss": 3.369101482073985, + "tokens_seen": 2494627840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482828282828283, + "loss": 2.6373, + "theoretical_loss": 3.3690944190938614, + "tokens_seen": 2494693376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024826262626262625, + "loss": 2.6464, + "theoretical_loss": 3.3690873563512325, + "tokens_seen": 2494758912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024824242424242426, + "loss": 2.7628, + "theoretical_loss": 3.3690802938460833, + "tokens_seen": 2494824448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482222222222222, + "loss": 2.6169, + "theoretical_loss": 3.3690732315784, + "tokens_seen": 2494889984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002482020202020202, + "loss": 2.7709, + "theoretical_loss": 3.369066169548169, + "tokens_seen": 2494955520 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481818181818182, + "loss": 2.6001, + "theoretical_loss": 3.3690591077553753, + "tokens_seen": 2495021056 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481616161616162, + "loss": 2.4049, + "theoretical_loss": 3.3690520462000046, + "tokens_seen": 2495086592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024814141414141415, + "loss": 2.6124, + "theoretical_loss": 3.369044984882043, + "tokens_seen": 2495152128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024812121212121216, + "loss": 2.5184, + "theoretical_loss": 3.369037923801477, + "tokens_seen": 2495217664 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002481010101010101, + "loss": 2.4434, + "theoretical_loss": 3.369030862958291, + "tokens_seen": 2495283200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480808080808081, + "loss": 2.9209, + "theoretical_loss": 3.369023802352472, + "tokens_seen": 2495348736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002480606060606061, + "loss": 2.7159, + "theoretical_loss": 3.3690167419840047, + "tokens_seen": 2495414272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024804040404040404, + "loss": 2.388, + "theoretical_loss": 3.3690096818528756, + "tokens_seen": 2495479808 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1404179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3843698501586914, + "objective/train/theoretical_loss": 3.36900262195907, + "objective/train/tokens_used": 854404576, + "theoretical_loss": 3.36900262195907, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024802020202020205, + "loss": 2.7738, + "theoretical_loss": 3.36900262195907, + "tokens_seen": 2495545344 + }, + { + "epoch": 0.51, + "learning_rate": 0.000248, + "loss": 2.664, + "theoretical_loss": 3.3689955623025747, + "tokens_seen": 2495610880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024797979797979797, + "loss": 2.401, + "theoretical_loss": 3.368988502883374, + "tokens_seen": 2495676416 + }, + { + "epoch": 0.51, + "learning_rate": 0.000247959595959596, + "loss": 2.7789, + "theoretical_loss": 3.368981443701455, + "tokens_seen": 2495741952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024793939393939394, + "loss": 2.6515, + "theoretical_loss": 3.368974384756803, + "tokens_seen": 2495807488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024791919191919195, + "loss": 2.6769, + "theoretical_loss": 3.3689673260494035, + "tokens_seen": 2495873024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002478989898989899, + "loss": 2.4972, + "theoretical_loss": 3.3689602675792427, + "tokens_seen": 2495938560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024787878787878786, + "loss": 2.6512, + "theoretical_loss": 3.368953209346306, + "tokens_seen": 2496004096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024785858585858587, + "loss": 2.5758, + "theoretical_loss": 3.36894615135058, + "tokens_seen": 2496069632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024783838383838383, + "loss": 2.5826, + "theoretical_loss": 3.36893909359205, + "tokens_seen": 2496135168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024781818181818184, + "loss": 2.661, + "theoretical_loss": 3.368932036070701, + "tokens_seen": 2496200704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477979797979798, + "loss": 2.6731, + "theoretical_loss": 3.36892497878652, + "tokens_seen": 2496266240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477777777777778, + "loss": 2.6617, + "theoretical_loss": 3.3689179217394924, + "tokens_seen": 2496331776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024775757575757576, + "loss": 2.5079, + "theoretical_loss": 3.368910864929604, + "tokens_seen": 2496397312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002477373737373737, + "loss": 2.6326, + "theoretical_loss": 3.3689038083568406, + "tokens_seen": 2496462848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024771717171717173, + "loss": 2.5912, + "theoretical_loss": 3.368896752021188, + "tokens_seen": 2496528384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476969696969697, + "loss": 2.6415, + "theoretical_loss": 3.3688896959226318, + "tokens_seen": 2496593920 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476767676767677, + "loss": 2.405, + "theoretical_loss": 3.368882640061158, + "tokens_seen": 2496659456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024765656565656566, + "loss": 2.6989, + "theoretical_loss": 3.3688755844367524, + "tokens_seen": 2496724992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476363636363636, + "loss": 2.5974, + "theoretical_loss": 3.368868529049401, + "tokens_seen": 2496790528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002476161616161616, + "loss": 2.6055, + "theoretical_loss": 3.3688614738990896, + "tokens_seen": 2496856064 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475959595959596, + "loss": 2.603, + "theoretical_loss": 3.368854418985804, + "tokens_seen": 2496921600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475757575757576, + "loss": 2.4991, + "theoretical_loss": 3.3688473643095294, + "tokens_seen": 2496987136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024755555555555555, + "loss": 2.7993, + "theoretical_loss": 3.368840309870252, + "tokens_seen": 2497052672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475353535353535, + "loss": 2.6738, + "theoretical_loss": 3.3688332556679583, + "tokens_seen": 2497118208 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1405043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676100015640259, + "objective/train/theoretical_loss": 3.368826201702633, + "objective/train/tokens_used": 856042976, + "theoretical_loss": 3.368826201702633, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002475151515151515, + "loss": 2.6648, + "theoretical_loss": 3.368826201702633, + "tokens_seen": 2497183744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474949494949495, + "loss": 2.4154, + "theoretical_loss": 3.368819147974263, + "tokens_seen": 2497249280 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474747474747475, + "loss": 2.6635, + "theoretical_loss": 3.3688120944828333, + "tokens_seen": 2497314816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024745454545454544, + "loss": 2.5889, + "theoretical_loss": 3.36880504122833, + "tokens_seen": 2497380352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024743434343434345, + "loss": 2.3859, + "theoretical_loss": 3.368797988210739, + "tokens_seen": 2497445888 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002474141414141414, + "loss": 2.7261, + "theoretical_loss": 3.3687909354300456, + "tokens_seen": 2497511424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024739393939393937, + "loss": 2.7097, + "theoretical_loss": 3.3687838828862366, + "tokens_seen": 2497576960 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473737373737374, + "loss": 2.5528, + "theoretical_loss": 3.368776830579297, + "tokens_seen": 2497642496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024735353535353533, + "loss": 2.6589, + "theoretical_loss": 3.368769778509213, + "tokens_seen": 2497708032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024733333333333335, + "loss": 2.8677, + "theoretical_loss": 3.3687627266759703, + "tokens_seen": 2497773568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002473131313131313, + "loss": 2.6393, + "theoretical_loss": 3.368755675079555, + "tokens_seen": 2497839104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472929292929293, + "loss": 2.7135, + "theoretical_loss": 3.3687486237199527, + "tokens_seen": 2497904640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024727272727272727, + "loss": 2.7617, + "theoretical_loss": 3.3687415725971492, + "tokens_seen": 2497970176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472525252525253, + "loss": 2.6596, + "theoretical_loss": 3.36873452171113, + "tokens_seen": 2498035712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024723232323232324, + "loss": 2.6921, + "theoretical_loss": 3.368727471061882, + "tokens_seen": 2498101248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002472121212121212, + "loss": 2.6367, + "theoretical_loss": 3.3687204206493906, + "tokens_seen": 2498166784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471919191919192, + "loss": 2.4278, + "theoretical_loss": 3.3687133704736407, + "tokens_seen": 2498232320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024717171717171716, + "loss": 2.5444, + "theoretical_loss": 3.368706320534619, + "tokens_seen": 2498297856 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002471515151515152, + "loss": 2.7102, + "theoretical_loss": 3.3686992708323116, + "tokens_seen": 2498363392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024713131313131313, + "loss": 2.3439, + "theoretical_loss": 3.3686922213667034, + "tokens_seen": 2498428928 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024711111111111114, + "loss": 2.4562, + "theoretical_loss": 3.368685172137781, + "tokens_seen": 2498494464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470909090909091, + "loss": 2.6374, + "theoretical_loss": 3.3686781231455303, + "tokens_seen": 2498560000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002470707070707071, + "loss": 2.6192, + "theoretical_loss": 3.3686710743899364, + "tokens_seen": 2498625536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024705050505050507, + "loss": 2.7187, + "theoretical_loss": 3.368664025870986, + "tokens_seen": 2498691072 + }, + { + "epoch": 0.51, + "learning_rate": 0.000247030303030303, + "loss": 2.4662, + "theoretical_loss": 3.3686569775886643, + "tokens_seen": 2498756608 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1405984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6698999404907227, + "objective/train/theoretical_loss": 3.3686499295429577, + "objective/train/tokens_used": 857681376, + "theoretical_loss": 3.3686499295429577, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024701010101010104, + "loss": 2.6677, + "theoretical_loss": 3.3686499295429577, + "tokens_seen": 2498822144 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246989898989899, + "loss": 2.7627, + "theoretical_loss": 3.3686428817338516, + "tokens_seen": 2498887680 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246969696969697, + "loss": 2.6063, + "theoretical_loss": 3.368635834161332, + "tokens_seen": 2498953216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024694949494949496, + "loss": 2.4547, + "theoretical_loss": 3.3686287868253846, + "tokens_seen": 2499018752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024692929292929297, + "loss": 2.4741, + "theoretical_loss": 3.3686217397259957, + "tokens_seen": 2499084288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024690909090909093, + "loss": 2.6723, + "theoretical_loss": 3.368614692863151, + "tokens_seen": 2499149824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468888888888889, + "loss": 2.5823, + "theoretical_loss": 3.368607646236836, + "tokens_seen": 2499215360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468686868686869, + "loss": 2.506, + "theoretical_loss": 3.3686005998470367, + "tokens_seen": 2499280896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024684848484848485, + "loss": 2.4078, + "theoretical_loss": 3.3685935536937395, + "tokens_seen": 2499346432 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024682828282828286, + "loss": 2.6809, + "theoretical_loss": 3.3685865077769295, + "tokens_seen": 2499411968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002468080808080808, + "loss": 2.5419, + "theoretical_loss": 3.368579462096593, + "tokens_seen": 2499477504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467878787878788, + "loss": 2.7292, + "theoretical_loss": 3.368572416652716, + "tokens_seen": 2499543040 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467676767676768, + "loss": 2.2679, + "theoretical_loss": 3.3685653714452837, + "tokens_seen": 2499608576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024674747474747475, + "loss": 2.3969, + "theoretical_loss": 3.3685583264742824, + "tokens_seen": 2499674112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024672727272727276, + "loss": 2.7432, + "theoretical_loss": 3.3685512817396983, + "tokens_seen": 2499739648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002467070707070707, + "loss": 2.673, + "theoretical_loss": 3.3685442372415166, + "tokens_seen": 2499805184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024668686868686867, + "loss": 2.5325, + "theoretical_loss": 3.3685371929797236, + "tokens_seen": 2499870720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002466666666666667, + "loss": 2.5822, + "theoretical_loss": 3.368530148954305, + "tokens_seen": 2499936256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024664646464646464, + "loss": 2.6258, + "theoretical_loss": 3.368523105165247, + "tokens_seen": 2500001792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024662626262626265, + "loss": 2.9334, + "theoretical_loss": 3.3685160616125347, + "tokens_seen": 2500067328 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002466060606060606, + "loss": 2.6699, + "theoretical_loss": 3.368509018296155, + "tokens_seen": 2500132864 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024658585858585856, + "loss": 2.768, + "theoretical_loss": 3.368501975216093, + "tokens_seen": 2500198400 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465656565656566, + "loss": 2.7122, + "theoretical_loss": 3.368494932372335, + "tokens_seen": 2500263936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024654545454545453, + "loss": 2.7791, + "theoretical_loss": 3.3684878897648662, + "tokens_seen": 2500329472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024652525252525254, + "loss": 2.4732, + "theoretical_loss": 3.3684808473936734, + "tokens_seen": 2500395008 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1406694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.604163408279419, + "objective/train/theoretical_loss": 3.368473805258742, + "objective/train/tokens_used": 859319776, + "theoretical_loss": 3.368473805258742, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002465050505050505, + "loss": 2.6993, + "theoretical_loss": 3.368473805258742, + "tokens_seen": 2500460544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024648484848484846, + "loss": 2.6412, + "theoretical_loss": 3.3684667633600585, + "tokens_seen": 2500526080 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024646464646464647, + "loss": 2.6743, + "theoretical_loss": 3.3684597216976075, + "tokens_seen": 2500591616 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464444444444444, + "loss": 2.7191, + "theoretical_loss": 3.3684526802713757, + "tokens_seen": 2500657152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024642424242424243, + "loss": 2.5174, + "theoretical_loss": 3.368445639081349, + "tokens_seen": 2500722688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002464040404040404, + "loss": 2.8512, + "theoretical_loss": 3.3684385981275136, + "tokens_seen": 2500788224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463838383838384, + "loss": 2.7571, + "theoretical_loss": 3.3684315574098544, + "tokens_seen": 2500853760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024636363636363636, + "loss": 2.6315, + "theoretical_loss": 3.368424516928358, + "tokens_seen": 2500919296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463434343434343, + "loss": 2.5968, + "theoretical_loss": 3.3684174766830104, + "tokens_seen": 2500984832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024632323232323233, + "loss": 2.6307, + "theoretical_loss": 3.3684104366737975, + "tokens_seen": 2501050368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002463030303030303, + "loss": 2.5847, + "theoretical_loss": 3.368403396900704, + "tokens_seen": 2501115904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462828282828283, + "loss": 2.4898, + "theoretical_loss": 3.368396357363718, + "tokens_seen": 2501181440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024626262626262625, + "loss": 2.5648, + "theoretical_loss": 3.368389318062823, + "tokens_seen": 2501246976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024624242424242426, + "loss": 2.4336, + "theoretical_loss": 3.3683822789980065, + "tokens_seen": 2501312512 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002462222222222222, + "loss": 2.7479, + "theoretical_loss": 3.3683752401692537, + "tokens_seen": 2501378048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024620202020202023, + "loss": 2.5121, + "theoretical_loss": 3.3683682015765513, + "tokens_seen": 2501443584 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461818181818182, + "loss": 2.4414, + "theoretical_loss": 3.368361163219884, + "tokens_seen": 2501509120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024616161616161614, + "loss": 2.5248, + "theoretical_loss": 3.3683541250992386, + "tokens_seen": 2501574656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024614141414141416, + "loss": 2.6208, + "theoretical_loss": 3.368347087214601, + "tokens_seen": 2501640192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461212121212121, + "loss": 2.6006, + "theoretical_loss": 3.368340049565956, + "tokens_seen": 2501705728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002461010101010101, + "loss": 2.5475, + "theoretical_loss": 3.368333012153291, + "tokens_seen": 2501771264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460808080808081, + "loss": 2.4524, + "theoretical_loss": 3.368325974976591, + "tokens_seen": 2501836800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002460606060606061, + "loss": 2.5894, + "theoretical_loss": 3.3683189380358423, + "tokens_seen": 2501902336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024604040404040405, + "loss": 2.6401, + "theoretical_loss": 3.3683119013310305, + "tokens_seen": 2501967872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024602020202020206, + "loss": 2.7207, + "theoretical_loss": 3.368304864862142, + "tokens_seen": 2502033408 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1408162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8449513912200928, + "objective/train/theoretical_loss": 3.3682978286291623, + "objective/train/tokens_used": 860958176, + "theoretical_loss": 3.3682978286291623, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.51, + "learning_rate": 0.000246, + "loss": 2.6623, + "theoretical_loss": 3.3682978286291623, + "tokens_seen": 2502098944 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245979797979798, + "loss": 2.9538, + "theoretical_loss": 3.3682907926320773, + "tokens_seen": 2502164480 + }, + { + "epoch": 0.51, + "learning_rate": 0.000245959595959596, + "loss": 2.5054, + "theoretical_loss": 3.3682837568708726, + "tokens_seen": 2502230016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024593939393939394, + "loss": 2.5128, + "theoretical_loss": 3.368276721345535, + "tokens_seen": 2502295552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024591919191919195, + "loss": 2.5532, + "theoretical_loss": 3.36826968605605, + "tokens_seen": 2502361088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458989898989899, + "loss": 2.6788, + "theoretical_loss": 3.368262651002403, + "tokens_seen": 2502426624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458787878787879, + "loss": 2.5133, + "theoretical_loss": 3.368255616184581, + "tokens_seen": 2502492160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002458585858585859, + "loss": 2.6366, + "theoretical_loss": 3.3682485816025687, + "tokens_seen": 2502557696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024583838383838383, + "loss": 2.3964, + "theoretical_loss": 3.368241547256353, + "tokens_seen": 2502623232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024581818181818184, + "loss": 2.6147, + "theoretical_loss": 3.3682345131459193, + "tokens_seen": 2502688768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457979797979798, + "loss": 2.6191, + "theoretical_loss": 3.3682274792712534, + "tokens_seen": 2502754304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457777777777778, + "loss": 2.5426, + "theoretical_loss": 3.3682204456323417, + "tokens_seen": 2502819840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024575757575757577, + "loss": 2.6156, + "theoretical_loss": 3.36821341222917, + "tokens_seen": 2502885376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002457373737373737, + "loss": 2.5283, + "theoretical_loss": 3.368206379061724, + "tokens_seen": 2502950912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024571717171717174, + "loss": 2.6356, + "theoretical_loss": 3.36819934612999, + "tokens_seen": 2503016448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456969696969697, + "loss": 2.572, + "theoretical_loss": 3.3681923134339535, + "tokens_seen": 2503081984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456767676767677, + "loss": 2.5903, + "theoretical_loss": 3.3681852809736004, + "tokens_seen": 2503147520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024565656565656566, + "loss": 2.6767, + "theoretical_loss": 3.368178248748917, + "tokens_seen": 2503213056 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002456363636363636, + "loss": 2.9661, + "theoretical_loss": 3.3681712167598894, + "tokens_seen": 2503278592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024561616161616163, + "loss": 2.7203, + "theoretical_loss": 3.3681641850065027, + "tokens_seen": 2503344128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455959595959596, + "loss": 2.4551, + "theoretical_loss": 3.3681571534887436, + "tokens_seen": 2503409664 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455757575757576, + "loss": 2.5751, + "theoretical_loss": 3.3681501222065977, + "tokens_seen": 2503475200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024555555555555556, + "loss": 2.8298, + "theoretical_loss": 3.3681430911600514, + "tokens_seen": 2503540736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455353535353535, + "loss": 2.7275, + "theoretical_loss": 3.36813606034909, + "tokens_seen": 2503606272 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002455151515151515, + "loss": 2.7162, + "theoretical_loss": 3.3681290297736997, + "tokens_seen": 2503671808 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1408714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3051252365112305, + "objective/train/theoretical_loss": 3.3681219994338663, + "objective/train/tokens_used": 862596576, + "theoretical_loss": 3.3681219994338663, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454949494949495, + "loss": 2.3913, + "theoretical_loss": 3.3681219994338663, + "tokens_seen": 2503737344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454747474747475, + "loss": 2.6925, + "theoretical_loss": 3.368114969329576, + "tokens_seen": 2503802880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024545454545454545, + "loss": 2.4519, + "theoretical_loss": 3.3681079394608147, + "tokens_seen": 2503868416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454343434343434, + "loss": 2.518, + "theoretical_loss": 3.368100909827568, + "tokens_seen": 2503933952 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002454141414141414, + "loss": 2.6436, + "theoretical_loss": 3.3680938804298224, + "tokens_seen": 2503999488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024539393939393937, + "loss": 2.5994, + "theoretical_loss": 3.3680868512675635, + "tokens_seen": 2504065024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453737373737374, + "loss": 2.6982, + "theoretical_loss": 3.3680798223407775, + "tokens_seen": 2504130560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024535353535353534, + "loss": 2.6399, + "theoretical_loss": 3.36807279364945, + "tokens_seen": 2504196096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024533333333333335, + "loss": 2.4939, + "theoretical_loss": 3.368065765193567, + "tokens_seen": 2504261632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002453131313131313, + "loss": 2.5215, + "theoretical_loss": 3.3680587369731145, + "tokens_seen": 2504327168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024529292929292927, + "loss": 2.6578, + "theoretical_loss": 3.3680517089880784, + "tokens_seen": 2504392704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452727272727273, + "loss": 2.5407, + "theoretical_loss": 3.3680446812384455, + "tokens_seen": 2504458240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024525252525252523, + "loss": 2.5215, + "theoretical_loss": 3.3680376537242003, + "tokens_seen": 2504523776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024523232323232324, + "loss": 2.8301, + "theoretical_loss": 3.36803062644533, + "tokens_seen": 2504589312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002452121212121212, + "loss": 2.5564, + "theoretical_loss": 3.3680235994018197, + "tokens_seen": 2504654848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451919191919192, + "loss": 2.8627, + "theoretical_loss": 3.368016572593656, + "tokens_seen": 2504720384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024517171717171717, + "loss": 2.6089, + "theoretical_loss": 3.3680095460208244, + "tokens_seen": 2504785920 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451515151515152, + "loss": 2.5078, + "theoretical_loss": 3.3680025196833108, + "tokens_seen": 2504851456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024513131313131314, + "loss": 2.56, + "theoretical_loss": 3.367995493581102, + "tokens_seen": 2504916992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002451111111111111, + "loss": 2.5702, + "theoretical_loss": 3.3679884677141825, + "tokens_seen": 2504982528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0002450909090909091, + "loss": 2.5326, + "theoretical_loss": 3.36798144208254, + "tokens_seen": 2505048064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024507070707070706, + "loss": 2.5272, + "theoretical_loss": 3.3679744166861587, + "tokens_seen": 2505113600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024505050505050507, + "loss": 2.7747, + "theoretical_loss": 3.367967391525026, + "tokens_seen": 2505179136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024503030303030303, + "loss": 2.8732, + "theoretical_loss": 3.367960366599127, + "tokens_seen": 2505244672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00024501010101010104, + "loss": 2.7472, + "theoretical_loss": 3.3679533419084486, + "tokens_seen": 2505310208 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 1409820, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.442836046218872, + "objective/train/theoretical_loss": 3.367946317452976, + "objective/train/tokens_used": 864234976, + "theoretical_loss": 3.367946317452976, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244989898989899, + "loss": 2.6548, + "theoretical_loss": 3.367946317452976, + "tokens_seen": 2505375744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024496969696969695, + "loss": 2.7266, + "theoretical_loss": 3.367939293232695, + "tokens_seen": 2505441280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024494949494949497, + "loss": 2.7366, + "theoretical_loss": 3.367932269247592, + "tokens_seen": 2505506816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002449292929292929, + "loss": 2.577, + "theoretical_loss": 3.367925245497653, + "tokens_seen": 2505572352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024490909090909093, + "loss": 2.6956, + "theoretical_loss": 3.3679182219828636, + "tokens_seen": 2505637888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448888888888889, + "loss": 2.8271, + "theoretical_loss": 3.36791119870321, + "tokens_seen": 2505703424 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448686868686869, + "loss": 2.5368, + "theoretical_loss": 3.3679041756586785, + "tokens_seen": 2505768960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024484848484848486, + "loss": 2.4616, + "theoretical_loss": 3.3678971528492547, + "tokens_seen": 2505834496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024482828282828287, + "loss": 2.6359, + "theoretical_loss": 3.3678901302749247, + "tokens_seen": 2505900032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002448080808080808, + "loss": 2.7814, + "theoretical_loss": 3.3678831079356746, + "tokens_seen": 2505965568 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447878787878788, + "loss": 2.5985, + "theoretical_loss": 3.3678760858314902, + "tokens_seen": 2506031104 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447676767676768, + "loss": 2.6763, + "theoretical_loss": 3.367869063962357, + "tokens_seen": 2506096640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024474747474747475, + "loss": 2.5736, + "theoretical_loss": 3.3678620423282624, + "tokens_seen": 2506162176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024472727272727276, + "loss": 2.7067, + "theoretical_loss": 3.367855020929191, + "tokens_seen": 2506227712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002447070707070707, + "loss": 2.4288, + "theoretical_loss": 3.367847999765129, + "tokens_seen": 2506293248 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446868686868687, + "loss": 2.7332, + "theoretical_loss": 3.367840978836063, + "tokens_seen": 2506358784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446666666666667, + "loss": 2.6814, + "theoretical_loss": 3.3678339581419787, + "tokens_seen": 2506424320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024464646464646464, + "loss": 2.6186, + "theoretical_loss": 3.367826937682862, + "tokens_seen": 2506489856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024462626262626265, + "loss": 2.7213, + "theoretical_loss": 3.367819917458699, + "tokens_seen": 2506555392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002446060606060606, + "loss": 2.6211, + "theoretical_loss": 3.3678128974694754, + "tokens_seen": 2506620928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024458585858585857, + "loss": 2.7147, + "theoretical_loss": 3.367805877715178, + "tokens_seen": 2506686464 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445656565656566, + "loss": 2.728, + "theoretical_loss": 3.367798858195792, + "tokens_seen": 2506752000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024454545454545454, + "loss": 2.5691, + "theoretical_loss": 3.3677918389113035, + "tokens_seen": 2506817536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024452525252525255, + "loss": 2.4405, + "theoretical_loss": 3.3677848198616984, + "tokens_seen": 2506883072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002445050505050505, + "loss": 2.4404, + "theoretical_loss": 3.3677778010469632, + "tokens_seen": 2506948608 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1410261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1492111682891846, + "objective/train/theoretical_loss": 3.3677707824670837, + "objective/train/tokens_used": 865873376, + "theoretical_loss": 3.3677707824670837, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024448484848484846, + "loss": 2.3862, + "theoretical_loss": 3.3677707824670837, + "tokens_seen": 2507014144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024446464646464647, + "loss": 2.739, + "theoretical_loss": 3.3677637641220457, + "tokens_seen": 2507079680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024444444444444443, + "loss": 2.5928, + "theoretical_loss": 3.3677567460118354, + "tokens_seen": 2507145216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024442424242424244, + "loss": 2.5821, + "theoretical_loss": 3.3677497281364386, + "tokens_seen": 2507210752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002444040404040404, + "loss": 2.5836, + "theoretical_loss": 3.3677427104958415, + "tokens_seen": 2507276288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443838383838384, + "loss": 2.4684, + "theoretical_loss": 3.3677356930900304, + "tokens_seen": 2507341824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024436363636363636, + "loss": 2.5223, + "theoretical_loss": 3.3677286759189906, + "tokens_seen": 2507407360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443434343434343, + "loss": 2.6002, + "theoretical_loss": 3.367721658982709, + "tokens_seen": 2507472896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024432323232323233, + "loss": 2.6708, + "theoretical_loss": 3.3677146422811703, + "tokens_seen": 2507538432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002443030303030303, + "loss": 2.557, + "theoretical_loss": 3.3677076258143614, + "tokens_seen": 2507603968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442828282828283, + "loss": 2.6267, + "theoretical_loss": 3.367700609582269, + "tokens_seen": 2507669504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024426262626262626, + "loss": 2.6128, + "theoretical_loss": 3.3676935935848777, + "tokens_seen": 2507735040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442424242424242, + "loss": 2.6734, + "theoretical_loss": 3.367686577822174, + "tokens_seen": 2507800576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442222222222222, + "loss": 2.501, + "theoretical_loss": 3.3676795622941444, + "tokens_seen": 2507866112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002442020202020202, + "loss": 2.7571, + "theoretical_loss": 3.3676725470007747, + "tokens_seen": 2507931648 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002441818181818182, + "loss": 2.6032, + "theoretical_loss": 3.3676655319420505, + "tokens_seen": 2507997184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024416161616161615, + "loss": 2.6836, + "theoretical_loss": 3.3676585171179583, + "tokens_seen": 2508062720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024414141414141416, + "loss": 2.6438, + "theoretical_loss": 3.3676515025284837, + "tokens_seen": 2508128256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024412121212121215, + "loss": 2.6899, + "theoretical_loss": 3.367644488173613, + "tokens_seen": 2508193792 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002441010101010101, + "loss": 2.5892, + "theoretical_loss": 3.3676374740533324, + "tokens_seen": 2508259328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024408080808080809, + "loss": 2.6819, + "theoretical_loss": 3.3676304601676277, + "tokens_seen": 2508324864 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024406060606060607, + "loss": 2.6108, + "theoretical_loss": 3.3676234465164847, + "tokens_seen": 2508390400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024404040404040403, + "loss": 2.5527, + "theoretical_loss": 3.3676164330998897, + "tokens_seen": 2508455936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024402020202020204, + "loss": 2.646, + "theoretical_loss": 3.367609419917829, + "tokens_seen": 2508521472 + }, + { + "epoch": 0.52, + "learning_rate": 0.000244, + "loss": 2.5518, + "theoretical_loss": 3.367602406970288, + "tokens_seen": 2508587008 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1411438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.317744731903076, + "objective/train/theoretical_loss": 3.367595394257253, + "objective/train/tokens_used": 867511776, + "theoretical_loss": 3.367595394257253, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 0.000243979797979798, + "loss": 2.6732, + "theoretical_loss": 3.367595394257253, + "tokens_seen": 2508652544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024395959595959596, + "loss": 2.5419, + "theoretical_loss": 3.3675883817787104, + "tokens_seen": 2508718080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024393939393939392, + "loss": 2.3859, + "theoretical_loss": 3.367581369534646, + "tokens_seen": 2508783616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024391919191919193, + "loss": 2.6912, + "theoretical_loss": 3.3675743575250454, + "tokens_seen": 2508849152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438989898989899, + "loss": 2.5967, + "theoretical_loss": 3.367567345749895, + "tokens_seen": 2508914688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002438787878787879, + "loss": 2.7105, + "theoretical_loss": 3.3675603342091813, + "tokens_seen": 2508980224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024385858585858586, + "loss": 2.5176, + "theoretical_loss": 3.3675533229028893, + "tokens_seen": 2509045760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024383838383838384, + "loss": 2.7323, + "theoretical_loss": 3.367546311831006, + "tokens_seen": 2509111296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024381818181818182, + "loss": 2.4613, + "theoretical_loss": 3.3675393009935166, + "tokens_seen": 2509176832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437979797979798, + "loss": 2.4941, + "theoretical_loss": 3.367532290390408, + "tokens_seen": 2509242368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002437777777777778, + "loss": 2.5183, + "theoretical_loss": 3.3675252800216655, + "tokens_seen": 2509307904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024375757575757575, + "loss": 2.5193, + "theoretical_loss": 3.3675182698872757, + "tokens_seen": 2509373440 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024373737373737373, + "loss": 2.8342, + "theoretical_loss": 3.3675112599872246, + "tokens_seen": 2509438976 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024371717171717172, + "loss": 2.8013, + "theoretical_loss": 3.367504250321498, + "tokens_seen": 2509504512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002436969696969697, + "loss": 2.5729, + "theoretical_loss": 3.367497240890082, + "tokens_seen": 2509570048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024367676767676768, + "loss": 2.4149, + "theoretical_loss": 3.3674902316929622, + "tokens_seen": 2509635584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024365656565656567, + "loss": 2.5487, + "theoretical_loss": 3.3674832227301255, + "tokens_seen": 2509701120 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024363636363636362, + "loss": 2.7993, + "theoretical_loss": 3.3674762140015577, + "tokens_seen": 2509766656 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024361616161616164, + "loss": 2.6208, + "theoretical_loss": 3.3674692055072444, + "tokens_seen": 2509832192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002435959595959596, + "loss": 2.5095, + "theoretical_loss": 3.367462197247172, + "tokens_seen": 2509897728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024357575757575758, + "loss": 2.6433, + "theoretical_loss": 3.367455189221327, + "tokens_seen": 2509963264 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024355555555555556, + "loss": 2.4702, + "theoretical_loss": 3.367448181429695, + "tokens_seen": 2510028800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024353535353535352, + "loss": 2.7543, + "theoretical_loss": 3.3674411738722614, + "tokens_seen": 2510094336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024351515151515153, + "loss": 2.6611, + "theoretical_loss": 3.367434166549013, + "tokens_seen": 2510159872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024349494949494949, + "loss": 2.8118, + "theoretical_loss": 3.367427159459936, + "tokens_seen": 2510225408 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1412115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.639223575592041, + "objective/train/theoretical_loss": 3.3674201526050163, + "objective/train/tokens_used": 869150176, + "theoretical_loss": 3.3674201526050163, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002434747474747475, + "loss": 2.6991, + "theoretical_loss": 3.3674201526050163, + "tokens_seen": 2510290944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024345454545454545, + "loss": 2.7534, + "theoretical_loss": 3.36741314598424, + "tokens_seen": 2510356480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024343434343434346, + "loss": 2.5342, + "theoretical_loss": 3.3674061395975925, + "tokens_seen": 2510422016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024341414141414142, + "loss": 2.7901, + "theoretical_loss": 3.3673991334450606, + "tokens_seen": 2510487552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024339393939393938, + "loss": 2.5615, + "theoretical_loss": 3.3673921275266303, + "tokens_seen": 2510553088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002433737373737374, + "loss": 2.7923, + "theoretical_loss": 3.367385121842288, + "tokens_seen": 2510618624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024335353535353535, + "loss": 2.7026, + "theoretical_loss": 3.3673781163920187, + "tokens_seen": 2510684160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024333333333333336, + "loss": 2.8343, + "theoretical_loss": 3.3673711111758093, + "tokens_seen": 2510749696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024331313131313131, + "loss": 2.6506, + "theoretical_loss": 3.3673641061936457, + "tokens_seen": 2510815232 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432929292929293, + "loss": 2.5119, + "theoretical_loss": 3.3673571014455135, + "tokens_seen": 2510880768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024327272727272728, + "loss": 2.6209, + "theoretical_loss": 3.3673500969314, + "tokens_seen": 2510946304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024325252525252527, + "loss": 2.5742, + "theoretical_loss": 3.3673430926512897, + "tokens_seen": 2511011840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024323232323232325, + "loss": 2.6907, + "theoretical_loss": 3.3673360886051698, + "tokens_seen": 2511077376 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002432121212121212, + "loss": 2.6287, + "theoretical_loss": 3.367329084793026, + "tokens_seen": 2511142912 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002431919191919192, + "loss": 2.6303, + "theoretical_loss": 3.3673220812148443, + "tokens_seen": 2511208448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024317171717171717, + "loss": 2.5732, + "theoretical_loss": 3.3673150778706114, + "tokens_seen": 2511273984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024315151515151516, + "loss": 2.6674, + "theoretical_loss": 3.367308074760312, + "tokens_seen": 2511339520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024313131313131314, + "loss": 2.4475, + "theoretical_loss": 3.367301071883934, + "tokens_seen": 2511405056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024311111111111113, + "loss": 2.4469, + "theoretical_loss": 3.3672940692414617, + "tokens_seen": 2511470592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024309090909090908, + "loss": 2.6926, + "theoretical_loss": 3.3672870668328825, + "tokens_seen": 2511536128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002430707070707071, + "loss": 2.5, + "theoretical_loss": 3.367280064658182, + "tokens_seen": 2511601664 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024305050505050505, + "loss": 2.6144, + "theoretical_loss": 3.367273062717346, + "tokens_seen": 2511667200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024303030303030304, + "loss": 2.5839, + "theoretical_loss": 3.367266061010361, + "tokens_seen": 2511732736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024301010101010102, + "loss": 2.6033, + "theoretical_loss": 3.367259059537213, + "tokens_seen": 2511798272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024298989898989898, + "loss": 2.4894, + "theoretical_loss": 3.367252058297888, + "tokens_seen": 2511863808 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1412775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9854464530944824, + "objective/train/theoretical_loss": 3.3672450572923722, + "objective/train/tokens_used": 870788576, + "theoretical_loss": 3.3672450572923722, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242969696969697, + "loss": 2.8172, + "theoretical_loss": 3.3672450572923722, + "tokens_seen": 2511929344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024294949494949494, + "loss": 2.807, + "theoretical_loss": 3.3672380565206517, + "tokens_seen": 2511994880 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024292929292929296, + "loss": 2.7884, + "theoretical_loss": 3.367231055982712, + "tokens_seen": 2512060416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002429090909090909, + "loss": 2.6752, + "theoretical_loss": 3.36722405567854, + "tokens_seen": 2512125952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024288888888888887, + "loss": 2.588, + "theoretical_loss": 3.367217055608122, + "tokens_seen": 2512191488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024286868686868688, + "loss": 2.6469, + "theoretical_loss": 3.367210055771443, + "tokens_seen": 2512257024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024284848484848484, + "loss": 2.7241, + "theoretical_loss": 3.3672030561684902, + "tokens_seen": 2512322560 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024282828282828285, + "loss": 2.6932, + "theoretical_loss": 3.3671960567992487, + "tokens_seen": 2512388096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002428080808080808, + "loss": 2.8318, + "theoretical_loss": 3.3671890576637056, + "tokens_seen": 2512453632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002427878787878788, + "loss": 2.7596, + "theoretical_loss": 3.367182058761846, + "tokens_seen": 2512519168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024276767676767677, + "loss": 2.5241, + "theoretical_loss": 3.367175060093657, + "tokens_seen": 2512584704 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024274747474747476, + "loss": 2.3803, + "theoretical_loss": 3.367168061659124, + "tokens_seen": 2512650240 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024272727272727274, + "loss": 2.4713, + "theoretical_loss": 3.3671610634582336, + "tokens_seen": 2512715776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002427070707070707, + "loss": 2.8371, + "theoretical_loss": 3.367154065490971, + "tokens_seen": 2512781312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024268686868686868, + "loss": 2.8601, + "theoretical_loss": 3.3671470677573234, + "tokens_seen": 2512846848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024266666666666667, + "loss": 2.6499, + "theoretical_loss": 3.3671400702572765, + "tokens_seen": 2512912384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024264646464646465, + "loss": 2.6895, + "theoretical_loss": 3.367133072990816, + "tokens_seen": 2512977920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024262626262626263, + "loss": 2.7791, + "theoretical_loss": 3.3671260759579287, + "tokens_seen": 2513043456 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024260606060606062, + "loss": 2.6901, + "theoretical_loss": 3.3671190791586003, + "tokens_seen": 2513108992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002425858585858586, + "loss": 2.508, + "theoretical_loss": 3.3671120825928167, + "tokens_seen": 2513174528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024256565656565659, + "loss": 2.6241, + "theoretical_loss": 3.367105086260565, + "tokens_seen": 2513240064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024254545454545454, + "loss": 2.6216, + "theoretical_loss": 3.36709809016183, + "tokens_seen": 2513305600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024252525252525253, + "loss": 2.6338, + "theoretical_loss": 3.367091094296599, + "tokens_seen": 2513371136 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002425050505050505, + "loss": 2.5701, + "theoretical_loss": 3.367084098664857, + "tokens_seen": 2513436672 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424848484848485, + "loss": 2.6333, + "theoretical_loss": 3.367077103266591, + "tokens_seen": 2513502208 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1413871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.634793519973755, + "objective/train/theoretical_loss": 3.367070108101786, + "objective/train/tokens_used": 872426976, + "theoretical_loss": 3.367070108101786, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024246464646464648, + "loss": 2.7765, + "theoretical_loss": 3.367070108101786, + "tokens_seen": 2513567744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024244444444444443, + "loss": 2.6398, + "theoretical_loss": 3.36706311317043, + "tokens_seen": 2513633280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024242424242424245, + "loss": 2.5436, + "theoretical_loss": 3.3670561184725076, + "tokens_seen": 2513698816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002424040404040404, + "loss": 2.7026, + "theoretical_loss": 3.3670491240080054, + "tokens_seen": 2513764352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024238383838383841, + "loss": 2.5182, + "theoretical_loss": 3.3670421297769093, + "tokens_seen": 2513829888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024236363636363637, + "loss": 2.7835, + "theoretical_loss": 3.367035135779206, + "tokens_seen": 2513895424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024234343434343433, + "loss": 2.6859, + "theoretical_loss": 3.367028142014881, + "tokens_seen": 2513960960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024232323232323234, + "loss": 2.5767, + "theoretical_loss": 3.3670211484839205, + "tokens_seen": 2514026496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002423030303030303, + "loss": 2.6219, + "theoretical_loss": 3.367014155186311, + "tokens_seen": 2514092032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422828282828283, + "loss": 2.6758, + "theoretical_loss": 3.3670071621220385, + "tokens_seen": 2514157568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024226262626262626, + "loss": 2.5672, + "theoretical_loss": 3.367000169291089, + "tokens_seen": 2514223104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024224242424242425, + "loss": 2.7907, + "theoretical_loss": 3.366993176693449, + "tokens_seen": 2514288640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024222222222222223, + "loss": 2.7389, + "theoretical_loss": 3.3669861843291042, + "tokens_seen": 2514354176 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002422020202020202, + "loss": 2.549, + "theoretical_loss": 3.3669791921980408, + "tokens_seen": 2514419712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002421818181818182, + "loss": 2.59, + "theoretical_loss": 3.366972200300245, + "tokens_seen": 2514485248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024216161616161616, + "loss": 2.6229, + "theoretical_loss": 3.366965208635703, + "tokens_seen": 2514550784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024214141414141414, + "loss": 2.6735, + "theoretical_loss": 3.3669582172044006, + "tokens_seen": 2514616320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024212121212121212, + "loss": 2.7142, + "theoretical_loss": 3.3669512260063246, + "tokens_seen": 2514681856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002421010101010101, + "loss": 2.5451, + "theoretical_loss": 3.3669442350414602, + "tokens_seen": 2514747392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002420808080808081, + "loss": 2.2962, + "theoretical_loss": 3.3669372443097947, + "tokens_seen": 2514812928 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024206060606060608, + "loss": 2.4728, + "theoretical_loss": 3.3669302538113137, + "tokens_seen": 2514878464 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024204040404040403, + "loss": 2.6061, + "theoretical_loss": 3.366923263546003, + "tokens_seen": 2514944000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024202020202020202, + "loss": 2.6168, + "theoretical_loss": 3.366916273513849, + "tokens_seen": 2515009536 + }, + { + "epoch": 0.52, + "learning_rate": 0.000242, + "loss": 2.5068, + "theoretical_loss": 3.366909283714838, + "tokens_seen": 2515075072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024197979797979798, + "loss": 2.5711, + "theoretical_loss": 3.3669022941489564, + "tokens_seen": 2515140608 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1414570, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8205106258392334, + "objective/train/theoretical_loss": 3.3668953048161896, + "objective/train/tokens_used": 874065376, + "theoretical_loss": 3.3668953048161896, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024195959595959597, + "loss": 2.6816, + "theoretical_loss": 3.3668953048161896, + "tokens_seen": 2515206144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024193939393939393, + "loss": 2.7681, + "theoretical_loss": 3.3668883157165244, + "tokens_seen": 2515271680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024191919191919194, + "loss": 2.7257, + "theoretical_loss": 3.3668813268499465, + "tokens_seen": 2515337216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002418989898989899, + "loss": 2.8008, + "theoretical_loss": 3.366874338216442, + "tokens_seen": 2515402752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002418787878787879, + "loss": 2.6658, + "theoretical_loss": 3.3668673498159976, + "tokens_seen": 2515468288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024185858585858586, + "loss": 2.7461, + "theoretical_loss": 3.3668603616485995, + "tokens_seen": 2515533824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024183838383838382, + "loss": 2.6486, + "theoretical_loss": 3.366853373714233, + "tokens_seen": 2515599360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024181818181818183, + "loss": 2.7408, + "theoretical_loss": 3.366846386012885, + "tokens_seen": 2515664896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024179797979797979, + "loss": 2.5583, + "theoretical_loss": 3.3668393985445415, + "tokens_seen": 2515730432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002417777777777778, + "loss": 2.5938, + "theoretical_loss": 3.3668324113091885, + "tokens_seen": 2515795968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024175757575757575, + "loss": 2.7101, + "theoretical_loss": 3.3668254243068123, + "tokens_seen": 2515861504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024173737373737376, + "loss": 2.7944, + "theoretical_loss": 3.366818437537399, + "tokens_seen": 2515927040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024171717171717172, + "loss": 2.6779, + "theoretical_loss": 3.366811451000935, + "tokens_seen": 2515992576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416969696969697, + "loss": 2.6068, + "theoretical_loss": 3.366804464697406, + "tokens_seen": 2516058112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002416767676767677, + "loss": 2.5382, + "theoretical_loss": 3.3667974786267987, + "tokens_seen": 2516123648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024165656565656565, + "loss": 2.5611, + "theoretical_loss": 3.366790492789099, + "tokens_seen": 2516189184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024163636363636366, + "loss": 2.5758, + "theoretical_loss": 3.3667835071842926, + "tokens_seen": 2516254720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024161616161616161, + "loss": 2.4678, + "theoretical_loss": 3.3667765218123664, + "tokens_seen": 2516320256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002415959595959596, + "loss": 2.4954, + "theoretical_loss": 3.3667695366733064, + "tokens_seen": 2516385792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024157575757575758, + "loss": 2.5048, + "theoretical_loss": 3.3667625517670987, + "tokens_seen": 2516451328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024155555555555557, + "loss": 2.6903, + "theoretical_loss": 3.3667555670937297, + "tokens_seen": 2516516864 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024153535353535355, + "loss": 2.4873, + "theoretical_loss": 3.3667485826531847, + "tokens_seen": 2516582400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024151515151515153, + "loss": 2.7046, + "theoretical_loss": 3.3667415984454507, + "tokens_seen": 2516647936 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002414949494949495, + "loss": 2.7047, + "theoretical_loss": 3.3667346144705137, + "tokens_seen": 2516713472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024147474747474748, + "loss": 2.6794, + "theoretical_loss": 3.3667276307283602, + "tokens_seen": 2516779008 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1415934, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.781501293182373, + "objective/train/theoretical_loss": 3.3667206472189757, + "objective/train/tokens_used": 875703776, + "theoretical_loss": 3.3667206472189757, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024145454545454546, + "loss": 2.5803, + "theoretical_loss": 3.3667206472189757, + "tokens_seen": 2516844544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024143434343434344, + "loss": 2.2471, + "theoretical_loss": 3.366713663942347, + "tokens_seen": 2516910080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024141414141414143, + "loss": 2.6092, + "theoretical_loss": 3.3667066808984596, + "tokens_seen": 2516975616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024139393939393938, + "loss": 2.7092, + "theoretical_loss": 3.3666996980873005, + "tokens_seen": 2517041152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002413737373737374, + "loss": 2.4903, + "theoretical_loss": 3.3666927155088553, + "tokens_seen": 2517106688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024135353535353535, + "loss": 2.6981, + "theoretical_loss": 3.36668573316311, + "tokens_seen": 2517172224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024133333333333334, + "loss": 2.4496, + "theoretical_loss": 3.3666787510500518, + "tokens_seen": 2517237760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024131313131313132, + "loss": 2.6891, + "theoretical_loss": 3.366671769169666, + "tokens_seen": 2517303296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024129292929292928, + "loss": 2.3337, + "theoretical_loss": 3.366664787521939, + "tokens_seen": 2517368832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002412727272727273, + "loss": 2.8988, + "theoretical_loss": 3.366657806106857, + "tokens_seen": 2517434368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024125252525252524, + "loss": 2.6698, + "theoretical_loss": 3.3666508249244056, + "tokens_seen": 2517499904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024123232323232326, + "loss": 2.5677, + "theoretical_loss": 3.3666438439745723, + "tokens_seen": 2517565440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002412121212121212, + "loss": 2.5842, + "theoretical_loss": 3.3666368632573427, + "tokens_seen": 2517630976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002411919191919192, + "loss": 2.5688, + "theoretical_loss": 3.366629882772702, + "tokens_seen": 2517696512 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024117171717171718, + "loss": 2.6589, + "theoretical_loss": 3.3666229025206382, + "tokens_seen": 2517762048 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024115151515151514, + "loss": 2.7193, + "theoretical_loss": 3.3666159225011363, + "tokens_seen": 2517827584 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024113131313131315, + "loss": 2.6974, + "theoretical_loss": 3.3666089427141825, + "tokens_seen": 2517893120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002411111111111111, + "loss": 2.5005, + "theoretical_loss": 3.3666019631597637, + "tokens_seen": 2517958656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002410909090909091, + "loss": 2.6693, + "theoretical_loss": 3.3665949838378655, + "tokens_seen": 2518024192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024107070707070707, + "loss": 2.6106, + "theoretical_loss": 3.3665880047484746, + "tokens_seen": 2518089728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024105050505050506, + "loss": 2.6373, + "theoretical_loss": 3.3665810258915765, + "tokens_seen": 2518155264 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024103030303030304, + "loss": 2.4312, + "theoretical_loss": 3.3665740472671577, + "tokens_seen": 2518220800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024101010101010102, + "loss": 2.8967, + "theoretical_loss": 3.366567068875205, + "tokens_seen": 2518286336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024098989898989898, + "loss": 2.6558, + "theoretical_loss": 3.366560090715703, + "tokens_seen": 2518351872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024096969696969697, + "loss": 2.5246, + "theoretical_loss": 3.3665531127886403, + "tokens_seen": 2518417408 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1416375, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85703182220459, + "objective/train/theoretical_loss": 3.3665461350940014, + "objective/train/tokens_used": 877342176, + "theoretical_loss": 3.3665461350940014, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024094949494949495, + "loss": 2.5688, + "theoretical_loss": 3.3665461350940014, + "tokens_seen": 2518482944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024092929292929293, + "loss": 2.8735, + "theoretical_loss": 3.3665391576317725, + "tokens_seen": 2518548480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024090909090909092, + "loss": 2.6738, + "theoretical_loss": 3.366532180401941, + "tokens_seen": 2518614016 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408888888888889, + "loss": 2.8929, + "theoretical_loss": 3.3665252034044917, + "tokens_seen": 2518679552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024086868686868689, + "loss": 2.5308, + "theoretical_loss": 3.3665182266394114, + "tokens_seen": 2518745088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024084848484848484, + "loss": 2.6746, + "theoretical_loss": 3.366511250106687, + "tokens_seen": 2518810624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024082828282828285, + "loss": 2.5698, + "theoretical_loss": 3.366504273806304, + "tokens_seen": 2518876160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002408080808080808, + "loss": 2.6197, + "theoretical_loss": 3.366497297738248, + "tokens_seen": 2518941696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002407878787878788, + "loss": 2.6218, + "theoretical_loss": 3.3664903219025066, + "tokens_seen": 2519007232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024076767676767678, + "loss": 2.6236, + "theoretical_loss": 3.366483346299065, + "tokens_seen": 2519072768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024074747474747474, + "loss": 2.5063, + "theoretical_loss": 3.36647637092791, + "tokens_seen": 2519138304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024072727272727275, + "loss": 2.8555, + "theoretical_loss": 3.3664693957890277, + "tokens_seen": 2519203840 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002407070707070707, + "loss": 2.4973, + "theoretical_loss": 3.366462420882404, + "tokens_seen": 2519269376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024068686868686871, + "loss": 2.8083, + "theoretical_loss": 3.3664554462080254, + "tokens_seen": 2519334912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024066666666666667, + "loss": 2.7248, + "theoretical_loss": 3.366448471765878, + "tokens_seen": 2519400448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024064646464646463, + "loss": 2.4923, + "theoretical_loss": 3.3664414975559485, + "tokens_seen": 2519465984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024062626262626264, + "loss": 2.6096, + "theoretical_loss": 3.366434523578222, + "tokens_seen": 2519531520 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002406060606060606, + "loss": 2.6612, + "theoretical_loss": 3.366427549832686, + "tokens_seen": 2519597056 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002405858585858586, + "loss": 2.5096, + "theoretical_loss": 3.3664205763193262, + "tokens_seen": 2519662592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024056565656565656, + "loss": 2.6906, + "theoretical_loss": 3.3664136030381284, + "tokens_seen": 2519728128 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024054545454545455, + "loss": 2.7647, + "theoretical_loss": 3.3664066299890796, + "tokens_seen": 2519793664 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024052525252525253, + "loss": 2.7905, + "theoretical_loss": 3.366399657172166, + "tokens_seen": 2519859200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024050505050505052, + "loss": 2.717, + "theoretical_loss": 3.366392684587373, + "tokens_seen": 2519924736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002404848484848485, + "loss": 2.6441, + "theoretical_loss": 3.3663857122346874, + "tokens_seen": 2519990272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024046464646464646, + "loss": 2.4576, + "theoretical_loss": 3.3663787401140954, + "tokens_seen": 2520055808 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1417108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2882065773010254, + "objective/train/theoretical_loss": 3.3663717682255836, + "objective/train/tokens_used": 878980576, + "theoretical_loss": 3.3663717682255836, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024044444444444444, + "loss": 2.4815, + "theoretical_loss": 3.3663717682255836, + "tokens_seen": 2520121344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024042424242424242, + "loss": 2.5643, + "theoretical_loss": 3.3663647965691377, + "tokens_seen": 2520186880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002404040404040404, + "loss": 2.5623, + "theoretical_loss": 3.3663578251447435, + "tokens_seen": 2520252416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002403838383838384, + "loss": 2.7667, + "theoretical_loss": 3.3663508539523885, + "tokens_seen": 2520317952 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024036363636363638, + "loss": 2.6433, + "theoretical_loss": 3.3663438829920582, + "tokens_seen": 2520383488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024034343434343433, + "loss": 2.6386, + "theoretical_loss": 3.3663369122637388, + "tokens_seen": 2520449024 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024032323232323234, + "loss": 2.6179, + "theoretical_loss": 3.366329941767417, + "tokens_seen": 2520514560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002403030303030303, + "loss": 2.6697, + "theoretical_loss": 3.3663229715030787, + "tokens_seen": 2520580096 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024028282828282828, + "loss": 2.5472, + "theoretical_loss": 3.3663160014707096, + "tokens_seen": 2520645632 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024026262626262627, + "loss": 2.53, + "theoretical_loss": 3.3663090316702973, + "tokens_seen": 2520711168 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024024242424242423, + "loss": 2.5251, + "theoretical_loss": 3.366302062101827, + "tokens_seen": 2520776704 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024022222222222224, + "loss": 2.5421, + "theoretical_loss": 3.366295092765285, + "tokens_seen": 2520842240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002402020202020202, + "loss": 2.6601, + "theoretical_loss": 3.366288123660658, + "tokens_seen": 2520907776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002401818181818182, + "loss": 2.7461, + "theoretical_loss": 3.3662811547879317, + "tokens_seen": 2520973312 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024016161616161616, + "loss": 2.7013, + "theoretical_loss": 3.366274186147093, + "tokens_seen": 2521038848 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024014141414141415, + "loss": 2.7, + "theoretical_loss": 3.366267217738128, + "tokens_seen": 2521104384 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024012121212121213, + "loss": 2.4968, + "theoretical_loss": 3.366260249561023, + "tokens_seen": 2521169920 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024010101010101009, + "loss": 2.7496, + "theoretical_loss": 3.3662532816157635, + "tokens_seen": 2521235456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0002400808080808081, + "loss": 2.617, + "theoretical_loss": 3.3662463139023364, + "tokens_seen": 2521300992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024006060606060605, + "loss": 2.6511, + "theoretical_loss": 3.3662393464207283, + "tokens_seen": 2521366528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024004040404040407, + "loss": 2.6008, + "theoretical_loss": 3.366232379170925, + "tokens_seen": 2521432064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024002020202020202, + "loss": 2.6763, + "theoretical_loss": 3.3662254121529123, + "tokens_seen": 2521497600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00024, + "loss": 2.6885, + "theoretical_loss": 3.3662184453666777, + "tokens_seen": 2521563136 + }, + { + "epoch": 0.52, + "learning_rate": 0.000239979797979798, + "loss": 2.6791, + "theoretical_loss": 3.366211478812206, + "tokens_seen": 2521628672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00023995959595959597, + "loss": 2.7185, + "theoretical_loss": 3.3662045124894844, + "tokens_seen": 2521694208 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 1417967, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.030874490737915, + "objective/train/theoretical_loss": 3.3661975463984994, + "objective/train/tokens_used": 880618976, + "theoretical_loss": 3.3661975463984994, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023993939393939396, + "loss": 2.6709, + "theoretical_loss": 3.3661975463984994, + "tokens_seen": 2521759744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023991919191919191, + "loss": 2.638, + "theoretical_loss": 3.3661905805392367, + "tokens_seen": 2521825280 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002398989898989899, + "loss": 2.7067, + "theoretical_loss": 3.3661836149116824, + "tokens_seen": 2521890816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023987878787878788, + "loss": 2.8304, + "theoretical_loss": 3.3661766495158236, + "tokens_seen": 2521956352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023985858585858587, + "loss": 2.407, + "theoretical_loss": 3.3661696843516458, + "tokens_seen": 2522021888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023983838383838385, + "loss": 2.5359, + "theoretical_loss": 3.3661627194191355, + "tokens_seen": 2522087424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023981818181818183, + "loss": 2.8166, + "theoretical_loss": 3.366155754718279, + "tokens_seen": 2522152960 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002397979797979798, + "loss": 2.6281, + "theoretical_loss": 3.3661487902490626, + "tokens_seen": 2522218496 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002397777777777778, + "loss": 2.5531, + "theoretical_loss": 3.3661418260114724, + "tokens_seen": 2522284032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023975757575757576, + "loss": 2.4091, + "theoretical_loss": 3.366134862005495, + "tokens_seen": 2522349568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023973737373737374, + "loss": 2.4673, + "theoretical_loss": 3.3661278982311167, + "tokens_seen": 2522415104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023971717171717173, + "loss": 2.5453, + "theoretical_loss": 3.366120934688323, + "tokens_seen": 2522480640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023969696969696968, + "loss": 2.673, + "theoretical_loss": 3.3661139713771018, + "tokens_seen": 2522546176 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002396767676767677, + "loss": 2.4208, + "theoretical_loss": 3.366107008297438, + "tokens_seen": 2522611712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023965656565656565, + "loss": 2.7718, + "theoretical_loss": 3.3661000454493175, + "tokens_seen": 2522677248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023963636363636366, + "loss": 2.677, + "theoretical_loss": 3.366093082832728, + "tokens_seen": 2522742784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023961616161616162, + "loss": 2.74, + "theoretical_loss": 3.3660861204476555, + "tokens_seen": 2522808320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023959595959595958, + "loss": 2.5544, + "theoretical_loss": 3.3660791582940854, + "tokens_seen": 2522873856 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002395757575757576, + "loss": 2.393, + "theoretical_loss": 3.3660721963720044, + "tokens_seen": 2522939392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023955555555555554, + "loss": 2.5082, + "theoretical_loss": 3.366065234681399, + "tokens_seen": 2523004928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023953535353535356, + "loss": 2.439, + "theoretical_loss": 3.3660582732222553, + "tokens_seen": 2523070464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002395151515151515, + "loss": 2.5676, + "theoretical_loss": 3.36605131199456, + "tokens_seen": 2523136000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394949494949495, + "loss": 2.7415, + "theoretical_loss": 3.366044350998299, + "tokens_seen": 2523201536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023947474747474748, + "loss": 2.3254, + "theoretical_loss": 3.3660373902334584, + "tokens_seen": 2523267072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023945454545454546, + "loss": 2.4885, + "theoretical_loss": 3.366030429700025, + "tokens_seen": 2523332608 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1418616, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3458359241485596, + "objective/train/theoretical_loss": 3.3660234693979847, + "objective/train/tokens_used": 882257376, + "theoretical_loss": 3.3660234693979847, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023943434343434345, + "loss": 2.4373, + "theoretical_loss": 3.3660234693979847, + "tokens_seen": 2523398144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002394141414141414, + "loss": 2.5693, + "theoretical_loss": 3.366016509327324, + "tokens_seen": 2523463680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002393939393939394, + "loss": 2.4079, + "theoretical_loss": 3.3660095494880293, + "tokens_seen": 2523529216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023937373737373737, + "loss": 2.4754, + "theoretical_loss": 3.366002589880087, + "tokens_seen": 2523594752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023935353535353536, + "loss": 2.7231, + "theoretical_loss": 3.3659956305034826, + "tokens_seen": 2523660288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023933333333333334, + "loss": 2.6782, + "theoretical_loss": 3.365988671358203, + "tokens_seen": 2523725824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023931313131313133, + "loss": 2.5716, + "theoretical_loss": 3.365981712444235, + "tokens_seen": 2523791360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023929292929292928, + "loss": 2.6633, + "theoretical_loss": 3.365974753761564, + "tokens_seen": 2523856896 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002392727272727273, + "loss": 2.7855, + "theoretical_loss": 3.3659677953101763, + "tokens_seen": 2523922432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023925252525252525, + "loss": 2.5644, + "theoretical_loss": 3.3659608370900593, + "tokens_seen": 2523987968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023923232323232323, + "loss": 2.8526, + "theoretical_loss": 3.3659538791011983, + "tokens_seen": 2524053504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023921212121212122, + "loss": 2.6162, + "theoretical_loss": 3.3659469213435798, + "tokens_seen": 2524119040 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002391919191919192, + "loss": 2.5469, + "theoretical_loss": 3.3659399638171905, + "tokens_seen": 2524184576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023917171717171719, + "loss": 2.4154, + "theoretical_loss": 3.365933006522016, + "tokens_seen": 2524250112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023915151515151514, + "loss": 2.7288, + "theoretical_loss": 3.3659260494580434, + "tokens_seen": 2524315648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023913131313131315, + "loss": 2.4632, + "theoretical_loss": 3.3659190926252585, + "tokens_seen": 2524381184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002391111111111111, + "loss": 2.599, + "theoretical_loss": 3.365912136023648, + "tokens_seen": 2524446720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023909090909090912, + "loss": 2.3826, + "theoretical_loss": 3.365905179653198, + "tokens_seen": 2524512256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023907070707070708, + "loss": 2.5777, + "theoretical_loss": 3.365898223513894, + "tokens_seen": 2524577792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023905050505050504, + "loss": 2.5616, + "theoretical_loss": 3.3658912676057238, + "tokens_seen": 2524643328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023903030303030305, + "loss": 2.6783, + "theoretical_loss": 3.3658843119286734, + "tokens_seen": 2524708864 + }, + { + "epoch": 0.53, + "learning_rate": 0.000239010101010101, + "loss": 2.5858, + "theoretical_loss": 3.365877356482728, + "tokens_seen": 2524774400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023898989898989901, + "loss": 2.3414, + "theoretical_loss": 3.365870401267875, + "tokens_seen": 2524839936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023896969696969697, + "loss": 2.3953, + "theoretical_loss": 3.365863446284101, + "tokens_seen": 2524905472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023894949494949496, + "loss": 2.7412, + "theoretical_loss": 3.365856491531391, + "tokens_seen": 2524971008 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1419908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6612930297851562, + "objective/train/theoretical_loss": 3.3658495370097326, + "objective/train/tokens_used": 883895776, + "theoretical_loss": 3.3658495370097326, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023892929292929294, + "loss": 2.5871, + "theoretical_loss": 3.3658495370097326, + "tokens_seen": 2525036544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002389090909090909, + "loss": 2.5269, + "theoretical_loss": 3.365842582719111, + "tokens_seen": 2525102080 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002388888888888889, + "loss": 2.6154, + "theoretical_loss": 3.3658356286595135, + "tokens_seen": 2525167616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023886868686868686, + "loss": 2.8319, + "theoretical_loss": 3.365828674830926, + "tokens_seen": 2525233152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023884848484848485, + "loss": 2.6709, + "theoretical_loss": 3.365821721233335, + "tokens_seen": 2525298688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023882828282828283, + "loss": 2.7289, + "theoretical_loss": 3.3658147678667265, + "tokens_seen": 2525364224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023880808080808082, + "loss": 2.9957, + "theoretical_loss": 3.365807814731087, + "tokens_seen": 2525429760 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002387878787878788, + "loss": 2.5837, + "theoretical_loss": 3.3658008618264033, + "tokens_seen": 2525495296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023876767676767678, + "loss": 2.4674, + "theoretical_loss": 3.3657939091526607, + "tokens_seen": 2525560832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023874747474747474, + "loss": 2.6951, + "theoretical_loss": 3.3657869567098464, + "tokens_seen": 2525626368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023872727272727272, + "loss": 2.6582, + "theoretical_loss": 3.365780004497947, + "tokens_seen": 2525691904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002387070707070707, + "loss": 2.7373, + "theoretical_loss": 3.3657730525169476, + "tokens_seen": 2525757440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386868686868687, + "loss": 2.5161, + "theoretical_loss": 3.3657661007668356, + "tokens_seen": 2525822976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023866666666666668, + "loss": 2.5194, + "theoretical_loss": 3.365759149247597, + "tokens_seen": 2525888512 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023864646464646463, + "loss": 2.453, + "theoretical_loss": 3.3657521979592184, + "tokens_seen": 2525954048 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023862626262626264, + "loss": 2.8767, + "theoretical_loss": 3.3657452469016853, + "tokens_seen": 2526019584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002386060606060606, + "loss": 2.5562, + "theoretical_loss": 3.3657382960749853, + "tokens_seen": 2526085120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002385858585858586, + "loss": 2.7013, + "theoretical_loss": 3.365731345479104, + "tokens_seen": 2526150656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023856565656565657, + "loss": 2.5701, + "theoretical_loss": 3.3657243951140274, + "tokens_seen": 2526216192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023854545454545453, + "loss": 2.7516, + "theoretical_loss": 3.365717444979743, + "tokens_seen": 2526281728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023852525252525254, + "loss": 2.6064, + "theoretical_loss": 3.3657104950762355, + "tokens_seen": 2526347264 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002385050505050505, + "loss": 2.7554, + "theoretical_loss": 3.365703545403493, + "tokens_seen": 2526412800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002384848484848485, + "loss": 2.5722, + "theoretical_loss": 3.3656965959615004, + "tokens_seen": 2526478336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023846464646464646, + "loss": 2.575, + "theoretical_loss": 3.365689646750245, + "tokens_seen": 2526543872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023844444444444445, + "loss": 2.4308, + "theoretical_loss": 3.365682697769713, + "tokens_seen": 2526609408 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1420528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.277600049972534, + "objective/train/theoretical_loss": 3.3656757490198905, + "objective/train/tokens_used": 885534176, + "theoretical_loss": 3.3656757490198905, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023842424242424243, + "loss": 2.6528, + "theoretical_loss": 3.3656757490198905, + "tokens_seen": 2526674944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023840404040404041, + "loss": 2.6926, + "theoretical_loss": 3.365668800500764, + "tokens_seen": 2526740480 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002383838383838384, + "loss": 2.6503, + "theoretical_loss": 3.3656618522123196, + "tokens_seen": 2526806016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023836363636363635, + "loss": 2.7139, + "theoretical_loss": 3.3656549041545443, + "tokens_seen": 2526871552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023834343434343437, + "loss": 2.7394, + "theoretical_loss": 3.365647956327424, + "tokens_seen": 2526937088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023832323232323232, + "loss": 2.614, + "theoretical_loss": 3.3656410087309445, + "tokens_seen": 2527002624 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002383030303030303, + "loss": 2.4368, + "theoretical_loss": 3.3656340613650935, + "tokens_seen": 2527068160 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382828282828283, + "loss": 2.5084, + "theoretical_loss": 3.365627114229856, + "tokens_seen": 2527133696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023826262626262627, + "loss": 2.5868, + "theoretical_loss": 3.3656201673252193, + "tokens_seen": 2527199232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023824242424242426, + "loss": 2.6405, + "theoretical_loss": 3.3656132206511695, + "tokens_seen": 2527264768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023822222222222224, + "loss": 2.7056, + "theoretical_loss": 3.3656062742076926, + "tokens_seen": 2527330304 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002382020202020202, + "loss": 2.546, + "theoretical_loss": 3.365599327994776, + "tokens_seen": 2527395840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023818181818181818, + "loss": 2.7136, + "theoretical_loss": 3.3655923820124047, + "tokens_seen": 2527461376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023816161616161617, + "loss": 2.6564, + "theoretical_loss": 3.3655854362605657, + "tokens_seen": 2527526912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023814141414141415, + "loss": 2.5477, + "theoretical_loss": 3.365578490739246, + "tokens_seen": 2527592448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023812121212121214, + "loss": 2.4934, + "theoretical_loss": 3.365571545448431, + "tokens_seen": 2527657984 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002381010101010101, + "loss": 2.6381, + "theoretical_loss": 3.3655646003881072, + "tokens_seen": 2527723520 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002380808080808081, + "loss": 2.5954, + "theoretical_loss": 3.365557655558262, + "tokens_seen": 2527789056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023806060606060606, + "loss": 2.5988, + "theoretical_loss": 3.36555071095888, + "tokens_seen": 2527854592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023804040404040404, + "loss": 2.53, + "theoretical_loss": 3.3655437665899495, + "tokens_seen": 2527920128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023802020202020203, + "loss": 2.6605, + "theoretical_loss": 3.365536822451455, + "tokens_seen": 2527985664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023799999999999998, + "loss": 2.591, + "theoretical_loss": 3.3655298785433847, + "tokens_seen": 2528051200 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237979797979798, + "loss": 2.6748, + "theoretical_loss": 3.365522934865724, + "tokens_seen": 2528116736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023795959595959595, + "loss": 2.6483, + "theoretical_loss": 3.3655159914184587, + "tokens_seen": 2528182272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023793939393939396, + "loss": 2.676, + "theoretical_loss": 3.3655090482015764, + "tokens_seen": 2528247808 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1420951, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8490426540374756, + "objective/train/theoretical_loss": 3.365502105215063, + "objective/train/tokens_used": 887172576, + "theoretical_loss": 3.365502105215063, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023791919191919192, + "loss": 2.8995, + "theoretical_loss": 3.365502105215063, + "tokens_seen": 2528313344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378989898989899, + "loss": 2.5297, + "theoretical_loss": 3.3654951624589047, + "tokens_seen": 2528378880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378787878787879, + "loss": 2.6128, + "theoretical_loss": 3.365488219933088, + "tokens_seen": 2528444416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023785858585858585, + "loss": 2.4866, + "theoretical_loss": 3.3654812776376, + "tokens_seen": 2528509952 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023783838383838386, + "loss": 2.6399, + "theoretical_loss": 3.3654743355724257, + "tokens_seen": 2528575488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002378181818181818, + "loss": 2.6257, + "theoretical_loss": 3.365467393737552, + "tokens_seen": 2528641024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002377979797979798, + "loss": 2.7001, + "theoretical_loss": 3.3654604521329663, + "tokens_seen": 2528706560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023777777777777778, + "loss": 2.7071, + "theoretical_loss": 3.3654535107586536, + "tokens_seen": 2528772096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023775757575757577, + "loss": 2.5903, + "theoretical_loss": 3.365446569614601, + "tokens_seen": 2528837632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023773737373737375, + "loss": 2.7677, + "theoretical_loss": 3.365439628700795, + "tokens_seen": 2528903168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023771717171717173, + "loss": 2.5231, + "theoretical_loss": 3.365432688017221, + "tokens_seen": 2528968704 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002376969696969697, + "loss": 2.4897, + "theoretical_loss": 3.3654257475638674, + "tokens_seen": 2529034240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023767676767676767, + "loss": 2.7747, + "theoretical_loss": 3.3654188073407187, + "tokens_seen": 2529099776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023765656565656566, + "loss": 2.5088, + "theoretical_loss": 3.365411867347762, + "tokens_seen": 2529165312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023763636363636364, + "loss": 2.6682, + "theoretical_loss": 3.3654049275849838, + "tokens_seen": 2529230848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023761616161616163, + "loss": 2.5512, + "theoretical_loss": 3.36539798805237, + "tokens_seen": 2529296384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023759595959595958, + "loss": 2.6828, + "theoretical_loss": 3.365391048749908, + "tokens_seen": 2529361920 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002375757575757576, + "loss": 2.6982, + "theoretical_loss": 3.365384109677583, + "tokens_seen": 2529427456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023755555555555555, + "loss": 2.5338, + "theoretical_loss": 3.365377170835382, + "tokens_seen": 2529492992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023753535353535356, + "loss": 2.5648, + "theoretical_loss": 3.3653702322232917, + "tokens_seen": 2529558528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023751515151515152, + "loss": 2.593, + "theoretical_loss": 3.3653632938412983, + "tokens_seen": 2529624064 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002374949494949495, + "loss": 2.6367, + "theoretical_loss": 3.3653563556893875, + "tokens_seen": 2529689600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023747474747474749, + "loss": 2.4992, + "theoretical_loss": 3.365349417767547, + "tokens_seen": 2529755136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023745454545454544, + "loss": 2.6453, + "theoretical_loss": 3.3653424800757623, + "tokens_seen": 2529820672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023743434343434345, + "loss": 2.5873, + "theoretical_loss": 3.3653355426140195, + "tokens_seen": 2529886208 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1422170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6018807888031006, + "objective/train/theoretical_loss": 3.3653286053823064, + "objective/train/tokens_used": 888810976, + "theoretical_loss": 3.3653286053823064, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002374141414141414, + "loss": 2.5578, + "theoretical_loss": 3.3653286053823064, + "tokens_seen": 2529951744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023739393939393942, + "loss": 2.384, + "theoretical_loss": 3.365321668380608, + "tokens_seen": 2530017280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023737373737373738, + "loss": 2.5151, + "theoretical_loss": 3.365314731608912, + "tokens_seen": 2530082816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023735353535353534, + "loss": 2.6501, + "theoretical_loss": 3.3653077950672032, + "tokens_seen": 2530148352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023733333333333335, + "loss": 2.6451, + "theoretical_loss": 3.3653008587554694, + "tokens_seen": 2530213888 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002373131313131313, + "loss": 2.5615, + "theoretical_loss": 3.3652939226736964, + "tokens_seen": 2530279424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023729292929292931, + "loss": 2.4577, + "theoretical_loss": 3.3652869868218707, + "tokens_seen": 2530344960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023727272727272727, + "loss": 2.6705, + "theoretical_loss": 3.3652800511999788, + "tokens_seen": 2530410496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023725252525252526, + "loss": 2.7206, + "theoretical_loss": 3.3652731158080074, + "tokens_seen": 2530476032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023723232323232324, + "loss": 2.6829, + "theoretical_loss": 3.3652661806459423, + "tokens_seen": 2530541568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023721212121212122, + "loss": 2.4771, + "theoretical_loss": 3.36525924571377, + "tokens_seen": 2530607104 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002371919191919192, + "loss": 2.4474, + "theoretical_loss": 3.365252311011478, + "tokens_seen": 2530672640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023717171717171716, + "loss": 2.4941, + "theoretical_loss": 3.3652453765390513, + "tokens_seen": 2530738176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023715151515151515, + "loss": 2.4891, + "theoretical_loss": 3.365238442296477, + "tokens_seen": 2530803712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023713131313131313, + "loss": 2.7293, + "theoretical_loss": 3.3652315082837414, + "tokens_seen": 2530869248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023711111111111112, + "loss": 2.6175, + "theoretical_loss": 3.3652245745008313, + "tokens_seen": 2530934784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002370909090909091, + "loss": 2.384, + "theoretical_loss": 3.3652176409477326, + "tokens_seen": 2531000320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023707070707070708, + "loss": 2.6183, + "theoretical_loss": 3.3652107076244318, + "tokens_seen": 2531065856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023705050505050504, + "loss": 2.5702, + "theoretical_loss": 3.3652037745309156, + "tokens_seen": 2531131392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023703030303030305, + "loss": 2.3846, + "theoretical_loss": 3.3651968416671707, + "tokens_seen": 2531196928 + }, + { + "epoch": 0.53, + "learning_rate": 0.000237010101010101, + "loss": 2.4644, + "theoretical_loss": 3.3651899090331825, + "tokens_seen": 2531262464 + }, + { + "epoch": 0.53, + "learning_rate": 0.000236989898989899, + "loss": 2.7989, + "theoretical_loss": 3.3651829766289385, + "tokens_seen": 2531328000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023696969696969698, + "loss": 2.2047, + "theoretical_loss": 3.365176044454425, + "tokens_seen": 2531393536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023694949494949493, + "loss": 2.5696, + "theoretical_loss": 3.3651691125096277, + "tokens_seen": 2531459072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023692929292929294, + "loss": 2.4439, + "theoretical_loss": 3.3651621807945338, + "tokens_seen": 2531524608 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1422669, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8089168071746826, + "objective/train/theoretical_loss": 3.3651552493091295, + "objective/train/tokens_used": 890449376, + "theoretical_loss": 3.3651552493091295, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002369090909090909, + "loss": 2.477, + "theoretical_loss": 3.3651552493091295, + "tokens_seen": 2531590144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368888888888889, + "loss": 2.5073, + "theoretical_loss": 3.365148318053401, + "tokens_seen": 2531655680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023686868686868687, + "loss": 2.5045, + "theoretical_loss": 3.365141387027335, + "tokens_seen": 2531721216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023684848484848485, + "loss": 2.584, + "theoretical_loss": 3.365134456230918, + "tokens_seen": 2531786752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023682828282828284, + "loss": 2.5239, + "theoretical_loss": 3.365127525664136, + "tokens_seen": 2531852288 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002368080808080808, + "loss": 2.5525, + "theoretical_loss": 3.365120595326976, + "tokens_seen": 2531917824 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002367878787878788, + "loss": 2.2901, + "theoretical_loss": 3.365113665219424, + "tokens_seen": 2531983360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023676767676767676, + "loss": 2.8202, + "theoretical_loss": 3.365106735341467, + "tokens_seen": 2532048896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023674747474747475, + "loss": 2.5717, + "theoretical_loss": 3.365099805693091, + "tokens_seen": 2532114432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023672727272727273, + "loss": 2.66, + "theoretical_loss": 3.365092876274283, + "tokens_seen": 2532179968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023670707070707071, + "loss": 2.7933, + "theoretical_loss": 3.3650859470850287, + "tokens_seen": 2532245504 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002366868686868687, + "loss": 2.5435, + "theoretical_loss": 3.3650790181253147, + "tokens_seen": 2532311040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023666666666666668, + "loss": 2.988, + "theoretical_loss": 3.3650720893951283, + "tokens_seen": 2532376576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023664646464646464, + "loss": 2.4752, + "theoretical_loss": 3.3650651608944546, + "tokens_seen": 2532442112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023662626262626262, + "loss": 2.2891, + "theoretical_loss": 3.365058232623281, + "tokens_seen": 2532507648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002366060606060606, + "loss": 2.8176, + "theoretical_loss": 3.365051304581594, + "tokens_seen": 2532573184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002365858585858586, + "loss": 2.4652, + "theoretical_loss": 3.3650443767693794, + "tokens_seen": 2532638720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023656565656565657, + "loss": 2.8041, + "theoretical_loss": 3.365037449186624, + "tokens_seen": 2532704256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023654545454545456, + "loss": 2.5782, + "theoretical_loss": 3.3650305218333147, + "tokens_seen": 2532769792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023652525252525254, + "loss": 2.5682, + "theoretical_loss": 3.365023594709437, + "tokens_seen": 2532835328 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002365050505050505, + "loss": 2.79, + "theoretical_loss": 3.3650166678149787, + "tokens_seen": 2532900864 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002364848484848485, + "loss": 2.7371, + "theoretical_loss": 3.365009741149925, + "tokens_seen": 2532966400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023646464646464647, + "loss": 2.5751, + "theoretical_loss": 3.365002814714263, + "tokens_seen": 2533031936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023644444444444445, + "loss": 2.7704, + "theoretical_loss": 3.3649958885079796, + "tokens_seen": 2533097472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023642424242424244, + "loss": 2.4133, + "theoretical_loss": 3.36498896253106, + "tokens_seen": 2533163008 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1423865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5533924102783203, + "objective/train/theoretical_loss": 3.3649820367834917, + "objective/train/tokens_used": 892087776, + "theoretical_loss": 3.3649820367834917, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002364040404040404, + "loss": 2.6503, + "theoretical_loss": 3.3649820367834917, + "tokens_seen": 2533228544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002363838383838384, + "loss": 2.7309, + "theoretical_loss": 3.364975111265261, + "tokens_seen": 2533294080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023636363636363636, + "loss": 2.6446, + "theoretical_loss": 3.3649681859763536, + "tokens_seen": 2533359616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023634343434343437, + "loss": 2.6715, + "theoretical_loss": 3.3649612609167567, + "tokens_seen": 2533425152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023632323232323233, + "loss": 2.6803, + "theoretical_loss": 3.3649543360864573, + "tokens_seen": 2533490688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023630303030303028, + "loss": 2.5541, + "theoretical_loss": 3.3649474114854407, + "tokens_seen": 2533556224 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362828282828283, + "loss": 2.5429, + "theoretical_loss": 3.3649404871136945, + "tokens_seen": 2533621760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023626262626262625, + "loss": 2.607, + "theoretical_loss": 3.364933562971204, + "tokens_seen": 2533687296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023624242424242426, + "loss": 2.6695, + "theoretical_loss": 3.364926639057957, + "tokens_seen": 2533752832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023622222222222222, + "loss": 2.694, + "theoretical_loss": 3.3649197153739383, + "tokens_seen": 2533818368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002362020202020202, + "loss": 2.5918, + "theoretical_loss": 3.364912791919136, + "tokens_seen": 2533883904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361818181818182, + "loss": 2.6855, + "theoretical_loss": 3.364905868693536, + "tokens_seen": 2533949440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023616161616161617, + "loss": 2.5112, + "theoretical_loss": 3.3648989456971243, + "tokens_seen": 2534014976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023614141414141416, + "loss": 2.6073, + "theoretical_loss": 3.3648920229298884, + "tokens_seen": 2534080512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361212121212121, + "loss": 2.5411, + "theoretical_loss": 3.364885100391814, + "tokens_seen": 2534146048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002361010101010101, + "loss": 2.5685, + "theoretical_loss": 3.3648781780828876, + "tokens_seen": 2534211584 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023608080808080808, + "loss": 2.7623, + "theoretical_loss": 3.364871256003096, + "tokens_seen": 2534277120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023606060606060607, + "loss": 2.3202, + "theoretical_loss": 3.3648643341524256, + "tokens_seen": 2534342656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023604040404040405, + "loss": 2.5156, + "theoretical_loss": 3.3648574125308626, + "tokens_seen": 2534408192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023602020202020203, + "loss": 2.5395, + "theoretical_loss": 3.3648504911383936, + "tokens_seen": 2534473728 + }, + { + "epoch": 0.53, + "learning_rate": 0.000236, + "loss": 2.6383, + "theoretical_loss": 3.364843569975006, + "tokens_seen": 2534539264 + }, + { + "epoch": 0.53, + "learning_rate": 0.000235979797979798, + "loss": 2.4878, + "theoretical_loss": 3.3648366490406847, + "tokens_seen": 2534604800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023595959595959596, + "loss": 2.5951, + "theoretical_loss": 3.3648297283354176, + "tokens_seen": 2534670336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023593939393939394, + "loss": 2.6753, + "theoretical_loss": 3.3648228078591904, + "tokens_seen": 2534735872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023591919191919193, + "loss": 2.6084, + "theoretical_loss": 3.3648158876119902, + "tokens_seen": 2534801408 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1424480, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9231460094451904, + "objective/train/theoretical_loss": 3.3648089675938024, + "objective/train/tokens_used": 893726176, + "theoretical_loss": 3.3648089675938024, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023589898989898988, + "loss": 2.6245, + "theoretical_loss": 3.3648089675938024, + "tokens_seen": 2534866944 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002358787878787879, + "loss": 2.8305, + "theoretical_loss": 3.364802047804615, + "tokens_seen": 2534932480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023585858585858585, + "loss": 2.5212, + "theoretical_loss": 3.3647951282444133, + "tokens_seen": 2534998016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023583838383838386, + "loss": 2.602, + "theoretical_loss": 3.3647882089131844, + "tokens_seen": 2535063552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023581818181818182, + "loss": 2.4657, + "theoretical_loss": 3.3647812898109146, + "tokens_seen": 2535129088 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357979797979798, + "loss": 2.6885, + "theoretical_loss": 3.36477437093759, + "tokens_seen": 2535194624 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357777777777778, + "loss": 2.4857, + "theoretical_loss": 3.364767452293198, + "tokens_seen": 2535260160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023575757575757574, + "loss": 2.8829, + "theoretical_loss": 3.3647605338777247, + "tokens_seen": 2535325696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023573737373737375, + "loss": 2.7466, + "theoretical_loss": 3.3647536156911566, + "tokens_seen": 2535391232 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002357171717171717, + "loss": 2.6395, + "theoretical_loss": 3.36474669773348, + "tokens_seen": 2535456768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023569696969696972, + "loss": 2.4943, + "theoretical_loss": 3.364739780004682, + "tokens_seen": 2535522304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023567676767676768, + "loss": 2.608, + "theoretical_loss": 3.364732862504748, + "tokens_seen": 2535587840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023565656565656566, + "loss": 2.5244, + "theoretical_loss": 3.3647259452336655, + "tokens_seen": 2535653376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023563636363636365, + "loss": 2.6334, + "theoretical_loss": 3.3647190281914208, + "tokens_seen": 2535718912 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002356161616161616, + "loss": 2.7209, + "theoretical_loss": 3.3647121113780005, + "tokens_seen": 2535784448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023559595959595962, + "loss": 2.4807, + "theoretical_loss": 3.364705194793391, + "tokens_seen": 2535849984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023557575757575757, + "loss": 2.7331, + "theoretical_loss": 3.3646982784375785, + "tokens_seen": 2535915520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023555555555555556, + "loss": 2.5747, + "theoretical_loss": 3.36469136231055, + "tokens_seen": 2535981056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023553535353535354, + "loss": 2.8273, + "theoretical_loss": 3.3646844464122916, + "tokens_seen": 2536046592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023551515151515152, + "loss": 2.4342, + "theoretical_loss": 3.36467753074279, + "tokens_seen": 2536112128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354949494949495, + "loss": 2.5361, + "theoretical_loss": 3.364670615302032, + "tokens_seen": 2536177664 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002354747474747475, + "loss": 2.447, + "theoretical_loss": 3.364663700090004, + "tokens_seen": 2536243200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023545454545454545, + "loss": 2.6409, + "theoretical_loss": 3.3646567851066917, + "tokens_seen": 2536308736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023543434343434343, + "loss": 2.5765, + "theoretical_loss": 3.364649870352083, + "tokens_seen": 2536374272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023541414141414142, + "loss": 2.4693, + "theoretical_loss": 3.3646429558261635, + "tokens_seen": 2536439808 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1425841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.190178632736206, + "objective/train/theoretical_loss": 3.36463604152892, + "objective/train/tokens_used": 895364576, + "theoretical_loss": 3.36463604152892, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353939393939394, + "loss": 2.5223, + "theoretical_loss": 3.36463604152892, + "tokens_seen": 2536505344 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023537373737373738, + "loss": 2.8758, + "theoretical_loss": 3.364629127460339, + "tokens_seen": 2536570880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023535353535353534, + "loss": 2.4536, + "theoretical_loss": 3.3646222136204074, + "tokens_seen": 2536636416 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023533333333333335, + "loss": 2.5197, + "theoretical_loss": 3.364615300009111, + "tokens_seen": 2536701952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002353131313131313, + "loss": 2.5232, + "theoretical_loss": 3.364608386626437, + "tokens_seen": 2536767488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023529292929292932, + "loss": 2.7069, + "theoretical_loss": 3.3646014734723715, + "tokens_seen": 2536833024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023527272727272728, + "loss": 2.604, + "theoretical_loss": 3.3645945605469008, + "tokens_seen": 2536898560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023525252525252523, + "loss": 2.6766, + "theoretical_loss": 3.3645876478500125, + "tokens_seen": 2536964096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023523232323232325, + "loss": 2.4638, + "theoretical_loss": 3.364580735381692, + "tokens_seen": 2537029632 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002352121212121212, + "loss": 2.7092, + "theoretical_loss": 3.364573823141926, + "tokens_seen": 2537095168 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002351919191919192, + "loss": 2.6018, + "theoretical_loss": 3.3645669111307024, + "tokens_seen": 2537160704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023517171717171717, + "loss": 2.7132, + "theoretical_loss": 3.364559999348006, + "tokens_seen": 2537226240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023515151515151515, + "loss": 2.5244, + "theoretical_loss": 3.364553087793824, + "tokens_seen": 2537291776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023513131313131314, + "loss": 2.5673, + "theoretical_loss": 3.364546176468143, + "tokens_seen": 2537357312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023511111111111112, + "loss": 2.4043, + "theoretical_loss": 3.3645392653709494, + "tokens_seen": 2537422848 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002350909090909091, + "loss": 2.7437, + "theoretical_loss": 3.36453235450223, + "tokens_seen": 2537488384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023507070707070706, + "loss": 2.6513, + "theoretical_loss": 3.364525443861971, + "tokens_seen": 2537553920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023505050505050505, + "loss": 2.3796, + "theoretical_loss": 3.3645185334501595, + "tokens_seen": 2537619456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023503030303030303, + "loss": 2.7665, + "theoretical_loss": 3.3645116232667815, + "tokens_seen": 2537684992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023501010101010101, + "loss": 2.4904, + "theoretical_loss": 3.364504713311824, + "tokens_seen": 2537750528 + }, + { + "epoch": 0.53, + "learning_rate": 0.000234989898989899, + "loss": 2.3876, + "theoretical_loss": 3.3644978035852726, + "tokens_seen": 2537816064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023496969696969698, + "loss": 2.6037, + "theoretical_loss": 3.364490894087115, + "tokens_seen": 2537881600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023494949494949494, + "loss": 2.5417, + "theoretical_loss": 3.3644839848173373, + "tokens_seen": 2537947136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00023492929292929295, + "loss": 2.6828, + "theoretical_loss": 3.364477075775926, + "tokens_seen": 2538012672 + }, + { + "epoch": 0.53, + "learning_rate": 0.0002349090909090909, + "loss": 2.7915, + "theoretical_loss": 3.364470166962868, + "tokens_seen": 2538078208 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 1426240, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.39086651802063, + "objective/train/theoretical_loss": 3.364463258378149, + "objective/train/tokens_used": 897002976, + "theoretical_loss": 3.364463258378149, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002348888888888889, + "loss": 2.5316, + "theoretical_loss": 3.364463258378149, + "tokens_seen": 2538143744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023486868686868688, + "loss": 2.6716, + "theoretical_loss": 3.364456350021756, + "tokens_seen": 2538209280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023484848484848486, + "loss": 2.7466, + "theoretical_loss": 3.3644494418936763, + "tokens_seen": 2538274816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023482828282828284, + "loss": 2.5211, + "theoretical_loss": 3.3644425339938957, + "tokens_seen": 2538340352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002348080808080808, + "loss": 2.9627, + "theoretical_loss": 3.364435626322401, + "tokens_seen": 2538405888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347878787878788, + "loss": 2.656, + "theoretical_loss": 3.3644287188791786, + "tokens_seen": 2538471424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023476767676767677, + "loss": 2.5558, + "theoretical_loss": 3.364421811664215, + "tokens_seen": 2538536960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023474747474747475, + "loss": 2.6657, + "theoretical_loss": 3.3644149046774965, + "tokens_seen": 2538602496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023472727272727274, + "loss": 2.5209, + "theoretical_loss": 3.3644079979190105, + "tokens_seen": 2538668032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002347070707070707, + "loss": 2.6667, + "theoretical_loss": 3.364401091388743, + "tokens_seen": 2538733568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002346868686868687, + "loss": 2.6847, + "theoretical_loss": 3.364394185086681, + "tokens_seen": 2538799104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023466666666666666, + "loss": 2.815, + "theoretical_loss": 3.3643872790128104, + "tokens_seen": 2538864640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023464646464646467, + "loss": 2.607, + "theoretical_loss": 3.3643803731671182, + "tokens_seen": 2538930176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023462626262626263, + "loss": 2.7011, + "theoretical_loss": 3.364373467549591, + "tokens_seen": 2538995712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002346060606060606, + "loss": 2.4813, + "theoretical_loss": 3.3643665621602157, + "tokens_seen": 2539061248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345858585858586, + "loss": 2.4597, + "theoretical_loss": 3.3643596569989778, + "tokens_seen": 2539126784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023456565656565655, + "loss": 2.7229, + "theoretical_loss": 3.3643527520658645, + "tokens_seen": 2539192320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023454545454545456, + "loss": 2.7842, + "theoretical_loss": 3.3643458473608625, + "tokens_seen": 2539257856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023452525252525252, + "loss": 2.298, + "theoretical_loss": 3.3643389428839585, + "tokens_seen": 2539323392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002345050505050505, + "loss": 2.538, + "theoretical_loss": 3.3643320386351387, + "tokens_seen": 2539388928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002344848484848485, + "loss": 2.6203, + "theoretical_loss": 3.36432513461439, + "tokens_seen": 2539454464 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023446464646464647, + "loss": 2.547, + "theoretical_loss": 3.3643182308216986, + "tokens_seen": 2539520000 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023444444444444446, + "loss": 2.6875, + "theoretical_loss": 3.3643113272570515, + "tokens_seen": 2539585536 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023442424242424244, + "loss": 2.7686, + "theoretical_loss": 3.3643044239204345, + "tokens_seen": 2539651072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002344040404040404, + "loss": 2.6815, + "theoretical_loss": 3.3642975208118355, + "tokens_seen": 2539716608 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1426896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3084793090820312, + "objective/train/theoretical_loss": 3.36429061793124, + "objective/train/tokens_used": 898641376, + "theoretical_loss": 3.36429061793124, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023438383838383838, + "loss": 2.4745, + "theoretical_loss": 3.36429061793124, + "tokens_seen": 2539782144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023436363636363637, + "loss": 2.4969, + "theoretical_loss": 3.364283715278635, + "tokens_seen": 2539847680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023434343434343435, + "loss": 2.5865, + "theoretical_loss": 3.364276812854007, + "tokens_seen": 2539913216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023432323232323233, + "loss": 2.5789, + "theoretical_loss": 3.3642699106573426, + "tokens_seen": 2539978752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002343030303030303, + "loss": 2.6836, + "theoretical_loss": 3.3642630086886287, + "tokens_seen": 2540044288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002342828282828283, + "loss": 2.6818, + "theoretical_loss": 3.364256106947851, + "tokens_seen": 2540109824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023426262626262626, + "loss": 2.603, + "theoretical_loss": 3.364249205434997, + "tokens_seen": 2540175360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023424242424242427, + "loss": 2.8808, + "theoretical_loss": 3.364242304150053, + "tokens_seen": 2540240896 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023422222222222223, + "loss": 2.5574, + "theoretical_loss": 3.3642354030930055, + "tokens_seen": 2540306432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023420202020202018, + "loss": 2.7086, + "theoretical_loss": 3.3642285022638414, + "tokens_seen": 2540371968 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341818181818182, + "loss": 2.6047, + "theoretical_loss": 3.3642216016625466, + "tokens_seen": 2540437504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023416161616161615, + "loss": 2.544, + "theoretical_loss": 3.3642147012891086, + "tokens_seen": 2540503040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023414141414141416, + "loss": 2.5098, + "theoretical_loss": 3.3642078011435133, + "tokens_seen": 2540568576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023412121212121212, + "loss": 2.5344, + "theoretical_loss": 3.3642009012257477, + "tokens_seen": 2540634112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002341010101010101, + "loss": 2.3992, + "theoretical_loss": 3.364194001535798, + "tokens_seen": 2540699648 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002340808080808081, + "loss": 2.679, + "theoretical_loss": 3.3641871020736516, + "tokens_seen": 2540765184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023406060606060604, + "loss": 2.5844, + "theoretical_loss": 3.364180202839294, + "tokens_seen": 2540830720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023404040404040405, + "loss": 2.5741, + "theoretical_loss": 3.3641733038327124, + "tokens_seen": 2540896256 + }, + { + "epoch": 0.54, + "learning_rate": 0.000234020202020202, + "loss": 2.7243, + "theoretical_loss": 3.3641664050538935, + "tokens_seen": 2540961792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023400000000000002, + "loss": 2.5474, + "theoretical_loss": 3.364159506502824, + "tokens_seen": 2541027328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023397979797979798, + "loss": 2.4911, + "theoretical_loss": 3.36415260817949, + "tokens_seen": 2541092864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023395959595959596, + "loss": 2.5668, + "theoretical_loss": 3.3641457100838785, + "tokens_seen": 2541158400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023393939393939395, + "loss": 2.4905, + "theoretical_loss": 3.3641388122159763, + "tokens_seen": 2541223936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023391919191919193, + "loss": 2.4804, + "theoretical_loss": 3.3641319145757693, + "tokens_seen": 2541289472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023389898989898992, + "loss": 2.7076, + "theoretical_loss": 3.3641250171632446, + "tokens_seen": 2541355008 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1428293, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6481032371520996, + "objective/train/theoretical_loss": 3.364118119978389, + "objective/train/tokens_used": 900279776, + "theoretical_loss": 3.364118119978389, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023387878787878787, + "loss": 2.6578, + "theoretical_loss": 3.364118119978389, + "tokens_seen": 2541420544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023385858585858586, + "loss": 2.5254, + "theoretical_loss": 3.3641112230211885, + "tokens_seen": 2541486080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023383838383838384, + "loss": 2.48, + "theoretical_loss": 3.3641043262916304, + "tokens_seen": 2541551616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023381818181818182, + "loss": 2.5572, + "theoretical_loss": 3.3640974297897004, + "tokens_seen": 2541617152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002337979797979798, + "loss": 2.5627, + "theoretical_loss": 3.364090533515386, + "tokens_seen": 2541682688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002337777777777778, + "loss": 2.5605, + "theoretical_loss": 3.364083637468674, + "tokens_seen": 2541748224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023375757575757575, + "loss": 2.4456, + "theoretical_loss": 3.36407674164955, + "tokens_seen": 2541813760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023373737373737376, + "loss": 2.7007, + "theoretical_loss": 3.364069846058001, + "tokens_seen": 2541879296 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023371717171717172, + "loss": 2.5275, + "theoretical_loss": 3.3640629506940147, + "tokens_seen": 2541944832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002336969696969697, + "loss": 2.6397, + "theoretical_loss": 3.364056055557576, + "tokens_seen": 2542010368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023367676767676768, + "loss": 2.5393, + "theoretical_loss": 3.3640491606486727, + "tokens_seen": 2542075904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023365656565656564, + "loss": 2.5371, + "theoretical_loss": 3.364042265967291, + "tokens_seen": 2542141440 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023363636363636365, + "loss": 2.7341, + "theoretical_loss": 3.3640353715134172, + "tokens_seen": 2542206976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002336161616161616, + "loss": 2.4447, + "theoretical_loss": 3.364028477287039, + "tokens_seen": 2542272512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023359595959595962, + "loss": 2.6265, + "theoretical_loss": 3.364021583288142, + "tokens_seen": 2542338048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023357575757575758, + "loss": 2.6247, + "theoretical_loss": 3.364014689516713, + "tokens_seen": 2542403584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023355555555555556, + "loss": 2.5595, + "theoretical_loss": 3.3640077959727392, + "tokens_seen": 2542469120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023353535353535355, + "loss": 2.4965, + "theoretical_loss": 3.3640009026562065, + "tokens_seen": 2542534656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002335151515151515, + "loss": 2.542, + "theoretical_loss": 3.363994009567102, + "tokens_seen": 2542600192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002334949494949495, + "loss": 2.6446, + "theoretical_loss": 3.3639871167054123, + "tokens_seen": 2542665728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023347474747474747, + "loss": 2.6683, + "theoretical_loss": 3.3639802240711236, + "tokens_seen": 2542731264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023345454545454545, + "loss": 2.7166, + "theoretical_loss": 3.3639733316642233, + "tokens_seen": 2542796800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023343434343434344, + "loss": 2.525, + "theoretical_loss": 3.3639664394846975, + "tokens_seen": 2542862336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023341414141414142, + "loss": 2.3622, + "theoretical_loss": 3.363959547532533, + "tokens_seen": 2542927872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002333939393939394, + "loss": 2.601, + "theoretical_loss": 3.363952655807716, + "tokens_seen": 2542993408 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1429761, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6550161838531494, + "objective/train/theoretical_loss": 3.363945764310234, + "objective/train/tokens_used": 901918176, + "theoretical_loss": 3.363945764310234, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002333737373737374, + "loss": 2.6039, + "theoretical_loss": 3.363945764310234, + "tokens_seen": 2543058944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023335353535353535, + "loss": 2.6938, + "theoretical_loss": 3.363938873040073, + "tokens_seen": 2543124480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023333333333333333, + "loss": 2.4907, + "theoretical_loss": 3.36393198199722, + "tokens_seen": 2543190016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023331313131313131, + "loss": 2.4805, + "theoretical_loss": 3.3639250911816614, + "tokens_seen": 2543255552 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002332929292929293, + "loss": 2.4177, + "theoretical_loss": 3.3639182005933836, + "tokens_seen": 2543321088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023327272727272728, + "loss": 2.6002, + "theoretical_loss": 3.3639113102323743, + "tokens_seen": 2543386624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023325252525252524, + "loss": 2.5466, + "theoretical_loss": 3.363904420098619, + "tokens_seen": 2543452160 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023323232323232325, + "loss": 2.5548, + "theoretical_loss": 3.3638975301921046, + "tokens_seen": 2543517696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002332121212121212, + "loss": 2.6386, + "theoretical_loss": 3.363890640512818, + "tokens_seen": 2543583232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002331919191919192, + "loss": 2.4874, + "theoretical_loss": 3.3638837510607456, + "tokens_seen": 2543648768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023317171717171718, + "loss": 2.7552, + "theoretical_loss": 3.3638768618358745, + "tokens_seen": 2543714304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023315151515151516, + "loss": 2.3076, + "theoretical_loss": 3.363869972838191, + "tokens_seen": 2543779840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023313131313131314, + "loss": 2.7173, + "theoretical_loss": 3.363863084067682, + "tokens_seen": 2543845376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002331111111111111, + "loss": 2.5024, + "theoretical_loss": 3.363856195524334, + "tokens_seen": 2543910912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002330909090909091, + "loss": 2.678, + "theoretical_loss": 3.3638493072081332, + "tokens_seen": 2543976448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023307070707070707, + "loss": 2.3088, + "theoretical_loss": 3.363842419119067, + "tokens_seen": 2544041984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023305050505050508, + "loss": 2.6514, + "theoretical_loss": 3.3638355312571218, + "tokens_seen": 2544107520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023303030303030304, + "loss": 2.6218, + "theoretical_loss": 3.363828643622284, + "tokens_seen": 2544173056 + }, + { + "epoch": 0.54, + "learning_rate": 0.000233010101010101, + "loss": 2.7878, + "theoretical_loss": 3.363821756214541, + "tokens_seen": 2544238592 + }, + { + "epoch": 0.54, + "learning_rate": 0.000232989898989899, + "loss": 2.7236, + "theoretical_loss": 3.3638148690338783, + "tokens_seen": 2544304128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023296969696969696, + "loss": 2.705, + "theoretical_loss": 3.3638079820802838, + "tokens_seen": 2544369664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023294949494949497, + "loss": 2.4333, + "theoretical_loss": 3.363801095353743, + "tokens_seen": 2544435200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023292929292929293, + "loss": 2.5241, + "theoretical_loss": 3.3637942088542436, + "tokens_seen": 2544500736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002329090909090909, + "loss": 2.7265, + "theoretical_loss": 3.3637873225817714, + "tokens_seen": 2544566272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328888888888889, + "loss": 2.6989, + "theoretical_loss": 3.363780436536314, + "tokens_seen": 2544631808 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1430428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6049644947052, + "objective/train/theoretical_loss": 3.363773550717857, + "objective/train/tokens_used": 903556576, + "theoretical_loss": 3.363773550717857, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023286868686868688, + "loss": 2.7109, + "theoretical_loss": 3.363773550717857, + "tokens_seen": 2544697344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023284848484848486, + "loss": 2.7538, + "theoretical_loss": 3.3637666651263882, + "tokens_seen": 2544762880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023282828282828282, + "loss": 2.6761, + "theoretical_loss": 3.363759779761893, + "tokens_seen": 2544828416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002328080808080808, + "loss": 2.6416, + "theoretical_loss": 3.3637528946243593, + "tokens_seen": 2544893952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002327878787878788, + "loss": 2.6537, + "theoretical_loss": 3.3637460097137732, + "tokens_seen": 2544959488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023276767676767677, + "loss": 2.4957, + "theoretical_loss": 3.363739125030121, + "tokens_seen": 2545025024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023274747474747476, + "loss": 2.7084, + "theoretical_loss": 3.3637322405733903, + "tokens_seen": 2545090560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023272727272727274, + "loss": 2.6804, + "theoretical_loss": 3.363725356343567, + "tokens_seen": 2545156096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002327070707070707, + "loss": 2.5231, + "theoretical_loss": 3.363718472340638, + "tokens_seen": 2545221632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326868686868687, + "loss": 2.4066, + "theoretical_loss": 3.3637115885645903, + "tokens_seen": 2545287168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023266666666666667, + "loss": 2.4074, + "theoretical_loss": 3.3637047050154103, + "tokens_seen": 2545352704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023264646464646465, + "loss": 2.7877, + "theoretical_loss": 3.363697821693085, + "tokens_seen": 2545418240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023262626262626263, + "loss": 2.5607, + "theoretical_loss": 3.3636909385976, + "tokens_seen": 2545483776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002326060606060606, + "loss": 2.7239, + "theoretical_loss": 3.3636840557289434, + "tokens_seen": 2545549312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002325858585858586, + "loss": 2.4847, + "theoretical_loss": 3.3636771730871007, + "tokens_seen": 2545614848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023256565656565656, + "loss": 2.4837, + "theoretical_loss": 3.3636702906720592, + "tokens_seen": 2545680384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023254545454545457, + "loss": 2.4325, + "theoretical_loss": 3.3636634084838057, + "tokens_seen": 2545745920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023252525252525253, + "loss": 2.5413, + "theoretical_loss": 3.363656526522327, + "tokens_seen": 2545811456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023250505050505048, + "loss": 2.655, + "theoretical_loss": 3.363649644787609, + "tokens_seen": 2545876992 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002324848484848485, + "loss": 2.736, + "theoretical_loss": 3.363642763279639, + "tokens_seen": 2545942528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023246464646464645, + "loss": 2.6027, + "theoretical_loss": 3.3636358819984036, + "tokens_seen": 2546008064 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023244444444444446, + "loss": 2.7424, + "theoretical_loss": 3.36362900094389, + "tokens_seen": 2546073600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023242424242424242, + "loss": 2.5303, + "theoretical_loss": 3.3636221201160836, + "tokens_seen": 2546139136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002324040404040404, + "loss": 2.6262, + "theoretical_loss": 3.3636152395149725, + "tokens_seen": 2546204672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002323838383838384, + "loss": 2.5217, + "theoretical_loss": 3.3636083591405423, + "tokens_seen": 2546270208 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1431073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8763437271118164, + "objective/train/theoretical_loss": 3.36360147899278, + "objective/train/tokens_used": 905194976, + "theoretical_loss": 3.36360147899278, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023236363636363637, + "loss": 2.5494, + "theoretical_loss": 3.36360147899278, + "tokens_seen": 2546335744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023234343434343436, + "loss": 2.4378, + "theoretical_loss": 3.363594599071673, + "tokens_seen": 2546401280 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002323232323232323, + "loss": 2.4509, + "theoretical_loss": 3.3635877193772075, + "tokens_seen": 2546466816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023230303030303032, + "loss": 2.6737, + "theoretical_loss": 3.36358083990937, + "tokens_seen": 2546532352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023228282828282828, + "loss": 2.5036, + "theoretical_loss": 3.363573960668147, + "tokens_seen": 2546597888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023226262626262626, + "loss": 2.5813, + "theoretical_loss": 3.363567081653526, + "tokens_seen": 2546663424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023224242424242425, + "loss": 2.4959, + "theoretical_loss": 3.363560202865493, + "tokens_seen": 2546728960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023222222222222223, + "loss": 2.682, + "theoretical_loss": 3.363553324304035, + "tokens_seen": 2546794496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023220202020202022, + "loss": 2.54, + "theoretical_loss": 3.363546445969139, + "tokens_seen": 2546860032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002321818181818182, + "loss": 2.6508, + "theoretical_loss": 3.363539567860791, + "tokens_seen": 2546925568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023216161616161616, + "loss": 2.4827, + "theoretical_loss": 3.363532689978978, + "tokens_seen": 2546991104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023214141414141414, + "loss": 2.7021, + "theoretical_loss": 3.3635258123236875, + "tokens_seen": 2547056640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023212121212121212, + "loss": 2.5051, + "theoretical_loss": 3.363518934894905, + "tokens_seen": 2547122176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002321010101010101, + "loss": 2.4696, + "theoretical_loss": 3.363512057692618, + "tokens_seen": 2547187712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002320808080808081, + "loss": 2.5828, + "theoretical_loss": 3.3635051807168126, + "tokens_seen": 2547253248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023206060606060605, + "loss": 2.5522, + "theoretical_loss": 3.363498303967476, + "tokens_seen": 2547318784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023204040404040406, + "loss": 2.6036, + "theoretical_loss": 3.363491427444595, + "tokens_seen": 2547384320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023202020202020202, + "loss": 2.5428, + "theoretical_loss": 3.363484551148156, + "tokens_seen": 2547449856 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023200000000000003, + "loss": 2.4807, + "theoretical_loss": 3.363477675078146, + "tokens_seen": 2547515392 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023197979797979799, + "loss": 2.5676, + "theoretical_loss": 3.363470799234551, + "tokens_seen": 2547580928 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023195959595959594, + "loss": 2.4983, + "theoretical_loss": 3.3634639236173585, + "tokens_seen": 2547646464 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023193939393939395, + "loss": 2.5454, + "theoretical_loss": 3.363457048226555, + "tokens_seen": 2547712000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002319191919191919, + "loss": 2.6331, + "theoretical_loss": 3.3634501730621276, + "tokens_seen": 2547777536 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023189898989898992, + "loss": 2.6432, + "theoretical_loss": 3.363443298124062, + "tokens_seen": 2547843072 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023187878787878788, + "loss": 2.7183, + "theoretical_loss": 3.3634364234123457, + "tokens_seen": 2547908608 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1432125, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.702869176864624, + "objective/train/theoretical_loss": 3.363429548926965, + "objective/train/tokens_used": 906833376, + "theoretical_loss": 3.363429548926965, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023185858585858586, + "loss": 2.6774, + "theoretical_loss": 3.363429548926965, + "tokens_seen": 2547974144 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023183838383838385, + "loss": 2.7259, + "theoretical_loss": 3.3634226746679077, + "tokens_seen": 2548039680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023181818181818183, + "loss": 2.499, + "theoretical_loss": 3.3634158006351593, + "tokens_seen": 2548105216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023179797979797981, + "loss": 2.5567, + "theoretical_loss": 3.363408926828707, + "tokens_seen": 2548170752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023177777777777777, + "loss": 2.6026, + "theoretical_loss": 3.3634020532485374, + "tokens_seen": 2548236288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023175757575757575, + "loss": 2.6606, + "theoretical_loss": 3.3633951798946375, + "tokens_seen": 2548301824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023173737373737374, + "loss": 2.6981, + "theoretical_loss": 3.3633883067669936, + "tokens_seen": 2548367360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023171717171717172, + "loss": 2.3881, + "theoretical_loss": 3.3633814338655927, + "tokens_seen": 2548432896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316969696969697, + "loss": 2.5701, + "theoretical_loss": 3.363374561190422, + "tokens_seen": 2548498432 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002316767676767677, + "loss": 2.4476, + "theoretical_loss": 3.363367688741467, + "tokens_seen": 2548563968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023165656565656565, + "loss": 2.611, + "theoretical_loss": 3.363360816518716, + "tokens_seen": 2548629504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023163636363636366, + "loss": 2.6332, + "theoretical_loss": 3.3633539445221543, + "tokens_seen": 2548695040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023161616161616162, + "loss": 2.5833, + "theoretical_loss": 3.3633470727517696, + "tokens_seen": 2548760576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315959595959596, + "loss": 2.6032, + "theoretical_loss": 3.3633402012075484, + "tokens_seen": 2548826112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023157575757575758, + "loss": 2.6635, + "theoretical_loss": 3.363333329889477, + "tokens_seen": 2548891648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023155555555555554, + "loss": 2.6164, + "theoretical_loss": 3.363326458797543, + "tokens_seen": 2548957184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023153535353535355, + "loss": 2.6894, + "theoretical_loss": 3.363319587931732, + "tokens_seen": 2549022720 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002315151515151515, + "loss": 2.8809, + "theoretical_loss": 3.3633127172920316, + "tokens_seen": 2549088256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023149494949494952, + "loss": 2.4821, + "theoretical_loss": 3.3633058468784287, + "tokens_seen": 2549153792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023147474747474748, + "loss": 2.3896, + "theoretical_loss": 3.3632989766909094, + "tokens_seen": 2549219328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023145454545454546, + "loss": 2.5983, + "theoretical_loss": 3.3632921067294608, + "tokens_seen": 2549284864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023143434343434344, + "loss": 2.7556, + "theoretical_loss": 3.3632852369940696, + "tokens_seen": 2549350400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002314141414141414, + "loss": 2.7494, + "theoretical_loss": 3.3632783674847224, + "tokens_seen": 2549415936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002313939393939394, + "loss": 2.4444, + "theoretical_loss": 3.363271498201406, + "tokens_seen": 2549481472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023137373737373737, + "loss": 2.5418, + "theoretical_loss": 3.3632646291441075, + "tokens_seen": 2549547008 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1432724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.943678379058838, + "objective/train/theoretical_loss": 3.363257760312813, + "objective/train/tokens_used": 908471776, + "theoretical_loss": 3.363257760312813, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023135353535353538, + "loss": 2.8322, + "theoretical_loss": 3.363257760312813, + "tokens_seen": 2549612544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023133333333333334, + "loss": 2.3931, + "theoretical_loss": 3.36325089170751, + "tokens_seen": 2549678080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023131313131313132, + "loss": 2.5412, + "theoretical_loss": 3.3632440233281846, + "tokens_seen": 2549743616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312929292929293, + "loss": 2.6123, + "theoretical_loss": 3.3632371551748244, + "tokens_seen": 2549809152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023127272727272726, + "loss": 2.4285, + "theoretical_loss": 3.3632302872474154, + "tokens_seen": 2549874688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023125252525252527, + "loss": 2.6787, + "theoretical_loss": 3.3632234195459443, + "tokens_seen": 2549940224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023123232323232323, + "loss": 2.397, + "theoretical_loss": 3.3632165520703983, + "tokens_seen": 2550005760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002312121212121212, + "loss": 2.7275, + "theoretical_loss": 3.363209684820764, + "tokens_seen": 2550071296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311919191919192, + "loss": 2.8143, + "theoretical_loss": 3.363202817797028, + "tokens_seen": 2550136832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023117171717171718, + "loss": 2.7493, + "theoretical_loss": 3.3631959509991773, + "tokens_seen": 2550202368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023115151515151517, + "loss": 2.7119, + "theoretical_loss": 3.3631890844271988, + "tokens_seen": 2550267904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023113131313131315, + "loss": 2.6133, + "theoretical_loss": 3.363182218081079, + "tokens_seen": 2550333440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002311111111111111, + "loss": 2.4086, + "theoretical_loss": 3.3631753519608045, + "tokens_seen": 2550398976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002310909090909091, + "loss": 2.5384, + "theoretical_loss": 3.3631684860663627, + "tokens_seen": 2550464512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023107070707070707, + "loss": 2.328, + "theoretical_loss": 3.3631616203977392, + "tokens_seen": 2550530048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023105050505050506, + "loss": 2.8375, + "theoretical_loss": 3.3631547549549223, + "tokens_seen": 2550595584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023103030303030304, + "loss": 2.6283, + "theoretical_loss": 3.3631478897378977, + "tokens_seen": 2550661120 + }, + { + "epoch": 0.54, + "learning_rate": 0.000231010101010101, + "loss": 2.5101, + "theoretical_loss": 3.3631410247466524, + "tokens_seen": 2550726656 + }, + { + "epoch": 0.54, + "learning_rate": 0.000230989898989899, + "loss": 2.5476, + "theoretical_loss": 3.3631341599811737, + "tokens_seen": 2550792192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023096969696969697, + "loss": 2.6931, + "theoretical_loss": 3.3631272954414473, + "tokens_seen": 2550857728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023094949494949498, + "loss": 2.6573, + "theoretical_loss": 3.3631204311274607, + "tokens_seen": 2550923264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023092929292929293, + "loss": 2.5606, + "theoretical_loss": 3.3631135670392007, + "tokens_seen": 2550988800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002309090909090909, + "loss": 2.6286, + "theoretical_loss": 3.363106703176654, + "tokens_seen": 2551054336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308888888888889, + "loss": 2.6453, + "theoretical_loss": 3.3630998395398075, + "tokens_seen": 2551119872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023086868686868686, + "loss": 2.6776, + "theoretical_loss": 3.3630929761286477, + "tokens_seen": 2551185408 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1434145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.682487964630127, + "objective/train/theoretical_loss": 3.3630861129431615, + "objective/train/tokens_used": 910110176, + "theoretical_loss": 3.3630861129431615, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023084848484848487, + "loss": 2.7394, + "theoretical_loss": 3.3630861129431615, + "tokens_seen": 2551250944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023082828282828283, + "loss": 2.7459, + "theoretical_loss": 3.3630792499833357, + "tokens_seen": 2551316480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002308080808080808, + "loss": 2.5872, + "theoretical_loss": 3.363072387249157, + "tokens_seen": 2551382016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307878787878788, + "loss": 2.5209, + "theoretical_loss": 3.363065524740612, + "tokens_seen": 2551447552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023076767676767675, + "loss": 2.5792, + "theoretical_loss": 3.3630586624576884, + "tokens_seen": 2551513088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023074747474747476, + "loss": 2.6572, + "theoretical_loss": 3.3630518004003718, + "tokens_seen": 2551578624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023072727272727272, + "loss": 2.7233, + "theoretical_loss": 3.3630449385686494, + "tokens_seen": 2551644160 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002307070707070707, + "loss": 2.6003, + "theoretical_loss": 3.3630380769625083, + "tokens_seen": 2551709696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306868686868687, + "loss": 2.5249, + "theoretical_loss": 3.3630312155819353, + "tokens_seen": 2551775232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023066666666666667, + "loss": 2.5136, + "theoretical_loss": 3.3630243544269165, + "tokens_seen": 2551840768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023064646464646466, + "loss": 2.8992, + "theoretical_loss": 3.3630174934974395, + "tokens_seen": 2551906304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023062626262626264, + "loss": 2.383, + "theoretical_loss": 3.363010632793491, + "tokens_seen": 2551971840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002306060606060606, + "loss": 2.6977, + "theoretical_loss": 3.363003772315057, + "tokens_seen": 2552037376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023058585858585858, + "loss": 2.9549, + "theoretical_loss": 3.3629969120621257, + "tokens_seen": 2552102912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023056565656565656, + "loss": 2.5539, + "theoretical_loss": 3.3629900520346823, + "tokens_seen": 2552168448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023054545454545455, + "loss": 2.6807, + "theoretical_loss": 3.362983192232715, + "tokens_seen": 2552233984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023052525252525253, + "loss": 2.5499, + "theoretical_loss": 3.3629763326562094, + "tokens_seen": 2552299520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023050505050505052, + "loss": 2.6371, + "theoretical_loss": 3.362969473305153, + "tokens_seen": 2552365056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304848484848485, + "loss": 2.9068, + "theoretical_loss": 3.3629626141795326, + "tokens_seen": 2552430592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023046464646464646, + "loss": 2.6493, + "theoretical_loss": 3.3629557552793345, + "tokens_seen": 2552496128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023044444444444447, + "loss": 2.6197, + "theoretical_loss": 3.3629488966045464, + "tokens_seen": 2552561664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023042424242424243, + "loss": 2.5375, + "theoretical_loss": 3.362942038155154, + "tokens_seen": 2552627200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002304040404040404, + "loss": 2.5662, + "theoretical_loss": 3.362935179931145, + "tokens_seen": 2552692736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002303838383838384, + "loss": 2.5865, + "theoretical_loss": 3.3629283219325057, + "tokens_seen": 2552758272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023036363636363635, + "loss": 2.6507, + "theoretical_loss": 3.3629214641592236, + "tokens_seen": 2552823808 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1434830, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7586021423339844, + "objective/train/theoretical_loss": 3.3629146066112847, + "objective/train/tokens_used": 911748576, + "theoretical_loss": 3.3629146066112847, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023034343434343436, + "loss": 2.7714, + "theoretical_loss": 3.3629146066112847, + "tokens_seen": 2552889344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023032323232323232, + "loss": 2.3974, + "theoretical_loss": 3.3629077492886763, + "tokens_seen": 2552954880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023030303030303033, + "loss": 2.5255, + "theoretical_loss": 3.3629008921913845, + "tokens_seen": 2553020416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023028282828282829, + "loss": 2.4579, + "theoretical_loss": 3.362894035319397, + "tokens_seen": 2553085952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023026262626262627, + "loss": 2.7541, + "theoretical_loss": 3.3628871786727004, + "tokens_seen": 2553151488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023024242424242425, + "loss": 2.6789, + "theoretical_loss": 3.362880322251281, + "tokens_seen": 2553217024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002302222222222222, + "loss": 2.7117, + "theoretical_loss": 3.362873466055126, + "tokens_seen": 2553282560 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023020202020202022, + "loss": 2.6889, + "theoretical_loss": 3.3628666100842226, + "tokens_seen": 2553348096 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023018181818181818, + "loss": 2.5617, + "theoretical_loss": 3.3628597543385568, + "tokens_seen": 2553413632 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023016161616161616, + "loss": 2.6293, + "theoretical_loss": 3.362852898818116, + "tokens_seen": 2553479168 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023014141414141415, + "loss": 2.4177, + "theoretical_loss": 3.362846043522887, + "tokens_seen": 2553544704 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023012121212121213, + "loss": 2.8869, + "theoretical_loss": 3.3628391884528566, + "tokens_seen": 2553610240 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023010101010101011, + "loss": 2.6044, + "theoretical_loss": 3.362832333608011, + "tokens_seen": 2553675776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002300808080808081, + "loss": 2.723, + "theoretical_loss": 3.362825478988338, + "tokens_seen": 2553741312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023006060606060606, + "loss": 2.7284, + "theoretical_loss": 3.3628186245938236, + "tokens_seen": 2553806848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023004040404040404, + "loss": 2.4538, + "theoretical_loss": 3.362811770424455, + "tokens_seen": 2553872384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023002020202020202, + "loss": 2.5758, + "theoretical_loss": 3.3628049164802194, + "tokens_seen": 2553937920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00023, + "loss": 2.767, + "theoretical_loss": 3.362798062761103, + "tokens_seen": 2554003456 + }, + { + "epoch": 0.54, + "learning_rate": 0.000229979797979798, + "loss": 2.6725, + "theoretical_loss": 3.362791209267093, + "tokens_seen": 2554068992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022995959595959595, + "loss": 2.4399, + "theoretical_loss": 3.3627843559981763, + "tokens_seen": 2554134528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022993939393939396, + "loss": 2.6418, + "theoretical_loss": 3.362777502954339, + "tokens_seen": 2554200064 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022991919191919192, + "loss": 2.5739, + "theoretical_loss": 3.3627706501355688, + "tokens_seen": 2554265600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0002298989898989899, + "loss": 2.6504, + "theoretical_loss": 3.362763797541852, + "tokens_seen": 2554331136 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022987878787878788, + "loss": 2.5875, + "theoretical_loss": 3.3627569451731754, + "tokens_seen": 2554396672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00022985858585858584, + "loss": 2.6726, + "theoretical_loss": 3.3627500930295264, + "tokens_seen": 2554462208 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 1435894, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3847389221191406, + "objective/train/theoretical_loss": 3.3627432411108917, + "objective/train/tokens_used": 913386976, + "theoretical_loss": 3.3627432411108917, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022983838383838385, + "loss": 2.6372, + "theoretical_loss": 3.3627432411108917, + "tokens_seen": 2554527744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002298181818181818, + "loss": 2.6084, + "theoretical_loss": 3.3627363894172575, + "tokens_seen": 2554593280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022979797979797982, + "loss": 2.5866, + "theoretical_loss": 3.3627295379486117, + "tokens_seen": 2554658816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022977777777777778, + "loss": 2.8456, + "theoretical_loss": 3.36272268670494, + "tokens_seen": 2554724352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022975757575757576, + "loss": 2.648, + "theoretical_loss": 3.36271583568623, + "tokens_seen": 2554789888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022973737373737374, + "loss": 2.7749, + "theoretical_loss": 3.362708984892468, + "tokens_seen": 2554855424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002297171717171717, + "loss": 2.7972, + "theoretical_loss": 3.3627021343236416, + "tokens_seen": 2554920960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002296969696969697, + "loss": 2.4335, + "theoretical_loss": 3.362695283979737, + "tokens_seen": 2554986496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022967676767676767, + "loss": 2.6792, + "theoretical_loss": 3.362688433860741, + "tokens_seen": 2555052032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022965656565656568, + "loss": 2.9188, + "theoretical_loss": 3.362681583966641, + "tokens_seen": 2555117568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022963636363636364, + "loss": 2.5717, + "theoretical_loss": 3.3626747342974235, + "tokens_seen": 2555183104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022961616161616162, + "loss": 2.3936, + "theoretical_loss": 3.3626678848530753, + "tokens_seen": 2555248640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002295959595959596, + "loss": 2.6837, + "theoretical_loss": 3.3626610356335833, + "tokens_seen": 2555314176 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002295757575757576, + "loss": 2.7194, + "theoretical_loss": 3.3626541866389346, + "tokens_seen": 2555379712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022955555555555557, + "loss": 2.6931, + "theoretical_loss": 3.3626473378691157, + "tokens_seen": 2555445248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022953535353535353, + "loss": 2.5431, + "theoretical_loss": 3.3626404893241135, + "tokens_seen": 2555510784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022951515151515151, + "loss": 2.7541, + "theoretical_loss": 3.362633641003915, + "tokens_seen": 2555576320 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002294949494949495, + "loss": 2.7841, + "theoretical_loss": 3.3626267929085074, + "tokens_seen": 2555641856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022947474747474748, + "loss": 2.6457, + "theoretical_loss": 3.362619945037877, + "tokens_seen": 2555707392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022945454545454547, + "loss": 2.6707, + "theoretical_loss": 3.3626130973920105, + "tokens_seen": 2555772928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022943434343434345, + "loss": 2.752, + "theoretical_loss": 3.362606249970895, + "tokens_seen": 2555838464 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002294141414141414, + "loss": 2.6746, + "theoretical_loss": 3.362599402774518, + "tokens_seen": 2555904000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022939393939393942, + "loss": 2.6981, + "theoretical_loss": 3.3625925558028653, + "tokens_seen": 2555969536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022937373737373737, + "loss": 2.7287, + "theoretical_loss": 3.362585709055925, + "tokens_seen": 2556035072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022935353535353536, + "loss": 2.611, + "theoretical_loss": 3.3625788625336823, + "tokens_seen": 2556100608 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1436568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.730395793914795, + "objective/train/theoretical_loss": 3.3625720162361254, + "objective/train/tokens_used": 915025376, + "theoretical_loss": 3.3625720162361254, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022933333333333334, + "loss": 2.7887, + "theoretical_loss": 3.3625720162361254, + "tokens_seen": 2556166144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002293131313131313, + "loss": 2.5711, + "theoretical_loss": 3.362565170163241, + "tokens_seen": 2556231680 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002292929292929293, + "loss": 2.4224, + "theoretical_loss": 3.3625583243150157, + "tokens_seen": 2556297216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022927272727272727, + "loss": 2.4288, + "theoretical_loss": 3.362551478691436, + "tokens_seen": 2556362752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022925252525252528, + "loss": 2.8423, + "theoretical_loss": 3.3625446332924893, + "tokens_seen": 2556428288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022923232323232323, + "loss": 2.6031, + "theoretical_loss": 3.362537788118163, + "tokens_seen": 2556493824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002292121212121212, + "loss": 2.5505, + "theoretical_loss": 3.3625309431684425, + "tokens_seen": 2556559360 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291919191919192, + "loss": 2.8642, + "theoretical_loss": 3.362524098443316, + "tokens_seen": 2556624896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022917171717171716, + "loss": 2.7366, + "theoretical_loss": 3.3625172539427695, + "tokens_seen": 2556690432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022915151515151517, + "loss": 2.6521, + "theoretical_loss": 3.36251040966679, + "tokens_seen": 2556755968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022913131313131313, + "loss": 2.3654, + "theoretical_loss": 3.362503565615365, + "tokens_seen": 2556821504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002291111111111111, + "loss": 2.4612, + "theoretical_loss": 3.362496721788481, + "tokens_seen": 2556887040 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002290909090909091, + "loss": 2.5168, + "theoretical_loss": 3.3624898781861248, + "tokens_seen": 2556952576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022907070707070708, + "loss": 2.7297, + "theoretical_loss": 3.3624830348082835, + "tokens_seen": 2557018112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022905050505050506, + "loss": 2.3909, + "theoretical_loss": 3.3624761916549435, + "tokens_seen": 2557083648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022903030303030302, + "loss": 2.6925, + "theoretical_loss": 3.3624693487260924, + "tokens_seen": 2557149184 + }, + { + "epoch": 0.55, + "learning_rate": 0.000229010101010101, + "loss": 2.5454, + "theoretical_loss": 3.3624625060217164, + "tokens_seen": 2557214720 + }, + { + "epoch": 0.55, + "learning_rate": 0.000228989898989899, + "loss": 2.6087, + "theoretical_loss": 3.3624556635418026, + "tokens_seen": 2557280256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022896969696969697, + "loss": 2.4845, + "theoretical_loss": 3.362448821286338, + "tokens_seen": 2557345792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022894949494949496, + "loss": 2.6751, + "theoretical_loss": 3.3624419792553097, + "tokens_seen": 2557411328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022892929292929294, + "loss": 2.6913, + "theoretical_loss": 3.3624351374487045, + "tokens_seen": 2557476864 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002289090909090909, + "loss": 2.6043, + "theoretical_loss": 3.3624282958665086, + "tokens_seen": 2557542400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002288888888888889, + "loss": 2.4831, + "theoretical_loss": 3.3624214545087097, + "tokens_seen": 2557607936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022886868686868686, + "loss": 2.5128, + "theoretical_loss": 3.3624146133752943, + "tokens_seen": 2557673472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022884848484848485, + "loss": 2.4242, + "theoretical_loss": 3.3624077724662493, + "tokens_seen": 2557739008 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1437925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.271354913711548, + "objective/train/theoretical_loss": 3.362400931781562, + "objective/train/tokens_used": 916663776, + "theoretical_loss": 3.362400931781562, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022882828282828283, + "loss": 2.6889, + "theoretical_loss": 3.362400931781562, + "tokens_seen": 2557804544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022880808080808082, + "loss": 2.4222, + "theoretical_loss": 3.3623940913212187, + "tokens_seen": 2557870080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002287878787878788, + "loss": 2.5404, + "theoretical_loss": 3.362387251085207, + "tokens_seen": 2557935616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022876767676767676, + "loss": 2.4251, + "theoretical_loss": 3.362380411073513, + "tokens_seen": 2558001152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022874747474747477, + "loss": 2.545, + "theoretical_loss": 3.3623735712861236, + "tokens_seen": 2558066688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022872727272727273, + "loss": 2.8041, + "theoretical_loss": 3.362366731723027, + "tokens_seen": 2558132224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022870707070707074, + "loss": 2.6927, + "theoretical_loss": 3.3623598923842084, + "tokens_seen": 2558197760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002286868686868687, + "loss": 2.8236, + "theoretical_loss": 3.3623530532696555, + "tokens_seen": 2558263296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022866666666666665, + "loss": 2.8249, + "theoretical_loss": 3.362346214379355, + "tokens_seen": 2558328832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022864646464646466, + "loss": 2.5996, + "theoretical_loss": 3.3623393757132947, + "tokens_seen": 2558394368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022862626262626262, + "loss": 2.5373, + "theoretical_loss": 3.3623325372714605, + "tokens_seen": 2558459904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022860606060606063, + "loss": 2.7598, + "theoretical_loss": 3.362325699053839, + "tokens_seen": 2558525440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022858585858585859, + "loss": 2.4923, + "theoretical_loss": 3.3623188610604187, + "tokens_seen": 2558590976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022856565656565657, + "loss": 2.7246, + "theoretical_loss": 3.3623120232911847, + "tokens_seen": 2558656512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022854545454545455, + "loss": 2.6808, + "theoretical_loss": 3.3623051857461252, + "tokens_seen": 2558722048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022852525252525254, + "loss": 2.4361, + "theoretical_loss": 3.362298348425226, + "tokens_seen": 2558787584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022850505050505052, + "loss": 2.5495, + "theoretical_loss": 3.362291511328475, + "tokens_seen": 2558853120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022848484848484848, + "loss": 2.6956, + "theoretical_loss": 3.3622846744558585, + "tokens_seen": 2558918656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022846464646464646, + "loss": 2.7023, + "theoretical_loss": 3.362277837807364, + "tokens_seen": 2558984192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022844444444444445, + "loss": 2.5927, + "theoretical_loss": 3.362271001382978, + "tokens_seen": 2559049728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022842424242424243, + "loss": 2.6348, + "theoretical_loss": 3.3622641651826872, + "tokens_seen": 2559115264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022840404040404041, + "loss": 2.6435, + "theoretical_loss": 3.362257329206479, + "tokens_seen": 2559180800 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283838383838384, + "loss": 2.6642, + "theoretical_loss": 3.36225049345434, + "tokens_seen": 2559246336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022836363636363636, + "loss": 2.3048, + "theoretical_loss": 3.362243657926257, + "tokens_seen": 2559311872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022834343434343437, + "loss": 2.678, + "theoretical_loss": 3.3622368226222177, + "tokens_seen": 2559377408 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1438686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.241694927215576, + "objective/train/theoretical_loss": 3.362229987542208, + "objective/train/tokens_used": 918302176, + "theoretical_loss": 3.362229987542208, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022832323232323232, + "loss": 2.5167, + "theoretical_loss": 3.362229987542208, + "tokens_seen": 2559442944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002283030303030303, + "loss": 2.6161, + "theoretical_loss": 3.3622231526862154, + "tokens_seen": 2559508480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002282828282828283, + "loss": 2.6247, + "theoretical_loss": 3.3622163180542266, + "tokens_seen": 2559574016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022826262626262625, + "loss": 2.4729, + "theoretical_loss": 3.362209483646229, + "tokens_seen": 2559639552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022824242424242426, + "loss": 2.5519, + "theoretical_loss": 3.3622026494622084, + "tokens_seen": 2559705088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022822222222222222, + "loss": 2.837, + "theoretical_loss": 3.362195815502153, + "tokens_seen": 2559770624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022820202020202023, + "loss": 2.4176, + "theoretical_loss": 3.362188981766049, + "tokens_seen": 2559836160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022818181818181818, + "loss": 2.4739, + "theoretical_loss": 3.3621821482538836, + "tokens_seen": 2559901696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022816161616161614, + "loss": 2.6065, + "theoretical_loss": 3.362175314965644, + "tokens_seen": 2559967232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022814141414141415, + "loss": 2.3902, + "theoretical_loss": 3.3621684819013167, + "tokens_seen": 2560032768 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002281212121212121, + "loss": 2.7102, + "theoretical_loss": 3.3621616490608885, + "tokens_seen": 2560098304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022810101010101012, + "loss": 2.5281, + "theoretical_loss": 3.362154816444346, + "tokens_seen": 2560163840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022808080808080808, + "loss": 2.6491, + "theoretical_loss": 3.3621479840516777, + "tokens_seen": 2560229376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022806060606060606, + "loss": 2.6743, + "theoretical_loss": 3.362141151882869, + "tokens_seen": 2560294912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022804040404040404, + "loss": 2.4778, + "theoretical_loss": 3.3621343199379075, + "tokens_seen": 2560360448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022802020202020203, + "loss": 2.5804, + "theoretical_loss": 3.36212748821678, + "tokens_seen": 2560425984 + }, + { + "epoch": 0.55, + "learning_rate": 0.000228, + "loss": 2.5327, + "theoretical_loss": 3.3621206567194735, + "tokens_seen": 2560491520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022797979797979797, + "loss": 2.5317, + "theoretical_loss": 3.3621138254459746, + "tokens_seen": 2560557056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022795959595959598, + "loss": 2.6461, + "theoretical_loss": 3.3621069943962705, + "tokens_seen": 2560622592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022793939393939394, + "loss": 2.5232, + "theoretical_loss": 3.3621001635703482, + "tokens_seen": 2560688128 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022791919191919192, + "loss": 2.7867, + "theoretical_loss": 3.3620933329681946, + "tokens_seen": 2560753664 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002278989898989899, + "loss": 2.5941, + "theoretical_loss": 3.362086502589797, + "tokens_seen": 2560819200 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002278787878787879, + "loss": 2.7251, + "theoretical_loss": 3.3620796724351414, + "tokens_seen": 2560884736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022785858585858587, + "loss": 2.7012, + "theoretical_loss": 3.3620728425042157, + "tokens_seen": 2560950272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022783838383838386, + "loss": 2.6382, + "theoretical_loss": 3.3620660127970066, + "tokens_seen": 2561015808 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1440126, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9040749073028564, + "objective/train/theoretical_loss": 3.3620591833135003, + "objective/train/tokens_used": 919940576, + "theoretical_loss": 3.3620591833135003, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022781818181818181, + "loss": 2.7539, + "theoretical_loss": 3.3620591833135003, + "tokens_seen": 2561081344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002277979797979798, + "loss": 2.4677, + "theoretical_loss": 3.362052354053685, + "tokens_seen": 2561146880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022777777777777778, + "loss": 2.8325, + "theoretical_loss": 3.3620455250175465, + "tokens_seen": 2561212416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022775757575757577, + "loss": 2.5052, + "theoretical_loss": 3.3620386962050723, + "tokens_seen": 2561277952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022773737373737375, + "loss": 2.5451, + "theoretical_loss": 3.36203186761625, + "tokens_seen": 2561343488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002277171717171717, + "loss": 2.594, + "theoretical_loss": 3.362025039251065, + "tokens_seen": 2561409024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022769696969696972, + "loss": 2.4279, + "theoretical_loss": 3.362018211109506, + "tokens_seen": 2561474560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022767676767676767, + "loss": 2.697, + "theoretical_loss": 3.3620113831915583, + "tokens_seen": 2561540096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022765656565656569, + "loss": 2.5207, + "theoretical_loss": 3.36200455549721, + "tokens_seen": 2561605632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022763636363636364, + "loss": 2.6228, + "theoretical_loss": 3.3619977280264477, + "tokens_seen": 2561671168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002276161616161616, + "loss": 2.5549, + "theoretical_loss": 3.361990900779258, + "tokens_seen": 2561736704 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002275959595959596, + "loss": 2.4984, + "theoretical_loss": 3.361984073755629, + "tokens_seen": 2561802240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022757575757575757, + "loss": 2.6847, + "theoretical_loss": 3.3619772469555462, + "tokens_seen": 2561867776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022755555555555558, + "loss": 2.7405, + "theoretical_loss": 3.3619704203789973, + "tokens_seen": 2561933312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022753535353535354, + "loss": 2.7338, + "theoretical_loss": 3.361963594025969, + "tokens_seen": 2561998848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022751515151515152, + "loss": 2.457, + "theoretical_loss": 3.361956767896449, + "tokens_seen": 2562064384 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274949494949495, + "loss": 2.6227, + "theoretical_loss": 3.3619499419904235, + "tokens_seen": 2562129920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022747474747474746, + "loss": 2.673, + "theoretical_loss": 3.3619431163078795, + "tokens_seen": 2562195456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022745454545454547, + "loss": 2.5678, + "theoretical_loss": 3.3619362908488046, + "tokens_seen": 2562260992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022743434343434343, + "loss": 2.6431, + "theoretical_loss": 3.361929465613185, + "tokens_seen": 2562326528 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002274141414141414, + "loss": 2.4897, + "theoretical_loss": 3.3619226406010077, + "tokens_seen": 2562392064 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002273939393939394, + "loss": 2.7092, + "theoretical_loss": 3.36191581581226, + "tokens_seen": 2562457600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022737373737373738, + "loss": 2.4221, + "theoretical_loss": 3.3619089912469295, + "tokens_seen": 2562523136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022735353535353536, + "loss": 2.6105, + "theoretical_loss": 3.3619021669050024, + "tokens_seen": 2562588672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022733333333333335, + "loss": 2.745, + "theoretical_loss": 3.3618953427864655, + "tokens_seen": 2562654208 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1440814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.090264320373535, + "objective/train/theoretical_loss": 3.361888518891306, + "objective/train/tokens_used": 921578976, + "theoretical_loss": 3.361888518891306, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002273131313131313, + "loss": 2.5051, + "theoretical_loss": 3.361888518891306, + "tokens_seen": 2562719744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002272929292929293, + "loss": 2.5418, + "theoretical_loss": 3.3618816952195107, + "tokens_seen": 2562785280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022727272727272727, + "loss": 2.6144, + "theoretical_loss": 3.361874871771067, + "tokens_seen": 2562850816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022725252525252526, + "loss": 2.5611, + "theoretical_loss": 3.361868048545962, + "tokens_seen": 2562916352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022723232323232324, + "loss": 2.6543, + "theoretical_loss": 3.3618612255441827, + "tokens_seen": 2562981888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002272121212121212, + "loss": 2.435, + "theoretical_loss": 3.361854402765715, + "tokens_seen": 2563047424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002271919191919192, + "loss": 2.5302, + "theoretical_loss": 3.3618475802105467, + "tokens_seen": 2563112960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022717171717171717, + "loss": 2.6545, + "theoretical_loss": 3.361840757878665, + "tokens_seen": 2563178496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022715151515151518, + "loss": 2.643, + "theoretical_loss": 3.361833935770057, + "tokens_seen": 2563244032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022713131313131313, + "loss": 2.6419, + "theoretical_loss": 3.3618271138847082, + "tokens_seen": 2563309568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022711111111111112, + "loss": 2.6078, + "theoretical_loss": 3.3618202922226073, + "tokens_seen": 2563375104 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002270909090909091, + "loss": 2.4814, + "theoretical_loss": 3.3618134707837406, + "tokens_seen": 2563440640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022707070707070706, + "loss": 2.4238, + "theoretical_loss": 3.361806649568095, + "tokens_seen": 2563506176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022705050505050507, + "loss": 2.5106, + "theoretical_loss": 3.361799828575658, + "tokens_seen": 2563571712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022703030303030303, + "loss": 2.5746, + "theoretical_loss": 3.361793007806416, + "tokens_seen": 2563637248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022701010101010104, + "loss": 2.6468, + "theoretical_loss": 3.361786187260356, + "tokens_seen": 2563702784 + }, + { + "epoch": 0.55, + "learning_rate": 0.000226989898989899, + "loss": 2.622, + "theoretical_loss": 3.3617793669374656, + "tokens_seen": 2563768320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022696969696969698, + "loss": 2.7918, + "theoretical_loss": 3.361772546837731, + "tokens_seen": 2563833856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022694949494949496, + "loss": 2.5694, + "theoretical_loss": 3.36176572696114, + "tokens_seen": 2563899392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022692929292929292, + "loss": 2.3993, + "theoretical_loss": 3.361758907307679, + "tokens_seen": 2563964928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022690909090909093, + "loss": 2.3548, + "theoretical_loss": 3.361752087877335, + "tokens_seen": 2564030464 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002268888888888889, + "loss": 2.6825, + "theoretical_loss": 3.3617452686700955, + "tokens_seen": 2564096000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022686868686868687, + "loss": 2.6777, + "theoretical_loss": 3.361738449685947, + "tokens_seen": 2564161536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022684848484848485, + "loss": 2.5197, + "theoretical_loss": 3.3617316309248766, + "tokens_seen": 2564227072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022682828282828284, + "loss": 2.4808, + "theoretical_loss": 3.3617248123868713, + "tokens_seen": 2564292608 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1442376, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7244772911071777, + "objective/train/theoretical_loss": 3.3617179940719186, + "objective/train/tokens_used": 923217376, + "theoretical_loss": 3.3617179940719186, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022680808080808082, + "loss": 2.5389, + "theoretical_loss": 3.3617179940719186, + "tokens_seen": 2564358144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002267878787878788, + "loss": 2.4333, + "theoretical_loss": 3.361711175980005, + "tokens_seen": 2564423680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022676767676767676, + "loss": 2.5121, + "theoretical_loss": 3.3617043581111172, + "tokens_seen": 2564489216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022674747474747475, + "loss": 2.6639, + "theoretical_loss": 3.361697540465243, + "tokens_seen": 2564554752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022672727272727273, + "loss": 2.8286, + "theoretical_loss": 3.361690723042369, + "tokens_seen": 2564620288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022670707070707072, + "loss": 2.7181, + "theoretical_loss": 3.361683905842482, + "tokens_seen": 2564685824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266868686868687, + "loss": 2.6628, + "theoretical_loss": 3.3616770888655694, + "tokens_seen": 2564751360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022666666666666666, + "loss": 2.8506, + "theoretical_loss": 3.361670272111618, + "tokens_seen": 2564816896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022664646464646467, + "loss": 2.5608, + "theoretical_loss": 3.3616634555806146, + "tokens_seen": 2564882432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022662626262626262, + "loss": 2.4797, + "theoretical_loss": 3.3616566392725464, + "tokens_seen": 2564947968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002266060606060606, + "loss": 2.4432, + "theoretical_loss": 3.361649823187401, + "tokens_seen": 2565013504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002265858585858586, + "loss": 2.483, + "theoretical_loss": 3.3616430073251644, + "tokens_seen": 2565079040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022656565656565655, + "loss": 2.3746, + "theoretical_loss": 3.3616361916858244, + "tokens_seen": 2565144576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022654545454545456, + "loss": 2.4793, + "theoretical_loss": 3.3616293762693674, + "tokens_seen": 2565210112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022652525252525252, + "loss": 2.7895, + "theoretical_loss": 3.361622561075781, + "tokens_seen": 2565275648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022650505050505053, + "loss": 2.6571, + "theoretical_loss": 3.361615746105052, + "tokens_seen": 2565341184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022648484848484848, + "loss": 2.508, + "theoretical_loss": 3.361608931357167, + "tokens_seen": 2565406720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022646464646464647, + "loss": 2.9038, + "theoretical_loss": 3.3616021168321137, + "tokens_seen": 2565472256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022644444444444445, + "loss": 2.6158, + "theoretical_loss": 3.361595302529879, + "tokens_seen": 2565537792 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002264242424242424, + "loss": 2.6047, + "theoretical_loss": 3.361588488450449, + "tokens_seen": 2565603328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022640404040404042, + "loss": 2.7208, + "theoretical_loss": 3.361581674593812, + "tokens_seen": 2565668864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022638383838383838, + "loss": 2.6639, + "theoretical_loss": 3.3615748609599545, + "tokens_seen": 2565734400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022636363636363636, + "loss": 2.708, + "theoretical_loss": 3.361568047548863, + "tokens_seen": 2565799936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022634343434343435, + "loss": 2.3091, + "theoretical_loss": 3.3615612343605257, + "tokens_seen": 2565865472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022632323232323233, + "loss": 2.6603, + "theoretical_loss": 3.3615544213949287, + "tokens_seen": 2565931008 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1443131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7190606594085693, + "objective/train/theoretical_loss": 3.3615476086520593, + "objective/train/tokens_used": 924855776, + "theoretical_loss": 3.3615476086520593, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002263030303030303, + "loss": 2.6017, + "theoretical_loss": 3.3615476086520593, + "tokens_seen": 2565996544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262828282828283, + "loss": 2.7106, + "theoretical_loss": 3.3615407961319046, + "tokens_seen": 2566062080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022626262626262628, + "loss": 2.6109, + "theoretical_loss": 3.3615339838344513, + "tokens_seen": 2566127616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022624242424242424, + "loss": 2.8658, + "theoretical_loss": 3.361527171759687, + "tokens_seen": 2566193152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022622222222222222, + "loss": 2.5982, + "theoretical_loss": 3.3615203599075985, + "tokens_seen": 2566258688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002262020202020202, + "loss": 2.6677, + "theoretical_loss": 3.361513548278172, + "tokens_seen": 2566324224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002261818181818182, + "loss": 2.6065, + "theoretical_loss": 3.3615067368713962, + "tokens_seen": 2566389760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022616161616161617, + "loss": 2.4703, + "theoretical_loss": 3.361499925687257, + "tokens_seen": 2566455296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022614141414141416, + "loss": 2.4068, + "theoretical_loss": 3.361493114725741, + "tokens_seen": 2566520832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022612121212121211, + "loss": 2.5665, + "theoretical_loss": 3.361486303986837, + "tokens_seen": 2566586368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022610101010101013, + "loss": 2.5659, + "theoretical_loss": 3.36147949347053, + "tokens_seen": 2566651904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022608080808080808, + "loss": 2.5806, + "theoretical_loss": 3.3614726831768085, + "tokens_seen": 2566717440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022606060606060607, + "loss": 2.6193, + "theoretical_loss": 3.361465873105659, + "tokens_seen": 2566782976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022604040404040405, + "loss": 2.5874, + "theoretical_loss": 3.361459063257068, + "tokens_seen": 2566848512 + }, + { + "epoch": 0.55, + "learning_rate": 0.000226020202020202, + "loss": 2.5718, + "theoretical_loss": 3.3614522536310236, + "tokens_seen": 2566914048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022600000000000002, + "loss": 2.5105, + "theoretical_loss": 3.3614454442275123, + "tokens_seen": 2566979584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022597979797979798, + "loss": 2.6682, + "theoretical_loss": 3.3614386350465213, + "tokens_seen": 2567045120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022595959595959599, + "loss": 2.6637, + "theoretical_loss": 3.3614318260880376, + "tokens_seen": 2567110656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022593939393939394, + "loss": 2.6006, + "theoretical_loss": 3.361425017352048, + "tokens_seen": 2567176192 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002259191919191919, + "loss": 2.6031, + "theoretical_loss": 3.36141820883854, + "tokens_seen": 2567241728 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002258989898989899, + "loss": 2.821, + "theoretical_loss": 3.3614114005475004, + "tokens_seen": 2567307264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022587878787878787, + "loss": 2.6129, + "theoretical_loss": 3.361404592478916, + "tokens_seen": 2567372800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022585858585858588, + "loss": 2.4349, + "theoretical_loss": 3.361397784632774, + "tokens_seen": 2567438336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022583838383838384, + "loss": 2.6187, + "theoretical_loss": 3.361390977009062, + "tokens_seen": 2567503872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022581818181818182, + "loss": 2.6632, + "theoretical_loss": 3.361384169607766, + "tokens_seen": 2567569408 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1444461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7009668350219727, + "objective/train/theoretical_loss": 3.3613773624288745, + "objective/train/tokens_used": 926494176, + "theoretical_loss": 3.3613773624288745, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002257979797979798, + "loss": 2.7466, + "theoretical_loss": 3.3613773624288745, + "tokens_seen": 2567634944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002257777777777778, + "loss": 2.6698, + "theoretical_loss": 3.3613705554723734, + "tokens_seen": 2567700480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022575757575757577, + "loss": 2.8067, + "theoretical_loss": 3.36136374873825, + "tokens_seen": 2567766016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022573737373737373, + "loss": 2.8352, + "theoretical_loss": 3.3613569422264913, + "tokens_seen": 2567831552 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002257171717171717, + "loss": 2.6755, + "theoretical_loss": 3.361350135937085, + "tokens_seen": 2567897088 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256969696969697, + "loss": 2.689, + "theoretical_loss": 3.3613433298700173, + "tokens_seen": 2567962624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022567676767676768, + "loss": 2.4634, + "theoretical_loss": 3.3613365240252757, + "tokens_seen": 2568028160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022565656565656566, + "loss": 2.5643, + "theoretical_loss": 3.361329718402847, + "tokens_seen": 2568093696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022563636363636365, + "loss": 2.42, + "theoretical_loss": 3.361322913002719, + "tokens_seen": 2568159232 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002256161616161616, + "loss": 2.7875, + "theoretical_loss": 3.361316107824878, + "tokens_seen": 2568224768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022559595959595962, + "loss": 2.5319, + "theoretical_loss": 3.3613093028693113, + "tokens_seen": 2568290304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022557575757575757, + "loss": 2.4804, + "theoretical_loss": 3.3613024981360056, + "tokens_seen": 2568355840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022555555555555556, + "loss": 2.4664, + "theoretical_loss": 3.3612956936249487, + "tokens_seen": 2568421376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022553535353535354, + "loss": 2.7272, + "theoretical_loss": 3.3612888893361275, + "tokens_seen": 2568486912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002255151515151515, + "loss": 2.6827, + "theoretical_loss": 3.3612820852695284, + "tokens_seen": 2568552448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002254949494949495, + "loss": 2.8003, + "theoretical_loss": 3.361275281425139, + "tokens_seen": 2568617984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022547474747474747, + "loss": 2.7186, + "theoretical_loss": 3.361268477802947, + "tokens_seen": 2568683520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022545454545454548, + "loss": 2.4378, + "theoretical_loss": 3.361261674402938, + "tokens_seen": 2568749056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022543434343434343, + "loss": 2.7237, + "theoretical_loss": 3.3612548712251002, + "tokens_seen": 2568814592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022541414141414144, + "loss": 2.611, + "theoretical_loss": 3.3612480682694206, + "tokens_seen": 2568880128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002253939393939394, + "loss": 2.3423, + "theoretical_loss": 3.3612412655358854, + "tokens_seen": 2568945664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022537373737373736, + "loss": 2.5472, + "theoretical_loss": 3.361234463024483, + "tokens_seen": 2569011200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022535353535353537, + "loss": 2.7505, + "theoretical_loss": 3.3612276607351994, + "tokens_seen": 2569076736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022533333333333333, + "loss": 2.5017, + "theoretical_loss": 3.3612208586680223, + "tokens_seen": 2569142272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022531313131313134, + "loss": 2.7051, + "theoretical_loss": 3.3612140568229383, + "tokens_seen": 2569207808 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1445073, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0313658714294434, + "objective/train/theoretical_loss": 3.3612072551999352, + "objective/train/tokens_used": 928132576, + "theoretical_loss": 3.3612072551999352, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002252929292929293, + "loss": 2.7129, + "theoretical_loss": 3.3612072551999352, + "tokens_seen": 2569273344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022527272727272728, + "loss": 2.6217, + "theoretical_loss": 3.361200453798999, + "tokens_seen": 2569338880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022525252525252526, + "loss": 2.5273, + "theoretical_loss": 3.361193652620118, + "tokens_seen": 2569404416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022523232323232325, + "loss": 2.6365, + "theoretical_loss": 3.361186851663278, + "tokens_seen": 2569469952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022521212121212123, + "loss": 2.6279, + "theoretical_loss": 3.3611800509284677, + "tokens_seen": 2569535488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002251919191919192, + "loss": 2.5409, + "theoretical_loss": 3.3611732504156726, + "tokens_seen": 2569601024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022517171717171717, + "loss": 2.469, + "theoretical_loss": 3.361166450124881, + "tokens_seen": 2569666560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022515151515151515, + "loss": 2.6202, + "theoretical_loss": 3.361159650056079, + "tokens_seen": 2569732096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022513131313131314, + "loss": 2.5134, + "theoretical_loss": 3.361152850209254, + "tokens_seen": 2569797632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022511111111111112, + "loss": 2.5611, + "theoretical_loss": 3.3611460505843933, + "tokens_seen": 2569863168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002250909090909091, + "loss": 2.5348, + "theoretical_loss": 3.361139251181484, + "tokens_seen": 2569928704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022507070707070706, + "loss": 2.5507, + "theoretical_loss": 3.361132452000514, + "tokens_seen": 2569994240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022505050505050507, + "loss": 2.5952, + "theoretical_loss": 3.3611256530414684, + "tokens_seen": 2570059776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022503030303030303, + "loss": 2.628, + "theoretical_loss": 3.361118854304336, + "tokens_seen": 2570125312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022501010101010102, + "loss": 2.4441, + "theoretical_loss": 3.361112055789103, + "tokens_seen": 2570190848 + }, + { + "epoch": 0.55, + "learning_rate": 0.000224989898989899, + "loss": 2.6436, + "theoretical_loss": 3.361105257495757, + "tokens_seen": 2570256384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022496969696969696, + "loss": 2.4865, + "theoretical_loss": 3.361098459424285, + "tokens_seen": 2570321920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022494949494949497, + "loss": 2.6998, + "theoretical_loss": 3.3610916615746738, + "tokens_seen": 2570387456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022492929292929292, + "loss": 2.5809, + "theoretical_loss": 3.361084863946911, + "tokens_seen": 2570452992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022490909090909094, + "loss": 2.5023, + "theoretical_loss": 3.3610780665409834, + "tokens_seen": 2570518528 + }, + { + "epoch": 0.55, + "learning_rate": 0.0002248888888888889, + "loss": 2.5419, + "theoretical_loss": 3.361071269356878, + "tokens_seen": 2570584064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022486868686868685, + "loss": 2.5315, + "theoretical_loss": 3.361064472394582, + "tokens_seen": 2570649600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022484848484848486, + "loss": 2.6447, + "theoretical_loss": 3.3610576756540826, + "tokens_seen": 2570715136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022482828282828282, + "loss": 2.4467, + "theoretical_loss": 3.361050879135367, + "tokens_seen": 2570780672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00022480808080808083, + "loss": 2.5398, + "theoretical_loss": 3.361044082838422, + "tokens_seen": 2570846208 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 1446627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8491482734680176, + "objective/train/theoretical_loss": 3.361037286763235, + "objective/train/tokens_used": 929770976, + "theoretical_loss": 3.361037286763235, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022478787878787878, + "loss": 2.7224, + "theoretical_loss": 3.361037286763235, + "tokens_seen": 2570911744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022476767676767677, + "loss": 2.4579, + "theoretical_loss": 3.361030490909793, + "tokens_seen": 2570977280 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022474747474747475, + "loss": 2.6935, + "theoretical_loss": 3.361023695278083, + "tokens_seen": 2571042816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022472727272727274, + "loss": 2.7349, + "theoretical_loss": 3.3610168998680923, + "tokens_seen": 2571108352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022470707070707072, + "loss": 2.5912, + "theoretical_loss": 3.3610101046798078, + "tokens_seen": 2571173888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022468686868686868, + "loss": 2.5521, + "theoretical_loss": 3.3610033097132166, + "tokens_seen": 2571239424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022466666666666666, + "loss": 2.4564, + "theoretical_loss": 3.3609965149683063, + "tokens_seen": 2571304960 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022464646464646465, + "loss": 2.6869, + "theoretical_loss": 3.3609897204450636, + "tokens_seen": 2571370496 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022462626262626263, + "loss": 2.7986, + "theoretical_loss": 3.3609829261434756, + "tokens_seen": 2571436032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002246060606060606, + "loss": 2.6701, + "theoretical_loss": 3.36097613206353, + "tokens_seen": 2571501568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245858585858586, + "loss": 2.5022, + "theoretical_loss": 3.360969338205213, + "tokens_seen": 2571567104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022456565656565658, + "loss": 2.6675, + "theoretical_loss": 3.3609625445685123, + "tokens_seen": 2571632640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022454545454545457, + "loss": 2.5514, + "theoretical_loss": 3.3609557511534147, + "tokens_seen": 2571698176 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022452525252525252, + "loss": 2.6623, + "theoretical_loss": 3.3609489579599074, + "tokens_seen": 2571763712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002245050505050505, + "loss": 2.6394, + "theoretical_loss": 3.360942164987978, + "tokens_seen": 2571829248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002244848484848485, + "loss": 2.6759, + "theoretical_loss": 3.360935372237613, + "tokens_seen": 2571894784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022446464646464647, + "loss": 2.5771, + "theoretical_loss": 3.3609285797088004, + "tokens_seen": 2571960320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022444444444444446, + "loss": 2.5826, + "theoretical_loss": 3.3609217874015265, + "tokens_seen": 2572025856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022442424242424241, + "loss": 2.5967, + "theoretical_loss": 3.3609149953157784, + "tokens_seen": 2572091392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022440404040404043, + "loss": 2.5335, + "theoretical_loss": 3.360908203451543, + "tokens_seen": 2572156928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022438383838383838, + "loss": 2.6527, + "theoretical_loss": 3.3609014118088085, + "tokens_seen": 2572222464 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002243636363636364, + "loss": 2.5415, + "theoretical_loss": 3.3608946203875614, + "tokens_seen": 2572288000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022434343434343435, + "loss": 2.5228, + "theoretical_loss": 3.360887829187789, + "tokens_seen": 2572353536 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002243232323232323, + "loss": 2.6264, + "theoretical_loss": 3.3608810382094783, + "tokens_seen": 2572419072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022430303030303032, + "loss": 2.4653, + "theoretical_loss": 3.3608742474526165, + "tokens_seen": 2572484608 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1447125, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3464574813842773, + "objective/train/theoretical_loss": 3.36086745691719, + "objective/train/tokens_used": 931409376, + "theoretical_loss": 3.36086745691719, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022428282828282828, + "loss": 2.6554, + "theoretical_loss": 3.36086745691719, + "tokens_seen": 2572550144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002242626262626263, + "loss": 2.5537, + "theoretical_loss": 3.3608606666031875, + "tokens_seen": 2572615680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022424242424242424, + "loss": 2.3952, + "theoretical_loss": 3.360853876510595, + "tokens_seen": 2572681216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022422222222222223, + "loss": 2.5764, + "theoretical_loss": 3.3608470866394, + "tokens_seen": 2572746752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002242020202020202, + "loss": 2.7044, + "theoretical_loss": 3.360840296989589, + "tokens_seen": 2572812288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022418181818181817, + "loss": 2.7526, + "theoretical_loss": 3.3608335075611504, + "tokens_seen": 2572877824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022416161616161618, + "loss": 2.5758, + "theoretical_loss": 3.3608267183540703, + "tokens_seen": 2572943360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022414141414141414, + "loss": 2.491, + "theoretical_loss": 3.3608199293683363, + "tokens_seen": 2573008896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022412121212121212, + "loss": 2.5803, + "theoretical_loss": 3.360813140603935, + "tokens_seen": 2573074432 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002241010101010101, + "loss": 2.8337, + "theoretical_loss": 3.3608063520608544, + "tokens_seen": 2573139968 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002240808080808081, + "loss": 2.5888, + "theoretical_loss": 3.3607995637390813, + "tokens_seen": 2573205504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022406060606060607, + "loss": 2.6203, + "theoretical_loss": 3.3607927756386027, + "tokens_seen": 2573271040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022404040404040406, + "loss": 2.5177, + "theoretical_loss": 3.3607859877594057, + "tokens_seen": 2573336576 + }, + { + "epoch": 0.56, + "learning_rate": 0.000224020202020202, + "loss": 2.5044, + "theoretical_loss": 3.3607792001014776, + "tokens_seen": 2573402112 + }, + { + "epoch": 0.56, + "learning_rate": 0.000224, + "loss": 2.648, + "theoretical_loss": 3.3607724126648058, + "tokens_seen": 2573467648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022397979797979798, + "loss": 2.3524, + "theoretical_loss": 3.360765625449377, + "tokens_seen": 2573533184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022395959595959596, + "loss": 2.3914, + "theoretical_loss": 3.3607588384551788, + "tokens_seen": 2573598720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022393939393939395, + "loss": 2.5394, + "theoretical_loss": 3.3607520516821974, + "tokens_seen": 2573664256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002239191919191919, + "loss": 2.4885, + "theoretical_loss": 3.3607452651304213, + "tokens_seen": 2573729792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022389898989898992, + "loss": 2.669, + "theoretical_loss": 3.3607384787998367, + "tokens_seen": 2573795328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022387878787878787, + "loss": 2.6909, + "theoretical_loss": 3.360731692690431, + "tokens_seen": 2573860864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022385858585858588, + "loss": 2.5746, + "theoretical_loss": 3.360724906802192, + "tokens_seen": 2573926400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022383838383838384, + "loss": 2.5133, + "theoretical_loss": 3.3607181211351054, + "tokens_seen": 2573991936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002238181818181818, + "loss": 2.5961, + "theoretical_loss": 3.36071133568916, + "tokens_seen": 2574057472 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002237979797979798, + "loss": 2.2906, + "theoretical_loss": 3.360704550464342, + "tokens_seen": 2574123008 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1447717, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.865004777908325, + "objective/train/theoretical_loss": 3.3606977654606385, + "objective/train/tokens_used": 933047776, + "theoretical_loss": 3.3606977654606385, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022377777777777777, + "loss": 2.6219, + "theoretical_loss": 3.3606977654606385, + "tokens_seen": 2574188544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022375757575757578, + "loss": 2.594, + "theoretical_loss": 3.360690980678037, + "tokens_seen": 2574254080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022373737373737373, + "loss": 2.5033, + "theoretical_loss": 3.360684196116525, + "tokens_seen": 2574319616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022371717171717172, + "loss": 2.421, + "theoretical_loss": 3.3606774117760887, + "tokens_seen": 2574385152 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002236969696969697, + "loss": 2.5602, + "theoretical_loss": 3.360670627656716, + "tokens_seen": 2574450688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022367676767676769, + "loss": 2.4408, + "theoretical_loss": 3.3606638437583944, + "tokens_seen": 2574516224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022365656565656567, + "loss": 2.542, + "theoretical_loss": 3.36065706008111, + "tokens_seen": 2574581760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022363636363636363, + "loss": 2.7484, + "theoretical_loss": 3.3606502766248507, + "tokens_seen": 2574647296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022361616161616164, + "loss": 2.4591, + "theoretical_loss": 3.360643493389604, + "tokens_seen": 2574712832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002235959595959596, + "loss": 2.6197, + "theoretical_loss": 3.3606367103753563, + "tokens_seen": 2574778368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022357575757575758, + "loss": 2.4862, + "theoretical_loss": 3.360629927582095, + "tokens_seen": 2574843904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022355555555555556, + "loss": 2.5896, + "theoretical_loss": 3.360623145009807, + "tokens_seen": 2574909440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022353535353535355, + "loss": 2.5856, + "theoretical_loss": 3.3606163626584804, + "tokens_seen": 2574974976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022351515151515153, + "loss": 2.6805, + "theoretical_loss": 3.3606095805281018, + "tokens_seen": 2575040512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022349494949494951, + "loss": 2.699, + "theoretical_loss": 3.3606027986186584, + "tokens_seen": 2575106048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022347474747474747, + "loss": 2.4109, + "theoretical_loss": 3.360596016930137, + "tokens_seen": 2575171584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022345454545454546, + "loss": 2.4577, + "theoretical_loss": 3.3605892354625255, + "tokens_seen": 2575237120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022343434343434344, + "loss": 2.6547, + "theoretical_loss": 3.3605824542158107, + "tokens_seen": 2575302656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022341414141414142, + "loss": 2.6097, + "theoretical_loss": 3.3605756731899796, + "tokens_seen": 2575368192 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002233939393939394, + "loss": 2.2339, + "theoretical_loss": 3.3605688923850194, + "tokens_seen": 2575433728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022337373737373736, + "loss": 2.7589, + "theoretical_loss": 3.3605621118009177, + "tokens_seen": 2575499264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022335353535353537, + "loss": 2.3044, + "theoretical_loss": 3.360555331437662, + "tokens_seen": 2575564800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022333333333333333, + "loss": 2.6788, + "theoretical_loss": 3.3605485512952384, + "tokens_seen": 2575630336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022331313131313132, + "loss": 2.5407, + "theoretical_loss": 3.3605417713736347, + "tokens_seen": 2575695872 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002232929292929293, + "loss": 2.4157, + "theoretical_loss": 3.360534991672838, + "tokens_seen": 2575761408 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1449217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.484968423843384, + "objective/train/theoretical_loss": 3.3605282121928357, + "objective/train/tokens_used": 934686176, + "theoretical_loss": 3.3605282121928357, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022327272727272726, + "loss": 2.4656, + "theoretical_loss": 3.3605282121928357, + "tokens_seen": 2575826944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022325252525252527, + "loss": 2.5907, + "theoretical_loss": 3.3605214329336146, + "tokens_seen": 2575892480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022323232323232322, + "loss": 2.5295, + "theoretical_loss": 3.3605146538951627, + "tokens_seen": 2575958016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022321212121212124, + "loss": 2.5071, + "theoretical_loss": 3.360507875077466, + "tokens_seen": 2576023552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002231919191919192, + "loss": 2.5154, + "theoretical_loss": 3.3605010964805127, + "tokens_seen": 2576089088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022317171717171718, + "loss": 2.6657, + "theoretical_loss": 3.3604943181042897, + "tokens_seen": 2576154624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022315151515151516, + "loss": 2.2859, + "theoretical_loss": 3.3604875399487835, + "tokens_seen": 2576220160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022313131313131312, + "loss": 2.4837, + "theoretical_loss": 3.360480762013982, + "tokens_seen": 2576285696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022311111111111113, + "loss": 2.5533, + "theoretical_loss": 3.3604739842998725, + "tokens_seen": 2576351232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022309090909090909, + "loss": 2.5946, + "theoretical_loss": 3.360467206806442, + "tokens_seen": 2576416768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022307070707070707, + "loss": 2.5675, + "theoretical_loss": 3.3604604295336777, + "tokens_seen": 2576482304 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022305050505050505, + "loss": 2.3823, + "theoretical_loss": 3.360453652481567, + "tokens_seen": 2576547840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022303030303030304, + "loss": 2.5481, + "theoretical_loss": 3.360446875650097, + "tokens_seen": 2576613376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022301010101010102, + "loss": 2.4642, + "theoretical_loss": 3.3604400990392547, + "tokens_seen": 2576678912 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222989898989899, + "loss": 2.7264, + "theoretical_loss": 3.360433322649027, + "tokens_seen": 2576744448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022296969696969696, + "loss": 2.4568, + "theoretical_loss": 3.3604265464794016, + "tokens_seen": 2576809984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022294949494949495, + "loss": 2.5663, + "theoretical_loss": 3.360419770530366, + "tokens_seen": 2576875520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022292929292929293, + "loss": 2.5292, + "theoretical_loss": 3.360412994801907, + "tokens_seen": 2576941056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022290909090909091, + "loss": 2.4334, + "theoretical_loss": 3.3604062192940116, + "tokens_seen": 2577006592 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002228888888888889, + "loss": 2.7872, + "theoretical_loss": 3.360399444006667, + "tokens_seen": 2577072128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022286868686868685, + "loss": 2.5015, + "theoretical_loss": 3.3603926689398613, + "tokens_seen": 2577137664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022284848484848487, + "loss": 2.4019, + "theoretical_loss": 3.360385894093581, + "tokens_seen": 2577203200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022282828282828282, + "loss": 2.5493, + "theoretical_loss": 3.360379119467813, + "tokens_seen": 2577268736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022280808080808083, + "loss": 2.6546, + "theoretical_loss": 3.3603723450625456, + "tokens_seen": 2577334272 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002227878787878788, + "loss": 2.6499, + "theoretical_loss": 3.3603655708777644, + "tokens_seen": 2577399808 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1449913, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.645998477935791, + "objective/train/theoretical_loss": 3.3603587969134585, + "objective/train/tokens_used": 936324576, + "theoretical_loss": 3.3603587969134585, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022276767676767677, + "loss": 2.6697, + "theoretical_loss": 3.3603587969134585, + "tokens_seen": 2577465344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022274747474747476, + "loss": 2.5138, + "theoretical_loss": 3.3603520231696136, + "tokens_seen": 2577530880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022272727272727272, + "loss": 2.3667, + "theoretical_loss": 3.3603452496462176, + "tokens_seen": 2577596416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022270707070707073, + "loss": 2.5136, + "theoretical_loss": 3.3603384763432578, + "tokens_seen": 2577661952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022268686868686868, + "loss": 2.5735, + "theoretical_loss": 3.360331703260721, + "tokens_seen": 2577727488 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002226666666666667, + "loss": 2.3576, + "theoretical_loss": 3.3603249303985945, + "tokens_seen": 2577793024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022264646464646465, + "loss": 2.573, + "theoretical_loss": 3.360318157756866, + "tokens_seen": 2577858560 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002226262626262626, + "loss": 2.8539, + "theoretical_loss": 3.3603113853355224, + "tokens_seen": 2577924096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022260606060606062, + "loss": 2.3664, + "theoretical_loss": 3.360304613134551, + "tokens_seen": 2577989632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022258585858585858, + "loss": 2.609, + "theoretical_loss": 3.3602978411539386, + "tokens_seen": 2578055168 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002225656565656566, + "loss": 2.3983, + "theoretical_loss": 3.360291069393673, + "tokens_seen": 2578120704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022254545454545454, + "loss": 2.6169, + "theoretical_loss": 3.3602842978537413, + "tokens_seen": 2578186240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022252525252525253, + "loss": 2.5057, + "theoretical_loss": 3.360277526534131, + "tokens_seen": 2578251776 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002225050505050505, + "loss": 2.3975, + "theoretical_loss": 3.3602707554348283, + "tokens_seen": 2578317312 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224848484848485, + "loss": 2.3824, + "theoretical_loss": 3.360263984555821, + "tokens_seen": 2578382848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022246464646464648, + "loss": 2.6175, + "theoretical_loss": 3.360257213897097, + "tokens_seen": 2578448384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022244444444444444, + "loss": 2.6519, + "theoretical_loss": 3.360250443458643, + "tokens_seen": 2578513920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022242424242424242, + "loss": 2.7027, + "theoretical_loss": 3.360243673240446, + "tokens_seen": 2578579456 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002224040404040404, + "loss": 2.4098, + "theoretical_loss": 3.3602369032424937, + "tokens_seen": 2578644992 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223838383838384, + "loss": 2.4771, + "theoretical_loss": 3.3602301334647726, + "tokens_seen": 2578710528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022236363636363637, + "loss": 2.2204, + "theoretical_loss": 3.360223363907271, + "tokens_seen": 2578776064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022234343434343436, + "loss": 2.4379, + "theoretical_loss": 3.360216594569975, + "tokens_seen": 2578841600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002223232323232323, + "loss": 2.6906, + "theoretical_loss": 3.3602098254528725, + "tokens_seen": 2578907136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022230303030303032, + "loss": 2.5389, + "theoretical_loss": 3.3602030565559513, + "tokens_seen": 2578972672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022228282828282828, + "loss": 2.5421, + "theoretical_loss": 3.360196287879197, + "tokens_seen": 2579038208 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1451054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.580747127532959, + "objective/train/theoretical_loss": 3.360189519422599, + "objective/train/tokens_used": 937962976, + "theoretical_loss": 3.360189519422599, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022226262626262626, + "loss": 2.5739, + "theoretical_loss": 3.360189519422599, + "tokens_seen": 2579103744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022224242424242425, + "loss": 2.7404, + "theoretical_loss": 3.3601827511861426, + "tokens_seen": 2579169280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002222222222222222, + "loss": 2.5623, + "theoretical_loss": 3.3601759831698157, + "tokens_seen": 2579234816 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022220202020202022, + "loss": 2.5412, + "theoretical_loss": 3.3601692153736065, + "tokens_seen": 2579300352 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022218181818181817, + "loss": 2.4018, + "theoretical_loss": 3.3601624477975007, + "tokens_seen": 2579365888 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022216161616161618, + "loss": 2.518, + "theoretical_loss": 3.3601556804414865, + "tokens_seen": 2579431424 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022214141414141414, + "loss": 2.6292, + "theoretical_loss": 3.360148913305551, + "tokens_seen": 2579496960 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022212121212121213, + "loss": 2.3781, + "theoretical_loss": 3.3601421463896815, + "tokens_seen": 2579562496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002221010101010101, + "loss": 2.6753, + "theoretical_loss": 3.3601353796938644, + "tokens_seen": 2579628032 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022208080808080807, + "loss": 2.3896, + "theoretical_loss": 3.3601286132180883, + "tokens_seen": 2579693568 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022206060606060608, + "loss": 2.5222, + "theoretical_loss": 3.36012184696234, + "tokens_seen": 2579759104 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022204040404040403, + "loss": 2.6986, + "theoretical_loss": 3.3601150809266063, + "tokens_seen": 2579824640 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022202020202020202, + "loss": 2.756, + "theoretical_loss": 3.360108315110875, + "tokens_seen": 2579890176 + }, + { + "epoch": 0.56, + "learning_rate": 0.000222, + "loss": 2.5497, + "theoretical_loss": 3.360101549515133, + "tokens_seen": 2579955712 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022197979797979799, + "loss": 2.6312, + "theoretical_loss": 3.360094784139368, + "tokens_seen": 2580021248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022195959595959597, + "loss": 2.5929, + "theoretical_loss": 3.360088018983566, + "tokens_seen": 2580086784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022193939393939395, + "loss": 2.6155, + "theoretical_loss": 3.3600812540477163, + "tokens_seen": 2580152320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022191919191919194, + "loss": 2.4994, + "theoretical_loss": 3.3600744893318044, + "tokens_seen": 2580217856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002218989898989899, + "loss": 2.4933, + "theoretical_loss": 3.3600677248358184, + "tokens_seen": 2580283392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022187878787878788, + "loss": 2.5273, + "theoretical_loss": 3.3600609605597453, + "tokens_seen": 2580348928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022185858585858586, + "loss": 2.7317, + "theoretical_loss": 3.3600541965035724, + "tokens_seen": 2580414464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022183838383838385, + "loss": 2.5312, + "theoretical_loss": 3.360047432667287, + "tokens_seen": 2580480000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022181818181818183, + "loss": 2.5563, + "theoretical_loss": 3.3600406690508766, + "tokens_seen": 2580545536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022179797979797981, + "loss": 2.6197, + "theoretical_loss": 3.360033905654328, + "tokens_seen": 2580611072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022177777777777777, + "loss": 2.4196, + "theoretical_loss": 3.360027142477629, + "tokens_seen": 2580676608 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1451835, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7778964042663574, + "objective/train/theoretical_loss": 3.3600203795207664, + "objective/train/tokens_used": 939601376, + "theoretical_loss": 3.3600203795207664, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022175757575757576, + "loss": 2.5595, + "theoretical_loss": 3.3600203795207664, + "tokens_seen": 2580742144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022173737373737374, + "loss": 2.3671, + "theoretical_loss": 3.360013616783728, + "tokens_seen": 2580807680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022171717171717172, + "loss": 2.5664, + "theoretical_loss": 3.3600068542665005, + "tokens_seen": 2580873216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002216969696969697, + "loss": 2.6773, + "theoretical_loss": 3.3600000919690713, + "tokens_seen": 2580938752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022167676767676766, + "loss": 2.6465, + "theoretical_loss": 3.3599933298914277, + "tokens_seen": 2581004288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022165656565656568, + "loss": 2.6073, + "theoretical_loss": 3.359986568033557, + "tokens_seen": 2581069824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022163636363636363, + "loss": 2.5658, + "theoretical_loss": 3.3599798063954474, + "tokens_seen": 2581135360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022161616161616164, + "loss": 2.5917, + "theoretical_loss": 3.3599730449770844, + "tokens_seen": 2581200896 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002215959595959596, + "loss": 2.715, + "theoretical_loss": 3.3599662837784567, + "tokens_seen": 2581266432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022157575757575756, + "loss": 2.6146, + "theoretical_loss": 3.359959522799551, + "tokens_seen": 2581331968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022155555555555557, + "loss": 2.6441, + "theoretical_loss": 3.359952762040354, + "tokens_seen": 2581397504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022153535353535352, + "loss": 2.6613, + "theoretical_loss": 3.359946001500855, + "tokens_seen": 2581463040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022151515151515154, + "loss": 2.4067, + "theoretical_loss": 3.359939241181039, + "tokens_seen": 2581528576 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002214949494949495, + "loss": 2.5068, + "theoretical_loss": 3.3599324810808944, + "tokens_seen": 2581594112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022147474747474748, + "loss": 2.6141, + "theoretical_loss": 3.3599257212004083, + "tokens_seen": 2581659648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022145454545454546, + "loss": 2.358, + "theoretical_loss": 3.3599189615395675, + "tokens_seen": 2581725184 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022143434343434344, + "loss": 2.6173, + "theoretical_loss": 3.3599122020983607, + "tokens_seen": 2581790720 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022141414141414143, + "loss": 2.5244, + "theoretical_loss": 3.359905442876774, + "tokens_seen": 2581856256 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022139393939393939, + "loss": 2.4464, + "theoretical_loss": 3.3598986838747944, + "tokens_seen": 2581921792 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022137373737373737, + "loss": 2.5473, + "theoretical_loss": 3.35989192509241, + "tokens_seen": 2581987328 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022135353535353535, + "loss": 2.4485, + "theoretical_loss": 3.359885166529608, + "tokens_seen": 2582052864 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022133333333333334, + "loss": 2.4965, + "theoretical_loss": 3.3598784081863755, + "tokens_seen": 2582118400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022131313131313132, + "loss": 2.3844, + "theoretical_loss": 3.3598716500626997, + "tokens_seen": 2582183936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002212929292929293, + "loss": 2.643, + "theoretical_loss": 3.359864892158568, + "tokens_seen": 2582249472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022127272727272726, + "loss": 2.5671, + "theoretical_loss": 3.359858134473968, + "tokens_seen": 2582315008 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1453267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.432158946990967, + "objective/train/theoretical_loss": 3.3598513770088863, + "objective/train/tokens_used": 941239776, + "theoretical_loss": 3.3598513770088863, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022125252525252527, + "loss": 2.573, + "theoretical_loss": 3.3598513770088863, + "tokens_seen": 2582380544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022123232323232323, + "loss": 2.4338, + "theoretical_loss": 3.3598446197633107, + "tokens_seen": 2582446080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022121212121212121, + "loss": 2.7476, + "theoretical_loss": 3.3598378627372285, + "tokens_seen": 2582511616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002211919191919192, + "loss": 2.5367, + "theoretical_loss": 3.359831105930627, + "tokens_seen": 2582577152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022117171717171715, + "loss": 2.4126, + "theoretical_loss": 3.3598243493434934, + "tokens_seen": 2582642688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022115151515151517, + "loss": 2.6352, + "theoretical_loss": 3.359817592975815, + "tokens_seen": 2582708224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022113131313131312, + "loss": 2.6057, + "theoretical_loss": 3.359810836827579, + "tokens_seen": 2582773760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022111111111111113, + "loss": 2.5156, + "theoretical_loss": 3.359804080898773, + "tokens_seen": 2582839296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002210909090909091, + "loss": 2.6003, + "theoretical_loss": 3.3597973251893842, + "tokens_seen": 2582904832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002210707070707071, + "loss": 2.7287, + "theoretical_loss": 3.3597905696993995, + "tokens_seen": 2582970368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022105050505050506, + "loss": 2.3682, + "theoretical_loss": 3.359783814428807, + "tokens_seen": 2583035904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022103030303030302, + "loss": 2.7233, + "theoretical_loss": 3.359777059377593, + "tokens_seen": 2583101440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022101010101010103, + "loss": 2.5854, + "theoretical_loss": 3.3597703045457457, + "tokens_seen": 2583166976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022098989898989898, + "loss": 2.5875, + "theoretical_loss": 3.359763549933252, + "tokens_seen": 2583232512 + }, + { + "epoch": 0.56, + "learning_rate": 0.000220969696969697, + "loss": 2.669, + "theoretical_loss": 3.359756795540099, + "tokens_seen": 2583298048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022094949494949495, + "loss": 2.487, + "theoretical_loss": 3.359750041366275, + "tokens_seen": 2583363584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022092929292929294, + "loss": 2.5622, + "theoretical_loss": 3.359743287411766, + "tokens_seen": 2583429120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022090909090909092, + "loss": 2.6865, + "theoretical_loss": 3.35973653367656, + "tokens_seen": 2583494656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022088888888888888, + "loss": 2.4892, + "theoretical_loss": 3.3597297801606443, + "tokens_seen": 2583560192 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208686868686869, + "loss": 2.505, + "theoretical_loss": 3.3597230268640064, + "tokens_seen": 2583625728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022084848484848484, + "loss": 2.5454, + "theoretical_loss": 3.359716273786633, + "tokens_seen": 2583691264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022082828282828283, + "loss": 2.4544, + "theoretical_loss": 3.3597095209285124, + "tokens_seen": 2583756800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002208080808080808, + "loss": 2.4332, + "theoretical_loss": 3.3597027682896305, + "tokens_seen": 2583822336 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002207878787878788, + "loss": 2.5523, + "theoretical_loss": 3.359696015869976, + "tokens_seen": 2583887872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022076767676767678, + "loss": 2.5362, + "theoretical_loss": 3.3596892636695355, + "tokens_seen": 2583953408 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1453825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4089794158935547, + "objective/train/theoretical_loss": 3.3596825116882965, + "objective/train/tokens_used": 942878176, + "theoretical_loss": 3.3596825116882965, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022074747474747476, + "loss": 2.5401, + "theoretical_loss": 3.3596825116882965, + "tokens_seen": 2584018944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022072727272727272, + "loss": 2.6035, + "theoretical_loss": 3.3596757599262466, + "tokens_seen": 2584084480 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002207070707070707, + "loss": 2.6368, + "theoretical_loss": 3.3596690083833725, + "tokens_seen": 2584150016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002206868686868687, + "loss": 2.4476, + "theoretical_loss": 3.3596622570596617, + "tokens_seen": 2584215552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022066666666666667, + "loss": 2.6349, + "theoretical_loss": 3.3596555059551023, + "tokens_seen": 2584281088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022064646464646466, + "loss": 2.4827, + "theoretical_loss": 3.3596487550696805, + "tokens_seen": 2584346624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002206262626262626, + "loss": 2.5675, + "theoretical_loss": 3.3596420044033843, + "tokens_seen": 2584412160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022060606060606062, + "loss": 2.6863, + "theoretical_loss": 3.359635253956201, + "tokens_seen": 2584477696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022058585858585858, + "loss": 2.4859, + "theoretical_loss": 3.3596285037281177, + "tokens_seen": 2584543232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002205656565656566, + "loss": 2.3699, + "theoretical_loss": 3.3596217537191215, + "tokens_seen": 2584608768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022054545454545455, + "loss": 2.4782, + "theoretical_loss": 3.3596150039292008, + "tokens_seen": 2584674304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002205252525252525, + "loss": 2.4177, + "theoretical_loss": 3.359608254358342, + "tokens_seen": 2584739840 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022050505050505052, + "loss": 2.4675, + "theoretical_loss": 3.359601505006532, + "tokens_seen": 2584805376 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022048484848484847, + "loss": 2.6855, + "theoretical_loss": 3.3595947558737596, + "tokens_seen": 2584870912 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022046464646464649, + "loss": 2.5629, + "theoretical_loss": 3.359588006960011, + "tokens_seen": 2584936448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022044444444444444, + "loss": 2.5985, + "theoretical_loss": 3.359581258265274, + "tokens_seen": 2585001984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022042424242424243, + "loss": 2.5058, + "theoretical_loss": 3.3595745097895358, + "tokens_seen": 2585067520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002204040404040404, + "loss": 2.7875, + "theoretical_loss": 3.3595677615327837, + "tokens_seen": 2585133056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002203838383838384, + "loss": 2.4612, + "theoretical_loss": 3.359561013495005, + "tokens_seen": 2585198592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022036363636363638, + "loss": 2.6073, + "theoretical_loss": 3.359554265676187, + "tokens_seen": 2585264128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022034343434343433, + "loss": 2.4992, + "theoretical_loss": 3.3595475180763175, + "tokens_seen": 2585329664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022032323232323232, + "loss": 2.4493, + "theoretical_loss": 3.3595407706953835, + "tokens_seen": 2585395200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002203030303030303, + "loss": 2.5959, + "theoretical_loss": 3.3595340235333726, + "tokens_seen": 2585460736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002202828282828283, + "loss": 2.6291, + "theoretical_loss": 3.3595272765902715, + "tokens_seen": 2585526272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022026262626262627, + "loss": 2.7049, + "theoretical_loss": 3.3595205298660678, + "tokens_seen": 2585591808 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1454456, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.608703851699829, + "objective/train/theoretical_loss": 3.3595137833607494, + "objective/train/tokens_used": 944516576, + "theoretical_loss": 3.3595137833607494, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022024242424242425, + "loss": 2.6907, + "theoretical_loss": 3.3595137833607494, + "tokens_seen": 2585657344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022022222222222224, + "loss": 2.5564, + "theoretical_loss": 3.3595070370743034, + "tokens_seen": 2585722880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022020202020202022, + "loss": 2.6806, + "theoretical_loss": 3.3595002910067167, + "tokens_seen": 2585788416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022018181818181818, + "loss": 2.6951, + "theoretical_loss": 3.359493545157977, + "tokens_seen": 2585853952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022016161616161616, + "loss": 2.3779, + "theoretical_loss": 3.3594867995280717, + "tokens_seen": 2585919488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022014141414141415, + "loss": 2.7251, + "theoretical_loss": 3.3594800541169882, + "tokens_seen": 2585985024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022012121212121213, + "loss": 2.7228, + "theoretical_loss": 3.3594733089247137, + "tokens_seen": 2586050560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022010101010101012, + "loss": 2.4311, + "theoretical_loss": 3.3594665639512358, + "tokens_seen": 2586116096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022008080808080807, + "loss": 2.6744, + "theoretical_loss": 3.359459819196541, + "tokens_seen": 2586181632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022006060606060608, + "loss": 2.5692, + "theoretical_loss": 3.359453074660618, + "tokens_seen": 2586247168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022004040404040404, + "loss": 2.7079, + "theoretical_loss": 3.359446330343453, + "tokens_seen": 2586312704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022002020202020202, + "loss": 2.3541, + "theoretical_loss": 3.3594395862450344, + "tokens_seen": 2586378240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00022, + "loss": 2.4668, + "theoretical_loss": 3.3594328423653486, + "tokens_seen": 2586443776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021997979797979796, + "loss": 2.4758, + "theoretical_loss": 3.3594260987043834, + "tokens_seen": 2586509312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021995959595959598, + "loss": 2.46, + "theoretical_loss": 3.3594193552621263, + "tokens_seen": 2586574848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021993939393939393, + "loss": 2.5069, + "theoretical_loss": 3.3594126120385646, + "tokens_seen": 2586640384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021991919191919194, + "loss": 2.5794, + "theoretical_loss": 3.359405869033685, + "tokens_seen": 2586705920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002198989898989899, + "loss": 2.6016, + "theoretical_loss": 3.359399126247476, + "tokens_seen": 2586771456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021987878787878788, + "loss": 2.3811, + "theoretical_loss": 3.359392383679924, + "tokens_seen": 2586836992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021985858585858587, + "loss": 2.7408, + "theoretical_loss": 3.3593856413310172, + "tokens_seen": 2586902528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021983838383838383, + "loss": 2.3938, + "theoretical_loss": 3.3593788992007423, + "tokens_seen": 2586968064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021981818181818184, + "loss": 2.4581, + "theoretical_loss": 3.359372157289087, + "tokens_seen": 2587033600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0002197979797979798, + "loss": 2.5603, + "theoretical_loss": 3.3593654155960384, + "tokens_seen": 2587099136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021977777777777778, + "loss": 2.497, + "theoretical_loss": 3.359358674121584, + "tokens_seen": 2587164672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021975757575757576, + "loss": 2.4421, + "theoretical_loss": 3.3593519328657115, + "tokens_seen": 2587230208 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 1455631, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0374765396118164, + "objective/train/theoretical_loss": 3.359345191828408, + "objective/train/tokens_used": 946154976, + "theoretical_loss": 3.359345191828408, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.56, + "learning_rate": 0.00021973737373737375, + "loss": 2.7143, + "theoretical_loss": 3.359345191828408, + "tokens_seen": 2587295744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021971717171717173, + "loss": 2.5898, + "theoretical_loss": 3.359338451009661, + "tokens_seen": 2587361280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002196969696969697, + "loss": 2.4928, + "theoretical_loss": 3.3593317104094575, + "tokens_seen": 2587426816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021967676767676767, + "loss": 2.5271, + "theoretical_loss": 3.359324970027785, + "tokens_seen": 2587492352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021965656565656565, + "loss": 2.6254, + "theoretical_loss": 3.359318229864632, + "tokens_seen": 2587557888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021963636363636364, + "loss": 2.4582, + "theoretical_loss": 3.3593114899199836, + "tokens_seen": 2587623424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021961616161616162, + "loss": 2.2874, + "theoretical_loss": 3.3593047501938296, + "tokens_seen": 2587688960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002195959595959596, + "loss": 2.735, + "theoretical_loss": 3.359298010686156, + "tokens_seen": 2587754496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021957575757575756, + "loss": 2.3291, + "theoretical_loss": 3.35929127139695, + "tokens_seen": 2587820032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021955555555555557, + "loss": 2.7543, + "theoretical_loss": 3.3592845323261997, + "tokens_seen": 2587885568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021953535353535353, + "loss": 2.5383, + "theoretical_loss": 3.3592777934738924, + "tokens_seen": 2587951104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021951515151515154, + "loss": 2.6521, + "theoretical_loss": 3.359271054840015, + "tokens_seen": 2588016640 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002194949494949495, + "loss": 2.438, + "theoretical_loss": 3.3592643164245555, + "tokens_seen": 2588082176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021947474747474746, + "loss": 2.6958, + "theoretical_loss": 3.359257578227501, + "tokens_seen": 2588147712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021945454545454547, + "loss": 2.6025, + "theoretical_loss": 3.3592508402488384, + "tokens_seen": 2588213248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021943434343434342, + "loss": 2.529, + "theoretical_loss": 3.3592441024885558, + "tokens_seen": 2588278784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021941414141414143, + "loss": 2.5159, + "theoretical_loss": 3.3592373649466407, + "tokens_seen": 2588344320 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193939393939394, + "loss": 2.7353, + "theoretical_loss": 3.35923062762308, + "tokens_seen": 2588409856 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002193737373737374, + "loss": 2.694, + "theoretical_loss": 3.3592238905178613, + "tokens_seen": 2588475392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021935353535353536, + "loss": 2.6115, + "theoretical_loss": 3.359217153630972, + "tokens_seen": 2588540928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021933333333333332, + "loss": 2.7074, + "theoretical_loss": 3.3592104169623997, + "tokens_seen": 2588606464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021931313131313133, + "loss": 2.6111, + "theoretical_loss": 3.359203680512131, + "tokens_seen": 2588672000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021929292929292928, + "loss": 2.3475, + "theoretical_loss": 3.3591969442801544, + "tokens_seen": 2588737536 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002192727272727273, + "loss": 2.5087, + "theoretical_loss": 3.3591902082664564, + "tokens_seen": 2588803072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021925252525252525, + "loss": 2.4563, + "theoretical_loss": 3.359183472471025, + "tokens_seen": 2588868608 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1456190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.123682975769043, + "objective/train/theoretical_loss": 3.359176736893847, + "objective/train/tokens_used": 947793376, + "theoretical_loss": 3.359176736893847, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021923232323232324, + "loss": 2.3819, + "theoretical_loss": 3.359176736893847, + "tokens_seen": 2588934144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021921212121212122, + "loss": 2.3381, + "theoretical_loss": 3.3591700015349106, + "tokens_seen": 2588999680 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002191919191919192, + "loss": 2.7286, + "theoretical_loss": 3.3591632663942024, + "tokens_seen": 2589065216 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002191717171717172, + "loss": 2.5139, + "theoretical_loss": 3.3591565314717107, + "tokens_seen": 2589130752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021915151515151514, + "loss": 2.5029, + "theoretical_loss": 3.3591497967674218, + "tokens_seen": 2589196288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021913131313131313, + "loss": 2.6144, + "theoretical_loss": 3.3591430622813236, + "tokens_seen": 2589261824 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002191111111111111, + "loss": 2.4518, + "theoretical_loss": 3.3591363280134043, + "tokens_seen": 2589327360 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002190909090909091, + "loss": 2.5751, + "theoretical_loss": 3.35912959396365, + "tokens_seen": 2589392896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021907070707070708, + "loss": 2.4873, + "theoretical_loss": 3.3591228601320493, + "tokens_seen": 2589458432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021905050505050506, + "loss": 2.6142, + "theoretical_loss": 3.3591161265185887, + "tokens_seen": 2589523968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021903030303030302, + "loss": 2.3618, + "theoretical_loss": 3.359109393123256, + "tokens_seen": 2589589504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021901010101010103, + "loss": 2.6202, + "theoretical_loss": 3.3591026599460383, + "tokens_seen": 2589655040 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218989898989899, + "loss": 2.5666, + "theoretical_loss": 3.3590959269869236, + "tokens_seen": 2589720576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021896969696969697, + "loss": 2.5556, + "theoretical_loss": 3.359089194245899, + "tokens_seen": 2589786112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021894949494949496, + "loss": 2.607, + "theoretical_loss": 3.3590824617229513, + "tokens_seen": 2589851648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021892929292929291, + "loss": 2.5351, + "theoretical_loss": 3.3590757294180693, + "tokens_seen": 2589917184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021890909090909092, + "loss": 2.5513, + "theoretical_loss": 3.3590689973312395, + "tokens_seen": 2589982720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021888888888888888, + "loss": 2.526, + "theoretical_loss": 3.359062265462449, + "tokens_seen": 2590048256 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002188686868686869, + "loss": 2.6585, + "theoretical_loss": 3.359055533811686, + "tokens_seen": 2590113792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021884848484848485, + "loss": 2.512, + "theoretical_loss": 3.359048802378938, + "tokens_seen": 2590179328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021882828282828283, + "loss": 2.5487, + "theoretical_loss": 3.3590420711641915, + "tokens_seen": 2590244864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021880808080808082, + "loss": 2.6533, + "theoretical_loss": 3.3590353401674347, + "tokens_seen": 2590310400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021878787878787877, + "loss": 2.7055, + "theoretical_loss": 3.3590286093886546, + "tokens_seen": 2590375936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021876767676767679, + "loss": 2.4625, + "theoretical_loss": 3.359021878827839, + "tokens_seen": 2590441472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021874747474747474, + "loss": 2.8149, + "theoretical_loss": 3.359015148484975, + "tokens_seen": 2590507008 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1457612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.963740825653076, + "objective/train/theoretical_loss": 3.35900841836005, + "objective/train/tokens_used": 949431776, + "theoretical_loss": 3.35900841836005, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021872727272727273, + "loss": 2.7217, + "theoretical_loss": 3.35900841836005, + "tokens_seen": 2590572544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002187070707070707, + "loss": 2.6391, + "theoretical_loss": 3.359001688453052, + "tokens_seen": 2590638080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186868686868687, + "loss": 2.544, + "theoretical_loss": 3.358994958763968, + "tokens_seen": 2590703616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021866666666666668, + "loss": 2.618, + "theoretical_loss": 3.3589882292927853, + "tokens_seen": 2590769152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021864646464646466, + "loss": 2.3878, + "theoretical_loss": 3.3589815000394916, + "tokens_seen": 2590834688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021862626262626262, + "loss": 2.4346, + "theoretical_loss": 3.358974771004074, + "tokens_seen": 2590900224 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002186060606060606, + "loss": 2.4728, + "theoretical_loss": 3.35896804218652, + "tokens_seen": 2590965760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002185858585858586, + "loss": 2.4083, + "theoretical_loss": 3.3589613135868177, + "tokens_seen": 2591031296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021856565656565657, + "loss": 2.5961, + "theoretical_loss": 3.3589545852049536, + "tokens_seen": 2591096832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021854545454545455, + "loss": 2.5379, + "theoretical_loss": 3.358947857040916, + "tokens_seen": 2591162368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021852525252525254, + "loss": 2.5866, + "theoretical_loss": 3.3589411290946916, + "tokens_seen": 2591227904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021850505050505052, + "loss": 2.5915, + "theoretical_loss": 3.3589344013662683, + "tokens_seen": 2591293440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021848484848484848, + "loss": 2.4473, + "theoretical_loss": 3.358927673855633, + "tokens_seen": 2591358976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021846464646464646, + "loss": 2.6338, + "theoretical_loss": 3.3589209465627743, + "tokens_seen": 2591424512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021844444444444445, + "loss": 2.6702, + "theoretical_loss": 3.358914219487678, + "tokens_seen": 2591490048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021842424242424243, + "loss": 2.5901, + "theoretical_loss": 3.3589074926303333, + "tokens_seen": 2591555584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021840404040404042, + "loss": 2.4603, + "theoretical_loss": 3.358900765990726, + "tokens_seen": 2591621120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021838383838383837, + "loss": 2.6178, + "theoretical_loss": 3.358894039568845, + "tokens_seen": 2591686656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021836363636363638, + "loss": 2.4512, + "theoretical_loss": 3.3588873133646766, + "tokens_seen": 2591752192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021834343434343434, + "loss": 2.5769, + "theoretical_loss": 3.358880587378209, + "tokens_seen": 2591817728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021832323232323235, + "loss": 2.6731, + "theoretical_loss": 3.358873861609429, + "tokens_seen": 2591883264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002183030303030303, + "loss": 2.5008, + "theoretical_loss": 3.3588671360583247, + "tokens_seen": 2591948800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021828282828282827, + "loss": 2.6254, + "theoretical_loss": 3.358860410724883, + "tokens_seen": 2592014336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021826262626262628, + "loss": 2.6637, + "theoretical_loss": 3.358853685609092, + "tokens_seen": 2592079872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021824242424242423, + "loss": 2.5653, + "theoretical_loss": 3.358846960710938, + "tokens_seen": 2592145408 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1458174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4110045433044434, + "objective/train/theoretical_loss": 3.35884023603041, + "objective/train/tokens_used": 951070176, + "theoretical_loss": 3.35884023603041, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021822222222222224, + "loss": 2.458, + "theoretical_loss": 3.35884023603041, + "tokens_seen": 2592210944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002182020202020202, + "loss": 2.4369, + "theoretical_loss": 3.3588335115674943, + "tokens_seen": 2592276480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021818181818181818, + "loss": 2.7497, + "theoretical_loss": 3.358826787322179, + "tokens_seen": 2592342016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021816161616161617, + "loss": 2.3239, + "theoretical_loss": 3.358820063294451, + "tokens_seen": 2592407552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021814141414141415, + "loss": 2.5348, + "theoretical_loss": 3.3588133394842985, + "tokens_seen": 2592473088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021812121212121214, + "loss": 2.6352, + "theoretical_loss": 3.358806615891708, + "tokens_seen": 2592538624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002181010101010101, + "loss": 2.3332, + "theoretical_loss": 3.3587998925166676, + "tokens_seen": 2592604160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021808080808080808, + "loss": 2.6935, + "theoretical_loss": 3.358793169359165, + "tokens_seen": 2592669696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021806060606060606, + "loss": 2.498, + "theoretical_loss": 3.3587864464191868, + "tokens_seen": 2592735232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021804040404040405, + "loss": 2.4093, + "theoretical_loss": 3.358779723696721, + "tokens_seen": 2592800768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021802020202020203, + "loss": 2.5511, + "theoretical_loss": 3.3587730011917554, + "tokens_seen": 2592866304 + }, + { + "epoch": 0.57, + "learning_rate": 0.000218, + "loss": 2.494, + "theoretical_loss": 3.3587662789042767, + "tokens_seen": 2592931840 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021797979797979797, + "loss": 2.5501, + "theoretical_loss": 3.358759556834273, + "tokens_seen": 2592997376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021795959595959598, + "loss": 2.6462, + "theoretical_loss": 3.3587528349817313, + "tokens_seen": 2593062912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021793939393939394, + "loss": 2.6482, + "theoretical_loss": 3.3587461133466396, + "tokens_seen": 2593128448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021791919191919192, + "loss": 2.555, + "theoretical_loss": 3.358739391928985, + "tokens_seen": 2593193984 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002178989898989899, + "loss": 2.6946, + "theoretical_loss": 3.358732670728755, + "tokens_seen": 2593259520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021787878787878786, + "loss": 2.5992, + "theoretical_loss": 3.3587259497459367, + "tokens_seen": 2593325056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021785858585858587, + "loss": 2.8015, + "theoretical_loss": 3.358719228980519, + "tokens_seen": 2593390592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021783838383838383, + "loss": 2.5769, + "theoretical_loss": 3.3587125084324874, + "tokens_seen": 2593456128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021781818181818184, + "loss": 2.5486, + "theoretical_loss": 3.3587057881018305, + "tokens_seen": 2593521664 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002177979797979798, + "loss": 2.4591, + "theoretical_loss": 3.358699067988536, + "tokens_seen": 2593587200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021777777777777776, + "loss": 2.6515, + "theoretical_loss": 3.3586923480925908, + "tokens_seen": 2593652736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021775757575757577, + "loss": 2.6734, + "theoretical_loss": 3.358685628413982, + "tokens_seen": 2593718272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021773737373737372, + "loss": 2.443, + "theoretical_loss": 3.3586789089526983, + "tokens_seen": 2593783808 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1459433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8205974102020264, + "objective/train/theoretical_loss": 3.3586721897087264, + "objective/train/tokens_used": 952708576, + "theoretical_loss": 3.3586721897087264, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021771717171717173, + "loss": 2.742, + "theoretical_loss": 3.3586721897087264, + "tokens_seen": 2593849344 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002176969696969697, + "loss": 2.5261, + "theoretical_loss": 3.358665470682054, + "tokens_seen": 2593914880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021767676767676768, + "loss": 2.5252, + "theoretical_loss": 3.3586587518726683, + "tokens_seen": 2593980416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021765656565656566, + "loss": 2.6206, + "theoretical_loss": 3.358652033280557, + "tokens_seen": 2594045952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021763636363636364, + "loss": 2.688, + "theoretical_loss": 3.3586453149057074, + "tokens_seen": 2594111488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021761616161616163, + "loss": 2.5204, + "theoretical_loss": 3.3586385967481074, + "tokens_seen": 2594177024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021759595959595958, + "loss": 2.6933, + "theoretical_loss": 3.3586318788077443, + "tokens_seen": 2594242560 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002175757575757576, + "loss": 2.4494, + "theoretical_loss": 3.358625161084605, + "tokens_seen": 2594308096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021755555555555555, + "loss": 2.486, + "theoretical_loss": 3.358618443578678, + "tokens_seen": 2594373632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021753535353535354, + "loss": 2.6665, + "theoretical_loss": 3.3586117262899506, + "tokens_seen": 2594439168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021751515151515152, + "loss": 2.6142, + "theoretical_loss": 3.3586050092184094, + "tokens_seen": 2594504704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174949494949495, + "loss": 2.6319, + "theoretical_loss": 3.358598292364043, + "tokens_seen": 2594570240 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174747474747475, + "loss": 2.5072, + "theoretical_loss": 3.3585915757268374, + "tokens_seen": 2594635776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021745454545454547, + "loss": 2.5568, + "theoretical_loss": 3.3585848593067817, + "tokens_seen": 2594701312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021743434343434343, + "loss": 2.7704, + "theoretical_loss": 3.358578143103863, + "tokens_seen": 2594766848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002174141414141414, + "loss": 2.6865, + "theoretical_loss": 3.3585714271180684, + "tokens_seen": 2594832384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002173939393939394, + "loss": 2.3679, + "theoretical_loss": 3.3585647113493855, + "tokens_seen": 2594897920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021737373737373738, + "loss": 2.5792, + "theoretical_loss": 3.358557995797802, + "tokens_seen": 2594963456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021735353535353536, + "loss": 2.4241, + "theoretical_loss": 3.358551280463305, + "tokens_seen": 2595028992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021733333333333332, + "loss": 2.5528, + "theoretical_loss": 3.3585445653458823, + "tokens_seen": 2595094528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021731313131313133, + "loss": 2.6751, + "theoretical_loss": 3.3585378504455212, + "tokens_seen": 2595160064 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172929292929293, + "loss": 3.0009, + "theoretical_loss": 3.3585311357622096, + "tokens_seen": 2595225600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002172727272727273, + "loss": 2.5603, + "theoretical_loss": 3.3585244212959346, + "tokens_seen": 2595291136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021725252525252526, + "loss": 2.6912, + "theoretical_loss": 3.3585177070466843, + "tokens_seen": 2595356672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021723232323232321, + "loss": 2.7464, + "theoretical_loss": 3.3585109930144452, + "tokens_seen": 2595422208 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1459948, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.741441488265991, + "objective/train/theoretical_loss": 3.358504279199206, + "objective/train/tokens_used": 954346976, + "theoretical_loss": 3.358504279199206, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021721212121212123, + "loss": 2.702, + "theoretical_loss": 3.358504279199206, + "tokens_seen": 2595487744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021719191919191918, + "loss": 2.5492, + "theoretical_loss": 3.358497565600953, + "tokens_seen": 2595553280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002171717171717172, + "loss": 2.5536, + "theoretical_loss": 3.358490852219674, + "tokens_seen": 2595618816 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021715151515151515, + "loss": 2.571, + "theoretical_loss": 3.3584841390553577, + "tokens_seen": 2595684352 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021713131313131313, + "loss": 2.6935, + "theoretical_loss": 3.3584774261079904, + "tokens_seen": 2595749888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021711111111111112, + "loss": 2.789, + "theoretical_loss": 3.3584707133775598, + "tokens_seen": 2595815424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002170909090909091, + "loss": 2.5566, + "theoretical_loss": 3.3584640008640534, + "tokens_seen": 2595880960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021707070707070709, + "loss": 2.75, + "theoretical_loss": 3.3584572885674593, + "tokens_seen": 2595946496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021705050505050504, + "loss": 2.7775, + "theoretical_loss": 3.358450576487764, + "tokens_seen": 2596012032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021703030303030303, + "loss": 2.684, + "theoretical_loss": 3.358443864624956, + "tokens_seen": 2596077568 + }, + { + "epoch": 0.57, + "learning_rate": 0.000217010101010101, + "loss": 2.5521, + "theoretical_loss": 3.3584371529790222, + "tokens_seen": 2596143104 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216989898989899, + "loss": 2.6785, + "theoretical_loss": 3.3584304415499506, + "tokens_seen": 2596208640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021696969696969698, + "loss": 2.7715, + "theoretical_loss": 3.3584237303377282, + "tokens_seen": 2596274176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021694949494949496, + "loss": 2.3802, + "theoretical_loss": 3.3584170193423426, + "tokens_seen": 2596339712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021692929292929292, + "loss": 2.5266, + "theoretical_loss": 3.3584103085637818, + "tokens_seen": 2596405248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021690909090909093, + "loss": 2.5672, + "theoretical_loss": 3.358403598002033, + "tokens_seen": 2596470784 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002168888888888889, + "loss": 2.7345, + "theoretical_loss": 3.3583968876570833, + "tokens_seen": 2596536320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021686868686868687, + "loss": 2.7377, + "theoretical_loss": 3.3583901775289213, + "tokens_seen": 2596601856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021684848484848486, + "loss": 2.5713, + "theoretical_loss": 3.3583834676175335, + "tokens_seen": 2596667392 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002168282828282828, + "loss": 2.7975, + "theoretical_loss": 3.358376757922908, + "tokens_seen": 2596732928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021680808080808082, + "loss": 2.4246, + "theoretical_loss": 3.358370048445032, + "tokens_seen": 2596798464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021678787878787878, + "loss": 2.4586, + "theoretical_loss": 3.3583633391838927, + "tokens_seen": 2596864000 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002167676767676768, + "loss": 2.6395, + "theoretical_loss": 3.3583566301394785, + "tokens_seen": 2596929536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021674747474747475, + "loss": 2.5416, + "theoretical_loss": 3.3583499213117767, + "tokens_seen": 2596995072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021672727272727273, + "loss": 2.4292, + "theoretical_loss": 3.358343212700774, + "tokens_seen": 2597060608 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1461323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.994370698928833, + "objective/train/theoretical_loss": 3.3583365043064592, + "objective/train/tokens_used": 955985376, + "theoretical_loss": 3.3583365043064592, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021670707070707072, + "loss": 2.6462, + "theoretical_loss": 3.3583365043064592, + "tokens_seen": 2597126144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021668686868686867, + "loss": 2.6868, + "theoretical_loss": 3.358329796128819, + "tokens_seen": 2597191680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021666666666666668, + "loss": 2.5112, + "theoretical_loss": 3.3583230881678414, + "tokens_seen": 2597257216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021664646464646464, + "loss": 2.7314, + "theoretical_loss": 3.358316380423513, + "tokens_seen": 2597322752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021662626262626265, + "loss": 2.4965, + "theoretical_loss": 3.358309672895823, + "tokens_seen": 2597388288 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002166060606060606, + "loss": 2.7299, + "theoretical_loss": 3.358302965584757, + "tokens_seen": 2597453824 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002165858585858586, + "loss": 2.4831, + "theoretical_loss": 3.358296258490304, + "tokens_seen": 2597519360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021656565656565658, + "loss": 2.317, + "theoretical_loss": 3.358289551612451, + "tokens_seen": 2597584896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021654545454545453, + "loss": 2.5714, + "theoretical_loss": 3.3582828449511855, + "tokens_seen": 2597650432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021652525252525254, + "loss": 2.4602, + "theoretical_loss": 3.358276138506495, + "tokens_seen": 2597715968 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002165050505050505, + "loss": 2.4915, + "theoretical_loss": 3.3582694322783673, + "tokens_seen": 2597781504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021648484848484849, + "loss": 2.4791, + "theoretical_loss": 3.3582627262667897, + "tokens_seen": 2597847040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021646464646464647, + "loss": 2.3651, + "theoretical_loss": 3.35825602047175, + "tokens_seen": 2597912576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021644444444444445, + "loss": 2.5206, + "theoretical_loss": 3.3582493148932357, + "tokens_seen": 2597978112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021642424242424244, + "loss": 2.695, + "theoretical_loss": 3.358242609531234, + "tokens_seen": 2598043648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021640404040404042, + "loss": 2.5509, + "theoretical_loss": 3.3582359043857326, + "tokens_seen": 2598109184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021638383838383838, + "loss": 2.8706, + "theoretical_loss": 3.3582291994567193, + "tokens_seen": 2598174720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021636363636363636, + "loss": 2.5825, + "theoretical_loss": 3.3582224947441817, + "tokens_seen": 2598240256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021634343434343435, + "loss": 2.7108, + "theoretical_loss": 3.3582157902481065, + "tokens_seen": 2598305792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021632323232323233, + "loss": 2.6891, + "theoretical_loss": 3.3582090859684826, + "tokens_seen": 2598371328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021630303030303031, + "loss": 2.6725, + "theoretical_loss": 3.358202381905296, + "tokens_seen": 2598436864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021628282828282827, + "loss": 2.4976, + "theoretical_loss": 3.358195678058536, + "tokens_seen": 2598502400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021626262626262628, + "loss": 2.4662, + "theoretical_loss": 3.358188974428189, + "tokens_seen": 2598567936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021624242424242424, + "loss": 2.6624, + "theoretical_loss": 3.3581822710142424, + "tokens_seen": 2598633472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021622222222222225, + "loss": 2.4851, + "theoretical_loss": 3.3581755678166845, + "tokens_seen": 2598699008 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1462170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.566434144973755, + "objective/train/theoretical_loss": 3.358168864835503, + "objective/train/tokens_used": 957623776, + "theoretical_loss": 3.358168864835503, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002162020202020202, + "loss": 2.6232, + "theoretical_loss": 3.358168864835503, + "tokens_seen": 2598764544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021618181818181816, + "loss": 2.4553, + "theoretical_loss": 3.358162162070684, + "tokens_seen": 2598830080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021616161616161617, + "loss": 2.4023, + "theoretical_loss": 3.3581554595222167, + "tokens_seen": 2598895616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021614141414141413, + "loss": 2.5073, + "theoretical_loss": 3.3581487571900874, + "tokens_seen": 2598961152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021612121212121214, + "loss": 2.7168, + "theoretical_loss": 3.3581420550742846, + "tokens_seen": 2599026688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002161010101010101, + "loss": 2.4907, + "theoretical_loss": 3.358135353174796, + "tokens_seen": 2599092224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021608080808080808, + "loss": 2.4689, + "theoretical_loss": 3.358128651491608, + "tokens_seen": 2599157760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021606060606060607, + "loss": 2.763, + "theoretical_loss": 3.3581219500247093, + "tokens_seen": 2599223296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021604040404040402, + "loss": 2.6053, + "theoretical_loss": 3.358115248774087, + "tokens_seen": 2599288832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021602020202020204, + "loss": 2.6781, + "theoretical_loss": 3.3581085477397283, + "tokens_seen": 2599354368 + }, + { + "epoch": 0.57, + "learning_rate": 0.000216, + "loss": 2.7183, + "theoretical_loss": 3.3581018469216213, + "tokens_seen": 2599419904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021597979797979798, + "loss": 2.4916, + "theoretical_loss": 3.3580951463197537, + "tokens_seen": 2599485440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021595959595959596, + "loss": 2.6881, + "theoretical_loss": 3.3580884459341123, + "tokens_seen": 2599550976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021593939393939394, + "loss": 2.5707, + "theoretical_loss": 3.358081745764686, + "tokens_seen": 2599616512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021591919191919193, + "loss": 2.392, + "theoretical_loss": 3.3580750458114608, + "tokens_seen": 2599682048 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158989898989899, + "loss": 2.6594, + "theoretical_loss": 3.3580683460744254, + "tokens_seen": 2599747584 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002158787878787879, + "loss": 2.8039, + "theoretical_loss": 3.3580616465535664, + "tokens_seen": 2599813120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021585858585858585, + "loss": 2.6148, + "theoretical_loss": 3.3580549472488728, + "tokens_seen": 2599878656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021583838383838384, + "loss": 2.5502, + "theoretical_loss": 3.3580482481603307, + "tokens_seen": 2599944192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021581818181818182, + "loss": 2.5751, + "theoretical_loss": 3.358041549287929, + "tokens_seen": 2600009728 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157979797979798, + "loss": 2.5016, + "theoretical_loss": 3.358034850631654, + "tokens_seen": 2600075264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002157777777777778, + "loss": 2.4322, + "theoretical_loss": 3.358028152191494, + "tokens_seen": 2600140800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021575757575757577, + "loss": 2.5245, + "theoretical_loss": 3.3580214539674365, + "tokens_seen": 2600206336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021573737373737373, + "loss": 2.6831, + "theoretical_loss": 3.358014755959469, + "tokens_seen": 2600271872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021571717171717174, + "loss": 2.7061, + "theoretical_loss": 3.358008058167579, + "tokens_seen": 2600337408 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1463432, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2096810340881348, + "objective/train/theoretical_loss": 3.3580013605917545, + "objective/train/tokens_used": 959262176, + "theoretical_loss": 3.3580013605917545, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002156969696969697, + "loss": 2.5766, + "theoretical_loss": 3.3580013605917545, + "tokens_seen": 2600402944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021567676767676768, + "loss": 2.6149, + "theoretical_loss": 3.357994663231983, + "tokens_seen": 2600468480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021565656565656567, + "loss": 2.5317, + "theoretical_loss": 3.3579879660882512, + "tokens_seen": 2600534016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021563636363636362, + "loss": 2.6935, + "theoretical_loss": 3.3579812691605477, + "tokens_seen": 2600599552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021561616161616163, + "loss": 2.6155, + "theoretical_loss": 3.35797457244886, + "tokens_seen": 2600665088 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155959595959596, + "loss": 2.639, + "theoretical_loss": 3.3579678759531753, + "tokens_seen": 2600730624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002155757575757576, + "loss": 2.5653, + "theoretical_loss": 3.357961179673481, + "tokens_seen": 2600796160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021555555555555556, + "loss": 2.5488, + "theoretical_loss": 3.357954483609765, + "tokens_seen": 2600861696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021553535353535354, + "loss": 2.5489, + "theoretical_loss": 3.3579477877620154, + "tokens_seen": 2600927232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021551515151515153, + "loss": 2.6794, + "theoretical_loss": 3.3579410921302193, + "tokens_seen": 2600992768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021549494949494948, + "loss": 2.8566, + "theoretical_loss": 3.3579343967143642, + "tokens_seen": 2601058304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002154747474747475, + "loss": 2.613, + "theoretical_loss": 3.3579277015144378, + "tokens_seen": 2601123840 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021545454545454545, + "loss": 2.6274, + "theoretical_loss": 3.3579210065304275, + "tokens_seen": 2601189376 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021543434343434343, + "loss": 2.7238, + "theoretical_loss": 3.357914311762321, + "tokens_seen": 2601254912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021541414141414142, + "loss": 2.7354, + "theoretical_loss": 3.357907617210106, + "tokens_seen": 2601320448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002153939393939394, + "loss": 2.851, + "theoretical_loss": 3.3579009228737706, + "tokens_seen": 2601385984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021537373737373739, + "loss": 2.8603, + "theoretical_loss": 3.3578942287533016, + "tokens_seen": 2601451520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021535353535353537, + "loss": 2.7078, + "theoretical_loss": 3.3578875348486865, + "tokens_seen": 2601517056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021533333333333333, + "loss": 2.4595, + "theoretical_loss": 3.357880841159914, + "tokens_seen": 2601582592 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002153131313131313, + "loss": 2.4586, + "theoretical_loss": 3.3578741476869705, + "tokens_seen": 2601648128 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002152929292929293, + "loss": 2.5911, + "theoretical_loss": 3.357867454429844, + "tokens_seen": 2601713664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021527272727272728, + "loss": 2.4685, + "theoretical_loss": 3.3578607613885225, + "tokens_seen": 2601779200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021525252525252526, + "loss": 2.6212, + "theoretical_loss": 3.357854068562993, + "tokens_seen": 2601844736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021523232323232322, + "loss": 2.5048, + "theoretical_loss": 3.357847375953244, + "tokens_seen": 2601910272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021521212121212123, + "loss": 2.4812, + "theoretical_loss": 3.357840683559262, + "tokens_seen": 2601975808 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1464290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8142123222351074, + "objective/train/theoretical_loss": 3.3578339913810353, + "objective/train/tokens_used": 960900576, + "theoretical_loss": 3.3578339913810353, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151919191919192, + "loss": 2.6272, + "theoretical_loss": 3.3578339913810353, + "tokens_seen": 2602041344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021517171717171717, + "loss": 2.482, + "theoretical_loss": 3.3578272994185516, + "tokens_seen": 2602106880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021515151515151516, + "loss": 2.6025, + "theoretical_loss": 3.357820607671798, + "tokens_seen": 2602172416 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002151313131313131, + "loss": 2.5925, + "theoretical_loss": 3.3578139161407625, + "tokens_seen": 2602237952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021511111111111112, + "loss": 2.3742, + "theoretical_loss": 3.3578072248254327, + "tokens_seen": 2602303488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021509090909090908, + "loss": 2.697, + "theoretical_loss": 3.3578005337257957, + "tokens_seen": 2602369024 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002150707070707071, + "loss": 2.4962, + "theoretical_loss": 3.35779384284184, + "tokens_seen": 2602434560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021505050505050505, + "loss": 2.4979, + "theoretical_loss": 3.357787152173552, + "tokens_seen": 2602500096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021503030303030306, + "loss": 2.6275, + "theoretical_loss": 3.3577804617209206, + "tokens_seen": 2602565632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021501010101010102, + "loss": 2.8784, + "theoretical_loss": 3.3577737714839326, + "tokens_seen": 2602631168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021498989898989897, + "loss": 2.5864, + "theoretical_loss": 3.357767081462576, + "tokens_seen": 2602696704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021496969696969698, + "loss": 2.5144, + "theoretical_loss": 3.357760391656839, + "tokens_seen": 2602762240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021494949494949494, + "loss": 2.6646, + "theoretical_loss": 3.3577537020667076, + "tokens_seen": 2602827776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021492929292929295, + "loss": 2.693, + "theoretical_loss": 3.3577470126921707, + "tokens_seen": 2602893312 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002149090909090909, + "loss": 2.6023, + "theoretical_loss": 3.3577403235332155, + "tokens_seen": 2602958848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148888888888889, + "loss": 2.5357, + "theoretical_loss": 3.3577336345898297, + "tokens_seen": 2603024384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021486868686868688, + "loss": 2.504, + "theoretical_loss": 3.3577269458620007, + "tokens_seen": 2603089920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021484848484848486, + "loss": 2.5999, + "theoretical_loss": 3.357720257349717, + "tokens_seen": 2603155456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021482828282828284, + "loss": 2.8736, + "theoretical_loss": 3.357713569052965, + "tokens_seen": 2603220992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0002148080808080808, + "loss": 2.5419, + "theoretical_loss": 3.357706880971733, + "tokens_seen": 2603286528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021478787878787879, + "loss": 2.6566, + "theoretical_loss": 3.357700193106009, + "tokens_seen": 2603352064 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021476767676767677, + "loss": 2.5796, + "theoretical_loss": 3.3576935054557797, + "tokens_seen": 2603417600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021474747474747475, + "loss": 2.5752, + "theoretical_loss": 3.3576868180210333, + "tokens_seen": 2603483136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021472727272727274, + "loss": 2.7178, + "theoretical_loss": 3.357680130801757, + "tokens_seen": 2603548672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021470707070707072, + "loss": 2.3581, + "theoretical_loss": 3.3576734437979394, + "tokens_seen": 2603614208 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 1465580, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.97364342212677, + "objective/train/theoretical_loss": 3.357666757009567, + "objective/train/tokens_used": 962538976, + "theoretical_loss": 3.357666757009567, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00021468686868686868, + "loss": 2.4235, + "theoretical_loss": 3.357666757009567, + "tokens_seen": 2603679744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002146666666666667, + "loss": 2.5937, + "theoretical_loss": 3.357660070436628, + "tokens_seen": 2603745280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021464646464646465, + "loss": 2.5635, + "theoretical_loss": 3.3576533840791107, + "tokens_seen": 2603810816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021462626262626263, + "loss": 2.8888, + "theoretical_loss": 3.3576466979370014, + "tokens_seen": 2603876352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021460606060606061, + "loss": 2.4843, + "theoretical_loss": 3.3576400120102883, + "tokens_seen": 2603941888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021458585858585857, + "loss": 2.5585, + "theoretical_loss": 3.3576333262989593, + "tokens_seen": 2604007424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021456565656565658, + "loss": 2.7581, + "theoretical_loss": 3.3576266408030015, + "tokens_seen": 2604072960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021454545454545454, + "loss": 2.4747, + "theoretical_loss": 3.3576199555224036, + "tokens_seen": 2604138496 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021452525252525255, + "loss": 2.7235, + "theoretical_loss": 3.357613270457152, + "tokens_seen": 2604204032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002145050505050505, + "loss": 2.6275, + "theoretical_loss": 3.357606585607235, + "tokens_seen": 2604269568 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021448484848484846, + "loss": 2.5424, + "theoretical_loss": 3.35759990097264, + "tokens_seen": 2604335104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021446464646464647, + "loss": 2.6491, + "theoretical_loss": 3.3575932165533544, + "tokens_seen": 2604400640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021444444444444443, + "loss": 2.7668, + "theoretical_loss": 3.3575865323493668, + "tokens_seen": 2604466176 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021442424242424244, + "loss": 2.6518, + "theoretical_loss": 3.357579848360664, + "tokens_seen": 2604531712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002144040404040404, + "loss": 2.4887, + "theoretical_loss": 3.357573164587234, + "tokens_seen": 2604597248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021438383838383838, + "loss": 2.8693, + "theoretical_loss": 3.3575664810290644, + "tokens_seen": 2604662784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021436363636363637, + "loss": 2.4962, + "theoretical_loss": 3.3575597976861427, + "tokens_seen": 2604728320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021434343434343435, + "loss": 2.4814, + "theoretical_loss": 3.3575531145584567, + "tokens_seen": 2604793856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021432323232323234, + "loss": 2.5565, + "theoretical_loss": 3.3575464316459938, + "tokens_seen": 2604859392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002143030303030303, + "loss": 2.641, + "theoretical_loss": 3.357539748948742, + "tokens_seen": 2604924928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021428282828282828, + "loss": 2.7086, + "theoretical_loss": 3.357533066466689, + "tokens_seen": 2604990464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021426262626262626, + "loss": 2.3234, + "theoretical_loss": 3.357526384199822, + "tokens_seen": 2605056000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021424242424242424, + "loss": 2.6459, + "theoretical_loss": 3.3575197021481293, + "tokens_seen": 2605121536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021422222222222223, + "loss": 2.5519, + "theoretical_loss": 3.357513020311598, + "tokens_seen": 2605187072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002142020202020202, + "loss": 2.4098, + "theoretical_loss": 3.357506338690216, + "tokens_seen": 2605252608 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1466226, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.593043088912964, + "objective/train/theoretical_loss": 3.3574996572839706, + "objective/train/tokens_used": 964177376, + "theoretical_loss": 3.3574996572839706, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141818181818182, + "loss": 2.5618, + "theoretical_loss": 3.3574996572839706, + "tokens_seen": 2605318144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021416161616161618, + "loss": 2.7578, + "theoretical_loss": 3.3574929760928502, + "tokens_seen": 2605383680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021414141414141414, + "loss": 2.5828, + "theoretical_loss": 3.357486295116842, + "tokens_seen": 2605449216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021412121212121212, + "loss": 2.7187, + "theoretical_loss": 3.3574796143559333, + "tokens_seen": 2605514752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002141010101010101, + "loss": 2.5564, + "theoretical_loss": 3.3574729338101124, + "tokens_seen": 2605580288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002140808080808081, + "loss": 2.5208, + "theoretical_loss": 3.357466253479367, + "tokens_seen": 2605645824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021406060606060607, + "loss": 2.593, + "theoretical_loss": 3.357459573363684, + "tokens_seen": 2605711360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021404040404040403, + "loss": 2.8921, + "theoretical_loss": 3.357452893463052, + "tokens_seen": 2605776896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021402020202020204, + "loss": 2.4093, + "theoretical_loss": 3.357446213777458, + "tokens_seen": 2605842432 + }, + { + "epoch": 0.58, + "learning_rate": 0.000214, + "loss": 2.7918, + "theoretical_loss": 3.3574395343068897, + "tokens_seen": 2605907968 + }, + { + "epoch": 0.58, + "learning_rate": 0.000213979797979798, + "loss": 2.7517, + "theoretical_loss": 3.3574328550513353, + "tokens_seen": 2605973504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021395959595959597, + "loss": 2.5071, + "theoretical_loss": 3.357426176010782, + "tokens_seen": 2606039040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021393939393939392, + "loss": 2.5583, + "theoretical_loss": 3.3574194971852176, + "tokens_seen": 2606104576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021391919191919193, + "loss": 2.5909, + "theoretical_loss": 3.35741281857463, + "tokens_seen": 2606170112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002138989898989899, + "loss": 2.611, + "theoretical_loss": 3.3574061401790067, + "tokens_seen": 2606235648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002138787878787879, + "loss": 2.5691, + "theoretical_loss": 3.357399461998335, + "tokens_seen": 2606301184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021385858585858586, + "loss": 2.4184, + "theoretical_loss": 3.357392784032603, + "tokens_seen": 2606366720 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021383838383838384, + "loss": 2.5072, + "theoretical_loss": 3.3573861062817985, + "tokens_seen": 2606432256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021381818181818183, + "loss": 2.733, + "theoretical_loss": 3.357379428745909, + "tokens_seen": 2606497792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137979797979798, + "loss": 2.3973, + "theoretical_loss": 3.357372751424922, + "tokens_seen": 2606563328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002137777777777778, + "loss": 2.3472, + "theoretical_loss": 3.357366074318825, + "tokens_seen": 2606628864 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021375757575757575, + "loss": 2.4958, + "theoretical_loss": 3.3573593974276066, + "tokens_seen": 2606694400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021373737373737373, + "loss": 2.3468, + "theoretical_loss": 3.357352720751254, + "tokens_seen": 2606759936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021371717171717172, + "loss": 2.7144, + "theoretical_loss": 3.3573460442897542, + "tokens_seen": 2606825472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136969696969697, + "loss": 2.7818, + "theoretical_loss": 3.3573393680430956, + "tokens_seen": 2606891008 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1467591, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.599229335784912, + "objective/train/theoretical_loss": 3.357332692011266, + "objective/train/tokens_used": 965815776, + "theoretical_loss": 3.357332692011266, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002136767676767677, + "loss": 2.8561, + "theoretical_loss": 3.357332692011266, + "tokens_seen": 2606956544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021365656565656567, + "loss": 2.6343, + "theoretical_loss": 3.3573260161942526, + "tokens_seen": 2607022080 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021363636363636363, + "loss": 2.6763, + "theoretical_loss": 3.357319340592044, + "tokens_seen": 2607087616 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021361616161616164, + "loss": 2.6321, + "theoretical_loss": 3.3573126652046263, + "tokens_seen": 2607153152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002135959595959596, + "loss": 2.6763, + "theoretical_loss": 3.3573059900319886, + "tokens_seen": 2607218688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021357575757575758, + "loss": 2.7739, + "theoretical_loss": 3.357299315074118, + "tokens_seen": 2607284224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021355555555555556, + "loss": 2.5787, + "theoretical_loss": 3.357292640331002, + "tokens_seen": 2607349760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021353535353535352, + "loss": 2.6454, + "theoretical_loss": 3.357285965802629, + "tokens_seen": 2607415296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021351515151515153, + "loss": 2.7018, + "theoretical_loss": 3.357279291488986, + "tokens_seen": 2607480832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134949494949495, + "loss": 2.6875, + "theoretical_loss": 3.3572726173900613, + "tokens_seen": 2607546368 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134747474747475, + "loss": 2.5466, + "theoretical_loss": 3.3572659435058423, + "tokens_seen": 2607611904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021345454545454546, + "loss": 2.3637, + "theoretical_loss": 3.3572592698363164, + "tokens_seen": 2607677440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002134343434343434, + "loss": 2.759, + "theoretical_loss": 3.3572525963814717, + "tokens_seen": 2607742976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021341414141414142, + "loss": 2.3774, + "theoretical_loss": 3.3572459231412957, + "tokens_seen": 2607808512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021339393939393938, + "loss": 2.5663, + "theoretical_loss": 3.357239250115776, + "tokens_seen": 2607874048 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002133737373737374, + "loss": 2.7621, + "theoretical_loss": 3.3572325773049005, + "tokens_seen": 2607939584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021335353535353535, + "loss": 2.5613, + "theoretical_loss": 3.357225904708657, + "tokens_seen": 2608005120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021333333333333336, + "loss": 2.5525, + "theoretical_loss": 3.3572192323270333, + "tokens_seen": 2608070656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021331313131313132, + "loss": 2.5163, + "theoretical_loss": 3.3572125601600167, + "tokens_seen": 2608136192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132929292929293, + "loss": 2.4875, + "theoretical_loss": 3.357205888207595, + "tokens_seen": 2608201728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021327272727272728, + "loss": 2.5926, + "theoretical_loss": 3.357199216469756, + "tokens_seen": 2608267264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021325252525252524, + "loss": 2.6803, + "theoretical_loss": 3.3571925449464874, + "tokens_seen": 2608332800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021323232323232325, + "loss": 2.7728, + "theoretical_loss": 3.3571858736377767, + "tokens_seen": 2608398336 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002132121212121212, + "loss": 2.8872, + "theoretical_loss": 3.357179202543612, + "tokens_seen": 2608463872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002131919191919192, + "loss": 2.7421, + "theoretical_loss": 3.3571725316639807, + "tokens_seen": 2608529408 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1468389, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5926337242126465, + "objective/train/theoretical_loss": 3.357165860998871, + "objective/train/tokens_used": 967454176, + "theoretical_loss": 3.357165860998871, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021317171717171718, + "loss": 2.5656, + "theoretical_loss": 3.357165860998871, + "tokens_seen": 2608594944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021315151515151516, + "loss": 2.5753, + "theoretical_loss": 3.35715919054827, + "tokens_seen": 2608660480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021313131313131315, + "loss": 2.6419, + "theoretical_loss": 3.3571525203121655, + "tokens_seen": 2608726016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021311111111111113, + "loss": 2.6619, + "theoretical_loss": 3.3571458502905456, + "tokens_seen": 2608791552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021309090909090909, + "loss": 2.7206, + "theoretical_loss": 3.3571391804833977, + "tokens_seen": 2608857088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021307070707070707, + "loss": 2.4329, + "theoretical_loss": 3.3571325108907097, + "tokens_seen": 2608922624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021305050505050505, + "loss": 2.6645, + "theoretical_loss": 3.357125841512469, + "tokens_seen": 2608988160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021303030303030304, + "loss": 2.5366, + "theoretical_loss": 3.3571191723486637, + "tokens_seen": 2609053696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021301010101010102, + "loss": 2.3977, + "theoretical_loss": 3.3571125033992812, + "tokens_seen": 2609119232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021298989898989898, + "loss": 2.6939, + "theoretical_loss": 3.35710583466431, + "tokens_seen": 2609184768 + }, + { + "epoch": 0.58, + "learning_rate": 0.000212969696969697, + "loss": 2.8692, + "theoretical_loss": 3.357099166143737, + "tokens_seen": 2609250304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021294949494949495, + "loss": 2.4807, + "theoretical_loss": 3.3570924978375496, + "tokens_seen": 2609315840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021292929292929296, + "loss": 2.2968, + "theoretical_loss": 3.357085829745736, + "tokens_seen": 2609381376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021290909090909091, + "loss": 2.5508, + "theoretical_loss": 3.3570791618682843, + "tokens_seen": 2609446912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021288888888888887, + "loss": 2.65, + "theoretical_loss": 3.357072494205182, + "tokens_seen": 2609512448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021286868686868688, + "loss": 2.521, + "theoretical_loss": 3.3570658267564166, + "tokens_seen": 2609577984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021284848484848484, + "loss": 2.6472, + "theoretical_loss": 3.357059159521976, + "tokens_seen": 2609643520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021282828282828285, + "loss": 2.4437, + "theoretical_loss": 3.3570524925018477, + "tokens_seen": 2609709056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002128080808080808, + "loss": 2.78, + "theoretical_loss": 3.35704582569602, + "tokens_seen": 2609774592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127878787878788, + "loss": 2.4616, + "theoretical_loss": 3.3570391591044797, + "tokens_seen": 2609840128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021276767676767678, + "loss": 2.5299, + "theoretical_loss": 3.3570324927272157, + "tokens_seen": 2609905664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021274747474747473, + "loss": 2.7343, + "theoretical_loss": 3.3570258265642146, + "tokens_seen": 2609971200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021272727272727274, + "loss": 2.5403, + "theoretical_loss": 3.3570191606154647, + "tokens_seen": 2610036736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002127070707070707, + "loss": 2.6616, + "theoretical_loss": 3.3570124948809537, + "tokens_seen": 2610102272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021268686868686868, + "loss": 2.7336, + "theoretical_loss": 3.3570058293606695, + "tokens_seen": 2610167808 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1468957, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2268729209899902, + "objective/train/theoretical_loss": 3.3569991640545993, + "objective/train/tokens_used": 969092576, + "theoretical_loss": 3.3569991640545993, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021266666666666667, + "loss": 2.6221, + "theoretical_loss": 3.3569991640545993, + "tokens_seen": 2610233344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021264646464646465, + "loss": 2.7317, + "theoretical_loss": 3.3569924989627316, + "tokens_seen": 2610298880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021262626262626264, + "loss": 2.4412, + "theoretical_loss": 3.3569858340850534, + "tokens_seen": 2610364416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021260606060606062, + "loss": 2.3619, + "theoretical_loss": 3.3569791694215527, + "tokens_seen": 2610429952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021258585858585858, + "loss": 2.4371, + "theoretical_loss": 3.3569725049722177, + "tokens_seen": 2610495488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021256565656565656, + "loss": 2.4864, + "theoretical_loss": 3.3569658407370353, + "tokens_seen": 2610561024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021254545454545454, + "loss": 2.7456, + "theoretical_loss": 3.356959176715994, + "tokens_seen": 2610626560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021252525252525253, + "loss": 2.3399, + "theoretical_loss": 3.356952512909081, + "tokens_seen": 2610692096 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002125050505050505, + "loss": 2.3947, + "theoretical_loss": 3.3569458493162845, + "tokens_seen": 2610757632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124848484848485, + "loss": 2.3622, + "theoretical_loss": 3.356939185937592, + "tokens_seen": 2610823168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021246464646464648, + "loss": 2.5418, + "theoretical_loss": 3.356932522772991, + "tokens_seen": 2610888704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021244444444444444, + "loss": 2.7218, + "theoretical_loss": 3.3569258598224696, + "tokens_seen": 2610954240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021242424242424245, + "loss": 2.425, + "theoretical_loss": 3.3569191970860155, + "tokens_seen": 2611019776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002124040404040404, + "loss": 2.3927, + "theoretical_loss": 3.356912534563617, + "tokens_seen": 2611085312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002123838383838384, + "loss": 2.5613, + "theoretical_loss": 3.3569058722552603, + "tokens_seen": 2611150848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021236363636363637, + "loss": 2.6997, + "theoretical_loss": 3.3568992101609347, + "tokens_seen": 2611216384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021234343434343433, + "loss": 2.444, + "theoretical_loss": 3.3568925482806273, + "tokens_seen": 2611281920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021232323232323234, + "loss": 2.6472, + "theoretical_loss": 3.3568858866143256, + "tokens_seen": 2611347456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002123030303030303, + "loss": 2.3515, + "theoretical_loss": 3.356879225162018, + "tokens_seen": 2611412992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002122828282828283, + "loss": 2.4683, + "theoretical_loss": 3.356872563923692, + "tokens_seen": 2611478528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021226262626262627, + "loss": 2.6502, + "theoretical_loss": 3.356865902899335, + "tokens_seen": 2611544064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021224242424242425, + "loss": 2.5473, + "theoretical_loss": 3.356859242088935, + "tokens_seen": 2611609600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021222222222222223, + "loss": 2.4654, + "theoretical_loss": 3.35685258149248, + "tokens_seen": 2611675136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002122020202020202, + "loss": 2.5304, + "theoretical_loss": 3.3568459211099575, + "tokens_seen": 2611740672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121818181818182, + "loss": 2.3895, + "theoretical_loss": 3.356839260941355, + "tokens_seen": 2611806208 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1469548, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.637343406677246, + "objective/train/theoretical_loss": 3.356832600986661, + "objective/train/tokens_used": 970730976, + "theoretical_loss": 3.356832600986661, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021216161616161616, + "loss": 2.2675, + "theoretical_loss": 3.356832600986661, + "tokens_seen": 2611871744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021214141414141414, + "loss": 2.4182, + "theoretical_loss": 3.356825941245863, + "tokens_seen": 2611937280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021212121212121213, + "loss": 2.4305, + "theoretical_loss": 3.3568192817189484, + "tokens_seen": 2612002816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002121010101010101, + "loss": 2.5834, + "theoretical_loss": 3.356812622405905, + "tokens_seen": 2612068352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002120808080808081, + "loss": 2.3573, + "theoretical_loss": 3.356805963306721, + "tokens_seen": 2612133888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021206060606060608, + "loss": 2.495, + "theoretical_loss": 3.3567993044213837, + "tokens_seen": 2612199424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021204040404040404, + "loss": 2.4073, + "theoretical_loss": 3.3567926457498816, + "tokens_seen": 2612264960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021202020202020202, + "loss": 2.4205, + "theoretical_loss": 3.3567859872922012, + "tokens_seen": 2612330496 + }, + { + "epoch": 0.58, + "learning_rate": 0.000212, + "loss": 2.5273, + "theoretical_loss": 3.356779329048331, + "tokens_seen": 2612396032 + }, + { + "epoch": 0.58, + "learning_rate": 0.000211979797979798, + "loss": 2.2564, + "theoretical_loss": 3.3567726710182595, + "tokens_seen": 2612461568 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021195959595959597, + "loss": 2.5664, + "theoretical_loss": 3.356766013201973, + "tokens_seen": 2612527104 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021193939393939393, + "loss": 2.4545, + "theoretical_loss": 3.356759355599461, + "tokens_seen": 2612592640 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021191919191919194, + "loss": 2.4935, + "theoretical_loss": 3.35675269821071, + "tokens_seen": 2612658176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002118989898989899, + "loss": 2.3047, + "theoretical_loss": 3.3567460410357075, + "tokens_seen": 2612723712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021187878787878788, + "loss": 2.5916, + "theoretical_loss": 3.356739384074442, + "tokens_seen": 2612789248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021185858585858586, + "loss": 2.6664, + "theoretical_loss": 3.3567327273269014, + "tokens_seen": 2612854784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021183838383838382, + "loss": 2.6316, + "theoretical_loss": 3.356726070793073, + "tokens_seen": 2612920320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021181818181818183, + "loss": 2.4231, + "theoretical_loss": 3.356719414472945, + "tokens_seen": 2612985856 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117979797979798, + "loss": 2.3854, + "theoretical_loss": 3.3567127583665046, + "tokens_seen": 2613051392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002117777777777778, + "loss": 2.7364, + "theoretical_loss": 3.3567061024737406, + "tokens_seen": 2613116928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021175757575757576, + "loss": 2.45, + "theoretical_loss": 3.3566994467946394, + "tokens_seen": 2613182464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021173737373737374, + "loss": 2.6116, + "theoretical_loss": 3.35669279132919, + "tokens_seen": 2613248000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021171717171717172, + "loss": 2.7075, + "theoretical_loss": 3.3566861360773794, + "tokens_seen": 2613313536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021169696969696968, + "loss": 2.5171, + "theoretical_loss": 3.356679481039196, + "tokens_seen": 2613379072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002116767676767677, + "loss": 2.6269, + "theoretical_loss": 3.356672826214627, + "tokens_seen": 2613444608 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1470931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.269923686981201, + "objective/train/theoretical_loss": 3.3566661716036608, + "objective/train/tokens_used": 972369376, + "theoretical_loss": 3.3566661716036608, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021165656565656565, + "loss": 2.392, + "theoretical_loss": 3.3566661716036608, + "tokens_seen": 2613510144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021163636363636366, + "loss": 2.5181, + "theoretical_loss": 3.3566595172062845, + "tokens_seen": 2613575680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021161616161616162, + "loss": 2.4621, + "theoretical_loss": 3.3566528630224863, + "tokens_seen": 2613641216 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002115959595959596, + "loss": 2.6133, + "theoretical_loss": 3.356646209052254, + "tokens_seen": 2613706752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021157575757575758, + "loss": 2.3564, + "theoretical_loss": 3.3566395552955752, + "tokens_seen": 2613772288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021155555555555557, + "loss": 2.4073, + "theoretical_loss": 3.356632901752438, + "tokens_seen": 2613837824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021153535353535355, + "loss": 2.663, + "theoretical_loss": 3.3566262484228298, + "tokens_seen": 2613903360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002115151515151515, + "loss": 2.2037, + "theoretical_loss": 3.356619595306739, + "tokens_seen": 2613968896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002114949494949495, + "loss": 2.4887, + "theoretical_loss": 3.356612942404152, + "tokens_seen": 2614034432 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021147474747474748, + "loss": 2.609, + "theoretical_loss": 3.3566062897150584, + "tokens_seen": 2614099968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021145454545454546, + "loss": 2.4913, + "theoretical_loss": 3.3565996372394453, + "tokens_seen": 2614165504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021143434343434345, + "loss": 2.5054, + "theoretical_loss": 3.3565929849772997, + "tokens_seen": 2614231040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021141414141414143, + "loss": 2.7162, + "theoretical_loss": 3.356586332928611, + "tokens_seen": 2614296576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021139393939393939, + "loss": 2.3724, + "theoretical_loss": 3.3565796810933652, + "tokens_seen": 2614362112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002113737373737374, + "loss": 2.1943, + "theoretical_loss": 3.3565730294715515, + "tokens_seen": 2614427648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021135353535353535, + "loss": 2.2761, + "theoretical_loss": 3.356566378063157, + "tokens_seen": 2614493184 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021133333333333334, + "loss": 2.5363, + "theoretical_loss": 3.35655972686817, + "tokens_seen": 2614558720 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021131313131313132, + "loss": 2.4814, + "theoretical_loss": 3.3565530758865774, + "tokens_seen": 2614624256 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021129292929292928, + "loss": 2.5097, + "theoretical_loss": 3.356546425118368, + "tokens_seen": 2614689792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002112727272727273, + "loss": 2.4186, + "theoretical_loss": 3.356539774563529, + "tokens_seen": 2614755328 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021125252525252525, + "loss": 2.3673, + "theoretical_loss": 3.3565331242220484, + "tokens_seen": 2614820864 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021123232323232326, + "loss": 2.3268, + "theoretical_loss": 3.356526474093914, + "tokens_seen": 2614886400 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021121212121212121, + "loss": 2.485, + "theoretical_loss": 3.356519824179114, + "tokens_seen": 2614951936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021119191919191917, + "loss": 2.7139, + "theoretical_loss": 3.3565131744776355, + "tokens_seen": 2615017472 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021117171717171718, + "loss": 2.3598, + "theoretical_loss": 3.3565065249894666, + "tokens_seen": 2615083008 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1471590, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8833582401275635, + "objective/train/theoretical_loss": 3.3564998757145954, + "objective/train/tokens_used": 974007776, + "theoretical_loss": 3.3564998757145954, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021115151515151514, + "loss": 2.5636, + "theoretical_loss": 3.3564998757145954, + "tokens_seen": 2615148544 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021113131313131315, + "loss": 2.2779, + "theoretical_loss": 3.356493226653009, + "tokens_seen": 2615214080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002111111111111111, + "loss": 2.7673, + "theoretical_loss": 3.356486577804696, + "tokens_seen": 2615279616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002110909090909091, + "loss": 2.5905, + "theoretical_loss": 3.356479929169644, + "tokens_seen": 2615345152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021107070707070708, + "loss": 2.4661, + "theoretical_loss": 3.356473280747841, + "tokens_seen": 2615410688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021105050505050506, + "loss": 2.5, + "theoretical_loss": 3.3564666325392736, + "tokens_seen": 2615476224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021103030303030304, + "loss": 2.5195, + "theoretical_loss": 3.3564599845439314, + "tokens_seen": 2615541760 + }, + { + "epoch": 0.58, + "learning_rate": 0.000211010101010101, + "loss": 2.4499, + "theoretical_loss": 3.356453336761801, + "tokens_seen": 2615607296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021098989898989898, + "loss": 2.5267, + "theoretical_loss": 3.356446689192871, + "tokens_seen": 2615672832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021096969696969697, + "loss": 2.3362, + "theoretical_loss": 3.356440041837128, + "tokens_seen": 2615738368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021094949494949495, + "loss": 2.6923, + "theoretical_loss": 3.356433394694561, + "tokens_seen": 2615803904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021092929292929294, + "loss": 2.6559, + "theoretical_loss": 3.3564267477651577, + "tokens_seen": 2615869440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021090909090909092, + "loss": 2.6245, + "theoretical_loss": 3.356420101048905, + "tokens_seen": 2615934976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021088888888888888, + "loss": 2.6272, + "theoretical_loss": 3.3564134545457924, + "tokens_seen": 2616000512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002108686868686869, + "loss": 2.4496, + "theoretical_loss": 3.356406808255806, + "tokens_seen": 2616066048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021084848484848484, + "loss": 2.6318, + "theoretical_loss": 3.356400162178935, + "tokens_seen": 2616131584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021082828282828283, + "loss": 2.5235, + "theoretical_loss": 3.356393516315166, + "tokens_seen": 2616197120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002108080808080808, + "loss": 2.6046, + "theoretical_loss": 3.3563868706644877, + "tokens_seen": 2616262656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021078787878787877, + "loss": 2.4962, + "theoretical_loss": 3.3563802252268875, + "tokens_seen": 2616328192 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021076767676767678, + "loss": 2.3735, + "theoretical_loss": 3.3563735800023533, + "tokens_seen": 2616393728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021074747474747474, + "loss": 2.4959, + "theoretical_loss": 3.356366934990873, + "tokens_seen": 2616459264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021072727272727275, + "loss": 2.5892, + "theoretical_loss": 3.3563602901924345, + "tokens_seen": 2616524800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002107070707070707, + "loss": 2.5694, + "theoretical_loss": 3.3563536456070255, + "tokens_seen": 2616590336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021068686868686872, + "loss": 2.6584, + "theoretical_loss": 3.356347001234634, + "tokens_seen": 2616655872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021066666666666667, + "loss": 2.4911, + "theoretical_loss": 3.356340357075248, + "tokens_seen": 2616721408 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1472953, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5239038467407227, + "objective/train/theoretical_loss": 3.356333713128855, + "objective/train/tokens_used": 975646176, + "theoretical_loss": 3.356333713128855, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021064646464646463, + "loss": 2.4114, + "theoretical_loss": 3.356333713128855, + "tokens_seen": 2616786944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021062626262626264, + "loss": 2.3768, + "theoretical_loss": 3.3563270693954426, + "tokens_seen": 2616852480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002106060606060606, + "loss": 2.5661, + "theoretical_loss": 3.356320425874999, + "tokens_seen": 2616918016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002105858585858586, + "loss": 2.5791, + "theoretical_loss": 3.3563137825675122, + "tokens_seen": 2616983552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021056565656565657, + "loss": 2.4925, + "theoretical_loss": 3.3563071394729698, + "tokens_seen": 2617049088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021054545454545455, + "loss": 2.5195, + "theoretical_loss": 3.3563004965913597, + "tokens_seen": 2617114624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021052525252525253, + "loss": 2.5625, + "theoretical_loss": 3.3562938539226694, + "tokens_seen": 2617180160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021050505050505052, + "loss": 2.5874, + "theoretical_loss": 3.3562872114668876, + "tokens_seen": 2617245696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104848484848485, + "loss": 2.6045, + "theoretical_loss": 3.356280569224001, + "tokens_seen": 2617311232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021046464646464646, + "loss": 2.4164, + "theoretical_loss": 3.3562739271939988, + "tokens_seen": 2617376768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021044444444444444, + "loss": 2.6644, + "theoretical_loss": 3.356267285376868, + "tokens_seen": 2617442304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021042424242424243, + "loss": 2.6743, + "theoretical_loss": 3.356260643772596, + "tokens_seen": 2617507840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002104040404040404, + "loss": 2.6847, + "theoretical_loss": 3.3562540023811716, + "tokens_seen": 2617573376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103838383838384, + "loss": 2.6391, + "theoretical_loss": 3.356247361202582, + "tokens_seen": 2617638912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021036363636363638, + "loss": 2.6161, + "theoretical_loss": 3.356240720236815, + "tokens_seen": 2617704448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021034343434343434, + "loss": 2.7575, + "theoretical_loss": 3.356234079483859, + "tokens_seen": 2617769984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021032323232323232, + "loss": 2.5144, + "theoretical_loss": 3.3562274389437023, + "tokens_seen": 2617835520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002103030303030303, + "loss": 2.4688, + "theoretical_loss": 3.3562207986163313, + "tokens_seen": 2617901056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002102828282828283, + "loss": 2.3946, + "theoretical_loss": 3.356214158501735, + "tokens_seen": 2617966592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021026262626262627, + "loss": 2.5651, + "theoretical_loss": 3.3562075185999007, + "tokens_seen": 2618032128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021024242424242423, + "loss": 2.4002, + "theoretical_loss": 3.3562008789108164, + "tokens_seen": 2618097664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021022222222222224, + "loss": 2.3537, + "theoretical_loss": 3.35619423943447, + "tokens_seen": 2618163200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002102020202020202, + "loss": 2.4445, + "theoretical_loss": 3.3561876001708493, + "tokens_seen": 2618228736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002101818181818182, + "loss": 2.4778, + "theoretical_loss": 3.3561809611199425, + "tokens_seen": 2618294272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021016161616161616, + "loss": 2.5411, + "theoretical_loss": 3.356174322281737, + "tokens_seen": 2618359808 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1473611, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2712419033050537, + "objective/train/theoretical_loss": 3.3561676836562206, + "objective/train/tokens_used": 977284576, + "theoretical_loss": 3.3561676836562206, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021014141414141412, + "loss": 2.5081, + "theoretical_loss": 3.3561676836562206, + "tokens_seen": 2618425344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021012121212121213, + "loss": 2.5062, + "theoretical_loss": 3.3561610452433817, + "tokens_seen": 2618490880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002101010101010101, + "loss": 2.2996, + "theoretical_loss": 3.3561544070432077, + "tokens_seen": 2618556416 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002100808080808081, + "loss": 2.6746, + "theoretical_loss": 3.3561477690556867, + "tokens_seen": 2618621952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021006060606060606, + "loss": 2.7476, + "theoretical_loss": 3.3561411312808067, + "tokens_seen": 2618687488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021004040404040404, + "loss": 2.5286, + "theoretical_loss": 3.356134493718555, + "tokens_seen": 2618753024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021002020202020202, + "loss": 2.5391, + "theoretical_loss": 3.3561278563689196, + "tokens_seen": 2618818560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00021, + "loss": 2.5328, + "theoretical_loss": 3.356121219231889, + "tokens_seen": 2618884096 + }, + { + "epoch": 0.58, + "learning_rate": 0.000209979797979798, + "loss": 2.5745, + "theoretical_loss": 3.3561145823074505, + "tokens_seen": 2618949632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020995959595959595, + "loss": 2.7075, + "theoretical_loss": 3.3561079455955922, + "tokens_seen": 2619015168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020993939393939393, + "loss": 2.514, + "theoretical_loss": 3.356101309096302, + "tokens_seen": 2619080704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020991919191919192, + "loss": 2.7392, + "theoretical_loss": 3.3560946728095677, + "tokens_seen": 2619146240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002098989898989899, + "loss": 2.5714, + "theoretical_loss": 3.356088036735377, + "tokens_seen": 2619211776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020987878787878789, + "loss": 2.4611, + "theoretical_loss": 3.356081400873718, + "tokens_seen": 2619277312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020985858585858587, + "loss": 2.6503, + "theoretical_loss": 3.3560747652245784, + "tokens_seen": 2619342848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020983838383838385, + "loss": 2.326, + "theoretical_loss": 3.356068129787946, + "tokens_seen": 2619408384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020981818181818184, + "loss": 2.6297, + "theoretical_loss": 3.3560614945638094, + "tokens_seen": 2619473920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002097979797979798, + "loss": 2.4853, + "theoretical_loss": 3.3560548595521555, + "tokens_seen": 2619539456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020977777777777778, + "loss": 2.6574, + "theoretical_loss": 3.356048224752973, + "tokens_seen": 2619604992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020975757575757576, + "loss": 2.4382, + "theoretical_loss": 3.356041590166249, + "tokens_seen": 2619670528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020973737373737375, + "loss": 2.7159, + "theoretical_loss": 3.356034955791972, + "tokens_seen": 2619736064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020971717171717173, + "loss": 2.7148, + "theoretical_loss": 3.3560283216301294, + "tokens_seen": 2619801600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002096969696969697, + "loss": 2.4938, + "theoretical_loss": 3.35602168768071, + "tokens_seen": 2619867136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0002096767676767677, + "loss": 2.6807, + "theoretical_loss": 3.3560150539437004, + "tokens_seen": 2619932672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020965656565656565, + "loss": 2.6903, + "theoretical_loss": 3.356008420419089, + "tokens_seen": 2619998208 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 1474984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.471558094024658, + "objective/train/theoretical_loss": 3.356001787106864, + "objective/train/tokens_used": 978922976, + "theoretical_loss": 3.356001787106864, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00020963636363636367, + "loss": 2.7197, + "theoretical_loss": 3.356001787106864, + "tokens_seen": 2620063744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020961616161616162, + "loss": 2.5182, + "theoretical_loss": 3.3559951540070134, + "tokens_seen": 2620129280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020959595959595958, + "loss": 2.4476, + "theoretical_loss": 3.3559885211195244, + "tokens_seen": 2620194816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002095757575757576, + "loss": 2.4947, + "theoretical_loss": 3.3559818884443855, + "tokens_seen": 2620260352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020955555555555555, + "loss": 2.2894, + "theoretical_loss": 3.3559752559815843, + "tokens_seen": 2620325888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020953535353535356, + "loss": 2.5623, + "theoretical_loss": 3.355968623731109, + "tokens_seen": 2620391424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020951515151515152, + "loss": 2.6304, + "theoretical_loss": 3.3559619916929466, + "tokens_seen": 2620456960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002094949494949495, + "loss": 2.6061, + "theoretical_loss": 3.355955359867086, + "tokens_seen": 2620522496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020947474747474748, + "loss": 2.4499, + "theoretical_loss": 3.355948728253515, + "tokens_seen": 2620588032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020945454545454544, + "loss": 2.6729, + "theoretical_loss": 3.355942096852221, + "tokens_seen": 2620653568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020943434343434345, + "loss": 2.4747, + "theoretical_loss": 3.355935465663192, + "tokens_seen": 2620719104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002094141414141414, + "loss": 2.2555, + "theoretical_loss": 3.355928834686416, + "tokens_seen": 2620784640 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002093939393939394, + "loss": 2.5016, + "theoretical_loss": 3.3559222039218812, + "tokens_seen": 2620850176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020937373737373738, + "loss": 2.4267, + "theoretical_loss": 3.355915573369575, + "tokens_seen": 2620915712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020935353535353536, + "loss": 2.3962, + "theoretical_loss": 3.3559089430294855, + "tokens_seen": 2620981248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020933333333333334, + "loss": 2.5059, + "theoretical_loss": 3.3559023129016006, + "tokens_seen": 2621046784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020931313131313133, + "loss": 2.6395, + "theoretical_loss": 3.355895682985908, + "tokens_seen": 2621112320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020929292929292928, + "loss": 2.6206, + "theoretical_loss": 3.3558890532823966, + "tokens_seen": 2621177856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020927272727272727, + "loss": 2.6338, + "theoretical_loss": 3.355882423791053, + "tokens_seen": 2621243392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020925252525252525, + "loss": 2.4415, + "theoretical_loss": 3.355875794511866, + "tokens_seen": 2621308928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020923232323232324, + "loss": 2.5469, + "theoretical_loss": 3.3558691654448225, + "tokens_seen": 2621374464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020921212121212122, + "loss": 2.466, + "theoretical_loss": 3.3558625365899117, + "tokens_seen": 2621440000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020919191919191918, + "loss": 2.4332, + "theoretical_loss": 3.3558559079471206, + "tokens_seen": 2621505536 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091717171717172, + "loss": 2.4907, + "theoretical_loss": 3.355849279516437, + "tokens_seen": 2621571072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020915151515151515, + "loss": 2.4216, + "theoretical_loss": 3.35584265129785, + "tokens_seen": 2621636608 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1476237, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8851654529571533, + "objective/train/theoretical_loss": 3.355836023291346, + "objective/train/tokens_used": 980561376, + "theoretical_loss": 3.355836023291346, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020913131313131316, + "loss": 2.6402, + "theoretical_loss": 3.355836023291346, + "tokens_seen": 2621702144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002091111111111111, + "loss": 2.7565, + "theoretical_loss": 3.355829395496914, + "tokens_seen": 2621767680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020909090909090907, + "loss": 2.5604, + "theoretical_loss": 3.355822767914541, + "tokens_seen": 2621833216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020907070707070708, + "loss": 2.6009, + "theoretical_loss": 3.355816140544216, + "tokens_seen": 2621898752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020905050505050504, + "loss": 2.6209, + "theoretical_loss": 3.3558095133859265, + "tokens_seen": 2621964288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020903030303030305, + "loss": 2.6259, + "theoretical_loss": 3.3558028864396596, + "tokens_seen": 2622029824 + }, + { + "epoch": 0.59, + "learning_rate": 0.000209010101010101, + "loss": 2.5792, + "theoretical_loss": 3.3557962597054045, + "tokens_seen": 2622095360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020898989898989902, + "loss": 2.8762, + "theoretical_loss": 3.3557896331831483, + "tokens_seen": 2622160896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020896969696969697, + "loss": 2.3075, + "theoretical_loss": 3.355783006872879, + "tokens_seen": 2622226432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020894949494949496, + "loss": 2.6222, + "theoretical_loss": 3.3557763807745844, + "tokens_seen": 2622291968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020892929292929294, + "loss": 2.7128, + "theoretical_loss": 3.355769754888253, + "tokens_seen": 2622357504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002089090909090909, + "loss": 2.4695, + "theoretical_loss": 3.3557631292138725, + "tokens_seen": 2622423040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002088888888888889, + "loss": 2.7488, + "theoretical_loss": 3.3557565037514303, + "tokens_seen": 2622488576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020886868686868687, + "loss": 2.5927, + "theoretical_loss": 3.355749878500915, + "tokens_seen": 2622554112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020884848484848485, + "loss": 2.5077, + "theoretical_loss": 3.3557432534623146, + "tokens_seen": 2622619648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020882828282828283, + "loss": 2.6624, + "theoretical_loss": 3.3557366286356163, + "tokens_seen": 2622685184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020880808080808082, + "loss": 2.5274, + "theoretical_loss": 3.355730004020808, + "tokens_seen": 2622750720 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087878787878788, + "loss": 2.6484, + "theoretical_loss": 3.3557233796178787, + "tokens_seen": 2622816256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020876767676767679, + "loss": 2.5381, + "theoretical_loss": 3.3557167554268155, + "tokens_seen": 2622881792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020874747474747474, + "loss": 2.575, + "theoretical_loss": 3.3557101314476068, + "tokens_seen": 2622947328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020872727272727273, + "loss": 2.5799, + "theoretical_loss": 3.3557035076802397, + "tokens_seen": 2623012864 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002087070707070707, + "loss": 2.4605, + "theoretical_loss": 3.355696884124703, + "tokens_seen": 2623078400 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002086868686868687, + "loss": 2.4856, + "theoretical_loss": 3.355690260780984, + "tokens_seen": 2623143936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020866666666666668, + "loss": 2.342, + "theoretical_loss": 3.3556836376490713, + "tokens_seen": 2623209472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020864646464646464, + "loss": 2.6486, + "theoretical_loss": 3.3556770147289523, + "tokens_seen": 2623275008 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1476787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4551475048065186, + "objective/train/theoretical_loss": 3.355670392020615, + "objective/train/tokens_used": 982199776, + "theoretical_loss": 3.355670392020615, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020862626262626265, + "loss": 2.5775, + "theoretical_loss": 3.355670392020615, + "tokens_seen": 2623340544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002086060606060606, + "loss": 2.4056, + "theoretical_loss": 3.3556637695240474, + "tokens_seen": 2623406080 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002085858585858586, + "loss": 2.4308, + "theoretical_loss": 3.3556571472392376, + "tokens_seen": 2623471616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020856565656565657, + "loss": 2.5327, + "theoretical_loss": 3.3556505251661735, + "tokens_seen": 2623537152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020854545454545453, + "loss": 2.4088, + "theoretical_loss": 3.355643903304843, + "tokens_seen": 2623602688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020852525252525254, + "loss": 2.5882, + "theoretical_loss": 3.355637281655234, + "tokens_seen": 2623668224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002085050505050505, + "loss": 2.8227, + "theoretical_loss": 3.3556306602173342, + "tokens_seen": 2623733760 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084848484848485, + "loss": 2.5648, + "theoretical_loss": 3.3556240389911323, + "tokens_seen": 2623799296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020846464646464646, + "loss": 2.6722, + "theoretical_loss": 3.355617417976615, + "tokens_seen": 2623864832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020844444444444445, + "loss": 2.5203, + "theoretical_loss": 3.355610797173772, + "tokens_seen": 2623930368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020842424242424243, + "loss": 2.6294, + "theoretical_loss": 3.355604176582589, + "tokens_seen": 2623995904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002084040404040404, + "loss": 2.5702, + "theoretical_loss": 3.355597556203056, + "tokens_seen": 2624061440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083838383838384, + "loss": 2.6588, + "theoretical_loss": 3.35559093603516, + "tokens_seen": 2624126976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020836363636363636, + "loss": 2.7152, + "theoretical_loss": 3.3555843160788887, + "tokens_seen": 2624192512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020834343434343434, + "loss": 2.6834, + "theoretical_loss": 3.355577696334231, + "tokens_seen": 2624258048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020832323232323233, + "loss": 2.6195, + "theoretical_loss": 3.3555710768011737, + "tokens_seen": 2624323584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002083030303030303, + "loss": 2.3626, + "theoretical_loss": 3.355564457479706, + "tokens_seen": 2624389120 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082828282828283, + "loss": 2.6603, + "theoretical_loss": 3.3555578383698146, + "tokens_seen": 2624454656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020826262626262628, + "loss": 2.717, + "theoretical_loss": 3.3555512194714883, + "tokens_seen": 2624520192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020824242424242423, + "loss": 2.6203, + "theoretical_loss": 3.3555446007847145, + "tokens_seen": 2624585728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020822222222222222, + "loss": 2.3374, + "theoretical_loss": 3.3555379823094817, + "tokens_seen": 2624651264 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002082020202020202, + "loss": 2.6371, + "theoretical_loss": 3.3555313640457776, + "tokens_seen": 2624716800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020818181818181819, + "loss": 2.6649, + "theoretical_loss": 3.35552474599359, + "tokens_seen": 2624782336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020816161616161617, + "loss": 2.6976, + "theoretical_loss": 3.355518128152907, + "tokens_seen": 2624847872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020814141414141415, + "loss": 2.5888, + "theoretical_loss": 3.3555115105237165, + "tokens_seen": 2624913408 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1477458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3693456649780273, + "objective/train/theoretical_loss": 3.3555048931060067, + "objective/train/tokens_used": 983838176, + "theoretical_loss": 3.3555048931060067, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020812121212121214, + "loss": 2.354, + "theoretical_loss": 3.3555048931060067, + "tokens_seen": 2624978944 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002081010101010101, + "loss": 2.4097, + "theoretical_loss": 3.3554982758997656, + "tokens_seen": 2625044480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002080808080808081, + "loss": 2.5212, + "theoretical_loss": 3.355491658904981, + "tokens_seen": 2625110016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020806060606060606, + "loss": 2.4364, + "theoretical_loss": 3.3554850421216402, + "tokens_seen": 2625175552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020804040404040405, + "loss": 2.4973, + "theoretical_loss": 3.3554784255497325, + "tokens_seen": 2625241088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020802020202020203, + "loss": 2.4843, + "theoretical_loss": 3.355471809189245, + "tokens_seen": 2625306624 + }, + { + "epoch": 0.59, + "learning_rate": 0.000208, + "loss": 2.4115, + "theoretical_loss": 3.3554651930401658, + "tokens_seen": 2625372160 + }, + { + "epoch": 0.59, + "learning_rate": 0.000207979797979798, + "loss": 2.5443, + "theoretical_loss": 3.3554585771024827, + "tokens_seen": 2625437696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020795959595959596, + "loss": 2.4532, + "theoretical_loss": 3.3554519613761835, + "tokens_seen": 2625503232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020793939393939397, + "loss": 2.6349, + "theoretical_loss": 3.3554453458612574, + "tokens_seen": 2625568768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020791919191919192, + "loss": 2.5215, + "theoretical_loss": 3.355438730557691, + "tokens_seen": 2625634304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020789898989898988, + "loss": 2.4377, + "theoretical_loss": 3.355432115465473, + "tokens_seen": 2625699840 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002078787878787879, + "loss": 2.5302, + "theoretical_loss": 3.355425500584591, + "tokens_seen": 2625765376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020785858585858585, + "loss": 2.2892, + "theoretical_loss": 3.355418885915033, + "tokens_seen": 2625830912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020783838383838386, + "loss": 2.5915, + "theoretical_loss": 3.355412271456787, + "tokens_seen": 2625896448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020781818181818182, + "loss": 2.675, + "theoretical_loss": 3.3554056572098414, + "tokens_seen": 2625961984 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002077979797979798, + "loss": 2.4699, + "theoretical_loss": 3.355399043174184, + "tokens_seen": 2626027520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020777777777777778, + "loss": 2.4554, + "theoretical_loss": 3.355392429349802, + "tokens_seen": 2626093056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020775757575757577, + "loss": 2.3263, + "theoretical_loss": 3.3553858157366845, + "tokens_seen": 2626158592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020773737373737375, + "loss": 2.5077, + "theoretical_loss": 3.3553792023348192, + "tokens_seen": 2626224128 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002077171717171717, + "loss": 2.3026, + "theoretical_loss": 3.3553725891441935, + "tokens_seen": 2626289664 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002076969696969697, + "loss": 2.5942, + "theoretical_loss": 3.3553659761647956, + "tokens_seen": 2626355200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020767676767676768, + "loss": 2.5763, + "theoretical_loss": 3.355359363396614, + "tokens_seen": 2626420736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020765656565656566, + "loss": 2.8331, + "theoretical_loss": 3.355352750839636, + "tokens_seen": 2626486272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020763636363636364, + "loss": 2.5616, + "theoretical_loss": 3.35534613849385, + "tokens_seen": 2626551808 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1478565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9924532175064087, + "objective/train/theoretical_loss": 3.355339526359244, + "objective/train/tokens_used": 985476576, + "theoretical_loss": 3.355339526359244, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020761616161616163, + "loss": 2.2151, + "theoretical_loss": 3.355339526359244, + "tokens_seen": 2626617344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020759595959595959, + "loss": 2.7033, + "theoretical_loss": 3.355332914435806, + "tokens_seen": 2626682880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002075757575757576, + "loss": 2.5844, + "theoretical_loss": 3.3553263027235234, + "tokens_seen": 2626748416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020755555555555555, + "loss": 2.6265, + "theoretical_loss": 3.355319691222385, + "tokens_seen": 2626813952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020753535353535354, + "loss": 2.6825, + "theoretical_loss": 3.3553130799323783, + "tokens_seen": 2626879488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020751515151515152, + "loss": 2.5548, + "theoretical_loss": 3.3553064688534913, + "tokens_seen": 2626945024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020749494949494948, + "loss": 2.5271, + "theoretical_loss": 3.3552998579857123, + "tokens_seen": 2627010560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002074747474747475, + "loss": 2.4651, + "theoretical_loss": 3.3552932473290293, + "tokens_seen": 2627076096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020745454545454545, + "loss": 2.3726, + "theoretical_loss": 3.35528663688343, + "tokens_seen": 2627141632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020743434343434346, + "loss": 2.4336, + "theoretical_loss": 3.3552800266489022, + "tokens_seen": 2627207168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020741414141414141, + "loss": 2.3595, + "theoretical_loss": 3.3552734166254345, + "tokens_seen": 2627272704 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073939393939394, + "loss": 2.6223, + "theoretical_loss": 3.3552668068130145, + "tokens_seen": 2627338240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020737373737373738, + "loss": 2.466, + "theoretical_loss": 3.35526019721163, + "tokens_seen": 2627403776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020735353535353534, + "loss": 2.5161, + "theoretical_loss": 3.3552535878212697, + "tokens_seen": 2627469312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020733333333333335, + "loss": 2.2853, + "theoretical_loss": 3.355246978641921, + "tokens_seen": 2627534848 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002073131313131313, + "loss": 2.6802, + "theoretical_loss": 3.3552403696735724, + "tokens_seen": 2627600384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020729292929292932, + "loss": 2.6099, + "theoretical_loss": 3.355233760916211, + "tokens_seen": 2627665920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020727272727272727, + "loss": 2.4489, + "theoretical_loss": 3.355227152369826, + "tokens_seen": 2627731456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020725252525252526, + "loss": 2.4901, + "theoretical_loss": 3.3552205440344043, + "tokens_seen": 2627796992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020723232323232324, + "loss": 2.4714, + "theoretical_loss": 3.355213935909935, + "tokens_seen": 2627862528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020721212121212123, + "loss": 2.3253, + "theoretical_loss": 3.355207327996405, + "tokens_seen": 2627928064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002071919191919192, + "loss": 2.4116, + "theoretical_loss": 3.355200720293803, + "tokens_seen": 2627993600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020717171717171717, + "loss": 2.5375, + "theoretical_loss": 3.3551941128021165, + "tokens_seen": 2628059136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020715151515151515, + "loss": 2.5241, + "theoretical_loss": 3.355187505521334, + "tokens_seen": 2628124672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020713131313131313, + "loss": 2.3681, + "theoretical_loss": 3.3551808984514437, + "tokens_seen": 2628190208 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1479263, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7428247928619385, + "objective/train/theoretical_loss": 3.3551742915924327, + "objective/train/tokens_used": 987114976, + "theoretical_loss": 3.3551742915924327, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020711111111111112, + "loss": 2.725, + "theoretical_loss": 3.3551742915924327, + "tokens_seen": 2628255744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070909090909091, + "loss": 2.3908, + "theoretical_loss": 3.35516768494429, + "tokens_seen": 2628321280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002070707070707071, + "loss": 2.5194, + "theoretical_loss": 3.3551610785070034, + "tokens_seen": 2628386816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020705050505050504, + "loss": 2.7018, + "theoretical_loss": 3.35515447228056, + "tokens_seen": 2628452352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020703030303030303, + "loss": 2.5328, + "theoretical_loss": 3.3551478662649488, + "tokens_seen": 2628517888 + }, + { + "epoch": 0.59, + "learning_rate": 0.000207010101010101, + "loss": 2.6142, + "theoretical_loss": 3.3551412604601576, + "tokens_seen": 2628583424 + }, + { + "epoch": 0.59, + "learning_rate": 0.000206989898989899, + "loss": 2.4071, + "theoretical_loss": 3.3551346548661742, + "tokens_seen": 2628648960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020696969696969698, + "loss": 2.6389, + "theoretical_loss": 3.3551280494829867, + "tokens_seen": 2628714496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020694949494949494, + "loss": 2.5381, + "theoretical_loss": 3.3551214443105835, + "tokens_seen": 2628780032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020692929292929295, + "loss": 2.4735, + "theoretical_loss": 3.3551148393489516, + "tokens_seen": 2628845568 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002069090909090909, + "loss": 2.6837, + "theoretical_loss": 3.35510823459808, + "tokens_seen": 2628911104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020688888888888892, + "loss": 2.4644, + "theoretical_loss": 3.355101630057957, + "tokens_seen": 2628976640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020686868686868687, + "loss": 2.6057, + "theoretical_loss": 3.3550950257285694, + "tokens_seen": 2629042176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020684848484848483, + "loss": 2.665, + "theoretical_loss": 3.355088421609906, + "tokens_seen": 2629107712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020682828282828284, + "loss": 2.4325, + "theoretical_loss": 3.355081817701955, + "tokens_seen": 2629173248 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002068080808080808, + "loss": 2.5544, + "theoretical_loss": 3.355075214004704, + "tokens_seen": 2629238784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002067878787878788, + "loss": 2.6826, + "theoretical_loss": 3.3550686105181406, + "tokens_seen": 2629304320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020676767676767676, + "loss": 2.6092, + "theoretical_loss": 3.355062007242254, + "tokens_seen": 2629369856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020674747474747475, + "loss": 2.5659, + "theoretical_loss": 3.3550554041770315, + "tokens_seen": 2629435392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020672727272727273, + "loss": 2.6142, + "theoretical_loss": 3.355048801322461, + "tokens_seen": 2629500928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020670707070707072, + "loss": 2.5219, + "theoretical_loss": 3.3550421986785306, + "tokens_seen": 2629566464 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066868686868687, + "loss": 2.5468, + "theoretical_loss": 3.355035596245229, + "tokens_seen": 2629632000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020666666666666666, + "loss": 2.5381, + "theoretical_loss": 3.3550289940225433, + "tokens_seen": 2629697536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020664646464646464, + "loss": 2.4247, + "theoretical_loss": 3.3550223920104623, + "tokens_seen": 2629763072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020662626262626263, + "loss": 2.4438, + "theoretical_loss": 3.3550157902089737, + "tokens_seen": 2629828608 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1480679, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2702696323394775, + "objective/train/theoretical_loss": 3.355009188618065, + "objective/train/tokens_used": 988753376, + "theoretical_loss": 3.355009188618065, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002066060606060606, + "loss": 2.401, + "theoretical_loss": 3.355009188618065, + "tokens_seen": 2629894144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065858585858586, + "loss": 2.5568, + "theoretical_loss": 3.3550025872377254, + "tokens_seen": 2629959680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020656565656565658, + "loss": 2.5634, + "theoretical_loss": 3.3549959860679417, + "tokens_seen": 2630025216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020654545454545453, + "loss": 2.5446, + "theoretical_loss": 3.354989385108703, + "tokens_seen": 2630090752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020652525252525255, + "loss": 2.5941, + "theoretical_loss": 3.3549827843599966, + "tokens_seen": 2630156288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002065050505050505, + "loss": 2.5055, + "theoretical_loss": 3.354976183821811, + "tokens_seen": 2630221824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020648484848484849, + "loss": 2.3999, + "theoretical_loss": 3.354969583494134, + "tokens_seen": 2630287360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020646464646464647, + "loss": 2.4385, + "theoretical_loss": 3.3549629833769536, + "tokens_seen": 2630352896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020644444444444445, + "loss": 2.4803, + "theoretical_loss": 3.354956383470258, + "tokens_seen": 2630418432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020642424242424244, + "loss": 2.5547, + "theoretical_loss": 3.354949783774035, + "tokens_seen": 2630483968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002064040404040404, + "loss": 2.7075, + "theoretical_loss": 3.354943184288273, + "tokens_seen": 2630549504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002063838383838384, + "loss": 2.5148, + "theoretical_loss": 3.3549365850129598, + "tokens_seen": 2630615040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020636363636363636, + "loss": 2.2582, + "theoretical_loss": 3.3549299859480834, + "tokens_seen": 2630680576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020634343434343437, + "loss": 2.2518, + "theoretical_loss": 3.354923387093632, + "tokens_seen": 2630746112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020632323232323233, + "loss": 2.379, + "theoretical_loss": 3.354916788449594, + "tokens_seen": 2630811648 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002063030303030303, + "loss": 2.5473, + "theoretical_loss": 3.354910190015957, + "tokens_seen": 2630877184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062828282828283, + "loss": 2.5483, + "theoretical_loss": 3.354903591792709, + "tokens_seen": 2630942720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020626262626262626, + "loss": 2.6827, + "theoretical_loss": 3.3548969937798376, + "tokens_seen": 2631008256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020624242424242427, + "loss": 2.2671, + "theoretical_loss": 3.354890395977332, + "tokens_seen": 2631073792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020622222222222222, + "loss": 2.4466, + "theoretical_loss": 3.3548837983851794, + "tokens_seen": 2631139328 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002062020202020202, + "loss": 2.5853, + "theoretical_loss": 3.3548772010033687, + "tokens_seen": 2631204864 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061818181818182, + "loss": 2.66, + "theoretical_loss": 3.3548706038318867, + "tokens_seen": 2631270400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020616161616161615, + "loss": 2.7669, + "theoretical_loss": 3.3548640068707223, + "tokens_seen": 2631335936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020614141414141416, + "loss": 2.5998, + "theoretical_loss": 3.3548574101198634, + "tokens_seen": 2631401472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020612121212121212, + "loss": 2.6151, + "theoretical_loss": 3.354850813579298, + "tokens_seen": 2631467008 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1481256, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.135030746459961, + "objective/train/theoretical_loss": 3.3548442172490147, + "objective/train/tokens_used": 990391776, + "theoretical_loss": 3.3548442172490147, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002061010101010101, + "loss": 2.6659, + "theoretical_loss": 3.3548442172490147, + "tokens_seen": 2631532544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020608080808080808, + "loss": 2.5708, + "theoretical_loss": 3.3548376211290005, + "tokens_seen": 2631598080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020606060606060607, + "loss": 2.7504, + "theoretical_loss": 3.3548310252192444, + "tokens_seen": 2631663616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020604040404040405, + "loss": 2.495, + "theoretical_loss": 3.354824429519734, + "tokens_seen": 2631729152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020602020202020204, + "loss": 2.5823, + "theoretical_loss": 3.354817834030457, + "tokens_seen": 2631794688 + }, + { + "epoch": 0.59, + "learning_rate": 0.000206, + "loss": 2.5823, + "theoretical_loss": 3.3548112387514024, + "tokens_seen": 2631860224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020597979797979798, + "loss": 2.5429, + "theoretical_loss": 3.3548046436825576, + "tokens_seen": 2631925760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020595959595959596, + "loss": 2.4596, + "theoretical_loss": 3.354798048823911, + "tokens_seen": 2631991296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020593939393939394, + "loss": 2.6657, + "theoretical_loss": 3.35479145417545, + "tokens_seen": 2632056832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020591919191919193, + "loss": 2.4285, + "theoretical_loss": 3.3547848597371637, + "tokens_seen": 2632122368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020589898989898989, + "loss": 2.3979, + "theoretical_loss": 3.3547782655090397, + "tokens_seen": 2632187904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002058787878787879, + "loss": 2.5799, + "theoretical_loss": 3.354771671491066, + "tokens_seen": 2632253440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020585858585858585, + "loss": 2.2362, + "theoretical_loss": 3.3547650776832305, + "tokens_seen": 2632318976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020583838383838386, + "loss": 2.5862, + "theoretical_loss": 3.3547584840855214, + "tokens_seen": 2632384512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020581818181818182, + "loss": 2.4507, + "theoretical_loss": 3.3547518906979272, + "tokens_seen": 2632450048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020579797979797978, + "loss": 2.3122, + "theoretical_loss": 3.3547452975204353, + "tokens_seen": 2632515584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002057777777777778, + "loss": 2.4182, + "theoretical_loss": 3.354738704553034, + "tokens_seen": 2632581120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020575757575757575, + "loss": 2.5374, + "theoretical_loss": 3.3547321117957116, + "tokens_seen": 2632646656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020573737373737376, + "loss": 2.3001, + "theoretical_loss": 3.354725519248456, + "tokens_seen": 2632712192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020571717171717171, + "loss": 2.4782, + "theoretical_loss": 3.3547189269112554, + "tokens_seen": 2632777728 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002056969696969697, + "loss": 2.6127, + "theoretical_loss": 3.354712334784098, + "tokens_seen": 2632843264 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020567676767676768, + "loss": 2.5146, + "theoretical_loss": 3.354705742866971, + "tokens_seen": 2632908800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020565656565656567, + "loss": 2.458, + "theoretical_loss": 3.354699151159864, + "tokens_seen": 2632974336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020563636363636365, + "loss": 2.3975, + "theoretical_loss": 3.3546925596627637, + "tokens_seen": 2633039872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002056161616161616, + "loss": 2.5513, + "theoretical_loss": 3.3546859683756587, + "tokens_seen": 2633105408 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1482418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.147402048110962, + "objective/train/theoretical_loss": 3.3546793772985373, + "objective/train/tokens_used": 992030176, + "theoretical_loss": 3.3546793772985373, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020559595959595962, + "loss": 2.3332, + "theoretical_loss": 3.3546793772985373, + "tokens_seen": 2633170944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020557575757575757, + "loss": 2.5785, + "theoretical_loss": 3.3546727864313874, + "tokens_seen": 2633236480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020555555555555556, + "loss": 2.4113, + "theoretical_loss": 3.354666195774197, + "tokens_seen": 2633302016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020553535353535354, + "loss": 2.8205, + "theoretical_loss": 3.354659605326954, + "tokens_seen": 2633367552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020551515151515153, + "loss": 2.5465, + "theoretical_loss": 3.354653015089647, + "tokens_seen": 2633433088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054949494949495, + "loss": 2.6418, + "theoretical_loss": 3.354646425062264, + "tokens_seen": 2633498624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002054747474747475, + "loss": 2.5448, + "theoretical_loss": 3.3546398352447926, + "tokens_seen": 2633564160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020545454545454545, + "loss": 2.672, + "theoretical_loss": 3.3546332456372214, + "tokens_seen": 2633629696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020543434343434344, + "loss": 2.8044, + "theoretical_loss": 3.3546266562395384, + "tokens_seen": 2633695232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020541414141414142, + "loss": 2.6319, + "theoretical_loss": 3.3546200670517314, + "tokens_seen": 2633760768 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053939393939394, + "loss": 2.5539, + "theoretical_loss": 3.354613478073789, + "tokens_seen": 2633826304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053737373737374, + "loss": 2.4542, + "theoretical_loss": 3.354606889305699, + "tokens_seen": 2633891840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020535353535353534, + "loss": 2.4059, + "theoretical_loss": 3.3546003007474487, + "tokens_seen": 2633957376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020533333333333336, + "loss": 2.4557, + "theoretical_loss": 3.3545937123990273, + "tokens_seen": 2634022912 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002053131313131313, + "loss": 2.3034, + "theoretical_loss": 3.354587124260423, + "tokens_seen": 2634088448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052929292929293, + "loss": 2.644, + "theoretical_loss": 3.354580536331623, + "tokens_seen": 2634153984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020527272727272728, + "loss": 2.6228, + "theoretical_loss": 3.3545739486126163, + "tokens_seen": 2634219520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020525252525252524, + "loss": 2.5276, + "theoretical_loss": 3.3545673611033906, + "tokens_seen": 2634285056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020523232323232325, + "loss": 2.6442, + "theoretical_loss": 3.3545607738039336, + "tokens_seen": 2634350592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002052121212121212, + "loss": 2.3979, + "theoretical_loss": 3.3545541867142337, + "tokens_seen": 2634416128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020519191919191922, + "loss": 2.5203, + "theoretical_loss": 3.3545475998342793, + "tokens_seen": 2634481664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020517171717171717, + "loss": 2.4769, + "theoretical_loss": 3.3545410131640585, + "tokens_seen": 2634547200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020515151515151516, + "loss": 2.5178, + "theoretical_loss": 3.354534426703559, + "tokens_seen": 2634612736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020513131313131314, + "loss": 2.7597, + "theoretical_loss": 3.3545278404527687, + "tokens_seen": 2634678272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002051111111111111, + "loss": 2.8056, + "theoretical_loss": 3.3545212544116767, + "tokens_seen": 2634743808 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1483155, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6380505561828613, + "objective/train/theoretical_loss": 3.3545146685802703, + "objective/train/tokens_used": 993668576, + "theoretical_loss": 3.3545146685802703, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002050909090909091, + "loss": 2.6396, + "theoretical_loss": 3.3545146685802703, + "tokens_seen": 2634809344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020507070707070707, + "loss": 2.4866, + "theoretical_loss": 3.354508082958538, + "tokens_seen": 2634874880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020505050505050505, + "loss": 2.6874, + "theoretical_loss": 3.354501497546467, + "tokens_seen": 2634940416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020503030303030303, + "loss": 2.7439, + "theoretical_loss": 3.354494912344047, + "tokens_seen": 2635005952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020501010101010102, + "loss": 2.4926, + "theoretical_loss": 3.3544883273512647, + "tokens_seen": 2635071488 + }, + { + "epoch": 0.59, + "learning_rate": 0.000204989898989899, + "loss": 2.6606, + "theoretical_loss": 3.354481742568109, + "tokens_seen": 2635137024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020496969696969699, + "loss": 2.722, + "theoretical_loss": 3.3544751579945675, + "tokens_seen": 2635202560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020494949494949494, + "loss": 2.4904, + "theoretical_loss": 3.354468573630629, + "tokens_seen": 2635268096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020492929292929293, + "loss": 2.6465, + "theoretical_loss": 3.354461989476281, + "tokens_seen": 2635333632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002049090909090909, + "loss": 2.4718, + "theoretical_loss": 3.3544554055315117, + "tokens_seen": 2635399168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002048888888888889, + "loss": 2.4431, + "theoretical_loss": 3.3544488217963093, + "tokens_seen": 2635464704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020486868686868688, + "loss": 2.736, + "theoretical_loss": 3.3544422382706625, + "tokens_seen": 2635530240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020484848484848483, + "loss": 2.5555, + "theoretical_loss": 3.354435654954558, + "tokens_seen": 2635595776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020482828282828285, + "loss": 2.5955, + "theoretical_loss": 3.3544290718479854, + "tokens_seen": 2635661312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002048080808080808, + "loss": 2.516, + "theoretical_loss": 3.354422488950932, + "tokens_seen": 2635726848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020478787878787881, + "loss": 2.735, + "theoretical_loss": 3.3544159062633865, + "tokens_seen": 2635792384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020476767676767677, + "loss": 2.6534, + "theoretical_loss": 3.3544093237853363, + "tokens_seen": 2635857920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020474747474747473, + "loss": 2.3835, + "theoretical_loss": 3.35440274151677, + "tokens_seen": 2635923456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020472727272727274, + "loss": 2.6428, + "theoretical_loss": 3.3543961594576754, + "tokens_seen": 2635988992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002047070707070707, + "loss": 2.5212, + "theoretical_loss": 3.3543895776080412, + "tokens_seen": 2636054528 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046868686868687, + "loss": 2.3697, + "theoretical_loss": 3.354382995967855, + "tokens_seen": 2636120064 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020466666666666666, + "loss": 2.4832, + "theoretical_loss": 3.354376414537105, + "tokens_seen": 2636185600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020464646464646467, + "loss": 2.4515, + "theoretical_loss": 3.354369833315779, + "tokens_seen": 2636251136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00020462626262626263, + "loss": 2.732, + "theoretical_loss": 3.3543632523038664, + "tokens_seen": 2636316672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002046060606060606, + "loss": 2.686, + "theoretical_loss": 3.354356671501354, + "tokens_seen": 2636382208 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 1483578, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.446268081665039, + "objective/train/theoretical_loss": 3.3543500909082304, + "objective/train/tokens_used": 995306976, + "theoretical_loss": 3.3543500909082304, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0002045858585858586, + "loss": 2.7105, + "theoretical_loss": 3.3543500909082304, + "tokens_seen": 2636447744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020456565656565656, + "loss": 2.3049, + "theoretical_loss": 3.354343510524484, + "tokens_seen": 2636513280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020454545454545457, + "loss": 2.5054, + "theoretical_loss": 3.3543369303501027, + "tokens_seen": 2636578816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020452525252525252, + "loss": 2.5091, + "theoretical_loss": 3.3543303503850748, + "tokens_seen": 2636644352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002045050505050505, + "loss": 2.5726, + "theoretical_loss": 3.354323770629388, + "tokens_seen": 2636709888 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044848484848485, + "loss": 2.65, + "theoretical_loss": 3.3543171910830303, + "tokens_seen": 2636775424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020446464646464648, + "loss": 2.692, + "theoretical_loss": 3.3543106117459907, + "tokens_seen": 2636840960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020444444444444446, + "loss": 2.473, + "theoretical_loss": 3.3543040326182565, + "tokens_seen": 2636906496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020442424242424242, + "loss": 2.7543, + "theoretical_loss": 3.3542974536998162, + "tokens_seen": 2636972032 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002044040404040404, + "loss": 2.4384, + "theoretical_loss": 3.3542908749906584, + "tokens_seen": 2637037568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020438383838383838, + "loss": 2.6584, + "theoretical_loss": 3.3542842964907704, + "tokens_seen": 2637103104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020436363636363637, + "loss": 2.5686, + "theoretical_loss": 3.354277718200141, + "tokens_seen": 2637168640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020434343434343435, + "loss": 2.5692, + "theoretical_loss": 3.354271140118758, + "tokens_seen": 2637234176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020432323232323234, + "loss": 2.8218, + "theoretical_loss": 3.3542645622466094, + "tokens_seen": 2637299712 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002043030303030303, + "loss": 2.405, + "theoretical_loss": 3.354257984583684, + "tokens_seen": 2637365248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002042828282828283, + "loss": 2.461, + "theoretical_loss": 3.354251407129969, + "tokens_seen": 2637430784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020426262626262626, + "loss": 2.5485, + "theoretical_loss": 3.3542448298854532, + "tokens_seen": 2637496320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020424242424242425, + "loss": 2.3132, + "theoretical_loss": 3.354238252850125, + "tokens_seen": 2637561856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020422222222222223, + "loss": 2.4418, + "theoretical_loss": 3.3542316760239714, + "tokens_seen": 2637627392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020420202020202019, + "loss": 2.3261, + "theoretical_loss": 3.3542250994069818, + "tokens_seen": 2637692928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041818181818182, + "loss": 2.5715, + "theoretical_loss": 3.354218522999144, + "tokens_seen": 2637758464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020416161616161615, + "loss": 2.6559, + "theoretical_loss": 3.354211946800446, + "tokens_seen": 2637824000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020414141414141416, + "loss": 2.5264, + "theoretical_loss": 3.3542053708108757, + "tokens_seen": 2637889536 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020412121212121212, + "loss": 2.6165, + "theoretical_loss": 3.3541987950304213, + "tokens_seen": 2637955072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002041010101010101, + "loss": 2.4747, + "theoretical_loss": 3.3541922194590716, + "tokens_seen": 2638020608 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1484926, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5205161571502686, + "objective/train/theoretical_loss": 3.3541856440968143, + "objective/train/tokens_used": 996945376, + "theoretical_loss": 3.3541856440968143, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002040808080808081, + "loss": 2.3891, + "theoretical_loss": 3.3541856440968143, + "tokens_seen": 2638086144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020406060606060605, + "loss": 2.7226, + "theoretical_loss": 3.3541790689436377, + "tokens_seen": 2638151680 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020404040404040406, + "loss": 2.5339, + "theoretical_loss": 3.3541724939995294, + "tokens_seen": 2638217216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020402020202020201, + "loss": 2.5819, + "theoretical_loss": 3.3541659192644784, + "tokens_seen": 2638282752 + }, + { + "epoch": 0.6, + "learning_rate": 0.000204, + "loss": 2.7279, + "theoretical_loss": 3.354159344738472, + "tokens_seen": 2638348288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020397979797979798, + "loss": 2.5889, + "theoretical_loss": 3.354152770421499, + "tokens_seen": 2638413824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020395959595959597, + "loss": 2.5148, + "theoretical_loss": 3.3541461963135477, + "tokens_seen": 2638479360 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020393939393939395, + "loss": 2.7408, + "theoretical_loss": 3.354139622414606, + "tokens_seen": 2638544896 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020391919191919193, + "loss": 2.6245, + "theoretical_loss": 3.3541330487246617, + "tokens_seen": 2638610432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002038989898989899, + "loss": 2.4716, + "theoretical_loss": 3.354126475243703, + "tokens_seen": 2638675968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020387878787878788, + "loss": 2.534, + "theoretical_loss": 3.354119901971719, + "tokens_seen": 2638741504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020385858585858586, + "loss": 2.625, + "theoretical_loss": 3.3541133289086966, + "tokens_seen": 2638807040 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020383838383838384, + "loss": 2.3583, + "theoretical_loss": 3.354106756054625, + "tokens_seen": 2638872576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020381818181818183, + "loss": 2.5826, + "theoretical_loss": 3.354100183409492, + "tokens_seen": 2638938112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037979797979798, + "loss": 2.6302, + "theoretical_loss": 3.3540936109732855, + "tokens_seen": 2639003648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002037777777777778, + "loss": 2.6846, + "theoretical_loss": 3.354087038745994, + "tokens_seen": 2639069184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020375757575757575, + "loss": 2.6437, + "theoretical_loss": 3.3540804667276056, + "tokens_seen": 2639134720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020373737373737374, + "loss": 2.7635, + "theoretical_loss": 3.354073894918108, + "tokens_seen": 2639200256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020371717171717172, + "loss": 2.4931, + "theoretical_loss": 3.3540673233174902, + "tokens_seen": 2639265792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036969696969697, + "loss": 2.5448, + "theoretical_loss": 3.35406075192574, + "tokens_seen": 2639331328 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036767676767677, + "loss": 2.5129, + "theoretical_loss": 3.3540541807428452, + "tokens_seen": 2639396864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020365656565656564, + "loss": 2.196, + "theoretical_loss": 3.354047609768795, + "tokens_seen": 2639462400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020363636363636366, + "loss": 2.716, + "theoretical_loss": 3.3540410390035764, + "tokens_seen": 2639527936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002036161616161616, + "loss": 2.5153, + "theoretical_loss": 3.354034468447178, + "tokens_seen": 2639593472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020359595959595962, + "loss": 2.7389, + "theoretical_loss": 3.354027898099588, + "tokens_seen": 2639659008 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1485572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5168557167053223, + "objective/train/theoretical_loss": 3.354021327960795, + "objective/train/tokens_used": 998583776, + "theoretical_loss": 3.354021327960795, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020357575757575758, + "loss": 2.5264, + "theoretical_loss": 3.354021327960795, + "tokens_seen": 2639724544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020355555555555554, + "loss": 2.4685, + "theoretical_loss": 3.354014758030787, + "tokens_seen": 2639790080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020353535353535355, + "loss": 2.2581, + "theoretical_loss": 3.3540081883095514, + "tokens_seen": 2639855616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002035151515151515, + "loss": 2.4214, + "theoretical_loss": 3.3540016187970774, + "tokens_seen": 2639921152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020349494949494952, + "loss": 2.7184, + "theoretical_loss": 3.353995049493353, + "tokens_seen": 2639986688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020347474747474747, + "loss": 2.4724, + "theoretical_loss": 3.3539884803983657, + "tokens_seen": 2640052224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020345454545454546, + "loss": 2.6067, + "theoretical_loss": 3.353981911512104, + "tokens_seen": 2640117760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020343434343434344, + "loss": 2.6775, + "theoretical_loss": 3.353975342834557, + "tokens_seen": 2640183296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020341414141414142, + "loss": 2.2831, + "theoretical_loss": 3.353968774365711, + "tokens_seen": 2640248832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002033939393939394, + "loss": 2.5077, + "theoretical_loss": 3.3539622061055563, + "tokens_seen": 2640314368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020337373737373737, + "loss": 2.3769, + "theoretical_loss": 3.35395563805408, + "tokens_seen": 2640379904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020335353535353535, + "loss": 2.5614, + "theoretical_loss": 3.3539490702112698, + "tokens_seen": 2640445440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020333333333333333, + "loss": 2.3829, + "theoretical_loss": 3.353942502577115, + "tokens_seen": 2640510976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020331313131313132, + "loss": 2.3913, + "theoretical_loss": 3.353935935151603, + "tokens_seen": 2640576512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032929292929293, + "loss": 2.571, + "theoretical_loss": 3.3539293679347226, + "tokens_seen": 2640642048 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020327272727272729, + "loss": 2.8006, + "theoretical_loss": 3.353922800926461, + "tokens_seen": 2640707584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020325252525252524, + "loss": 2.55, + "theoretical_loss": 3.3539162341268076, + "tokens_seen": 2640773120 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020323232323232325, + "loss": 2.1884, + "theoretical_loss": 3.35390966753575, + "tokens_seen": 2640838656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002032121212121212, + "loss": 2.3905, + "theoretical_loss": 3.3539031011532767, + "tokens_seen": 2640904192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002031919191919192, + "loss": 2.1925, + "theoretical_loss": 3.3538965349793752, + "tokens_seen": 2640969728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020317171717171718, + "loss": 2.6847, + "theoretical_loss": 3.3538899690140345, + "tokens_seen": 2641035264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020315151515151514, + "loss": 2.5435, + "theoretical_loss": 3.3538834032572424, + "tokens_seen": 2641100800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020313131313131315, + "loss": 2.5115, + "theoretical_loss": 3.3538768377089867, + "tokens_seen": 2641166336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002031111111111111, + "loss": 2.6397, + "theoretical_loss": 3.3538702723692566, + "tokens_seen": 2641231872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020309090909090911, + "loss": 2.3747, + "theoretical_loss": 3.3538637072380393, + "tokens_seen": 2641297408 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1487130, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2561533451080322, + "objective/train/theoretical_loss": 3.3538571423153236, + "objective/train/tokens_used": 1000222176, + "theoretical_loss": 3.3538571423153236, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020307070707070707, + "loss": 2.4871, + "theoretical_loss": 3.3538571423153236, + "tokens_seen": 2641362944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020305050505050503, + "loss": 2.4786, + "theoretical_loss": 3.353850577601098, + "tokens_seen": 2641428480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020303030303030304, + "loss": 2.5593, + "theoretical_loss": 3.3538440130953497, + "tokens_seen": 2641494016 + }, + { + "epoch": 0.6, + "learning_rate": 0.000203010101010101, + "loss": 2.6347, + "theoretical_loss": 3.353837448798068, + "tokens_seen": 2641559552 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202989898989899, + "loss": 2.4905, + "theoretical_loss": 3.3538308847092404, + "tokens_seen": 2641625088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020296969696969696, + "loss": 2.4866, + "theoretical_loss": 3.353824320828855, + "tokens_seen": 2641690624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020294949494949497, + "loss": 2.3796, + "theoretical_loss": 3.3538177571569006, + "tokens_seen": 2641756160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020292929292929293, + "loss": 2.5943, + "theoretical_loss": 3.3538111936933652, + "tokens_seen": 2641821696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020290909090909092, + "loss": 2.6447, + "theoretical_loss": 3.3538046304382365, + "tokens_seen": 2641887232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002028888888888889, + "loss": 2.5338, + "theoretical_loss": 3.3537980673915033, + "tokens_seen": 2641952768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020286868686868686, + "loss": 2.4973, + "theoretical_loss": 3.353791504553154, + "tokens_seen": 2642018304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020284848484848487, + "loss": 2.3825, + "theoretical_loss": 3.3537849419231764, + "tokens_seen": 2642083840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020282828282828282, + "loss": 2.4526, + "theoretical_loss": 3.3537783795015588, + "tokens_seen": 2642149376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002028080808080808, + "loss": 2.6922, + "theoretical_loss": 3.353771817288289, + "tokens_seen": 2642214912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002027878787878788, + "loss": 2.6442, + "theoretical_loss": 3.3537652552833563, + "tokens_seen": 2642280448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020276767676767678, + "loss": 2.5793, + "theoretical_loss": 3.3537586934867476, + "tokens_seen": 2642345984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020274747474747476, + "loss": 2.6222, + "theoretical_loss": 3.3537521318984522, + "tokens_seen": 2642411520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020272727272727274, + "loss": 2.4572, + "theoretical_loss": 3.3537455705184573, + "tokens_seen": 2642477056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002027070707070707, + "loss": 2.4538, + "theoretical_loss": 3.3537390093467523, + "tokens_seen": 2642542592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020268686868686868, + "loss": 2.5416, + "theoretical_loss": 3.3537324483833246, + "tokens_seen": 2642608128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020266666666666667, + "loss": 2.6398, + "theoretical_loss": 3.353725887628163, + "tokens_seen": 2642673664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020264646464646465, + "loss": 2.4549, + "theoretical_loss": 3.3537193270812544, + "tokens_seen": 2642739200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020262626262626264, + "loss": 2.5198, + "theoretical_loss": 3.3537127667425892, + "tokens_seen": 2642804736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002026060606060606, + "loss": 2.3483, + "theoretical_loss": 3.353706206612154, + "tokens_seen": 2642870272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002025858585858586, + "loss": 2.5218, + "theoretical_loss": 3.353699646689937, + "tokens_seen": 2642935808 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1487881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.594163656234741, + "objective/train/theoretical_loss": 3.353693086975927, + "objective/train/tokens_used": 1001860576, + "theoretical_loss": 3.353693086975927, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020256565656565656, + "loss": 2.6457, + "theoretical_loss": 3.353693086975927, + "tokens_seen": 2643001344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020254545454545457, + "loss": 2.3977, + "theoretical_loss": 3.3536865274701126, + "tokens_seen": 2643066880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020252525252525253, + "loss": 2.4793, + "theoretical_loss": 3.3536799681724814, + "tokens_seen": 2643132416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020250505050505049, + "loss": 2.244, + "theoretical_loss": 3.3536734090830214, + "tokens_seen": 2643197952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024848484848485, + "loss": 2.8157, + "theoretical_loss": 3.3536668502017215, + "tokens_seen": 2643263488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020246464646464645, + "loss": 2.5968, + "theoretical_loss": 3.35366029152857, + "tokens_seen": 2643329024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020244444444444447, + "loss": 2.53, + "theoretical_loss": 3.353653733063554, + "tokens_seen": 2643394560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020242424242424242, + "loss": 2.5884, + "theoretical_loss": 3.353647174806663, + "tokens_seen": 2643460096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002024040404040404, + "loss": 2.5783, + "theoretical_loss": 3.3536406167578843, + "tokens_seen": 2643525632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023838383838384, + "loss": 2.531, + "theoretical_loss": 3.353634058917207, + "tokens_seen": 2643591168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020236363636363637, + "loss": 2.8343, + "theoretical_loss": 3.353627501284619, + "tokens_seen": 2643656704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020234343434343436, + "loss": 2.6475, + "theoretical_loss": 3.353620943860108, + "tokens_seen": 2643722240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020232323232323231, + "loss": 2.5172, + "theoretical_loss": 3.353614386643663, + "tokens_seen": 2643787776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002023030303030303, + "loss": 2.4751, + "theoretical_loss": 3.353607829635272, + "tokens_seen": 2643853312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020228282828282828, + "loss": 2.6409, + "theoretical_loss": 3.3536012728349234, + "tokens_seen": 2643918848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020226262626262627, + "loss": 2.3888, + "theoretical_loss": 3.3535947162426045, + "tokens_seen": 2643984384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020224242424242425, + "loss": 2.5536, + "theoretical_loss": 3.353588159858305, + "tokens_seen": 2644049920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020222222222222223, + "loss": 2.3735, + "theoretical_loss": 3.353581603682012, + "tokens_seen": 2644115456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002022020202020202, + "loss": 2.613, + "theoretical_loss": 3.353575047713714, + "tokens_seen": 2644180992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020218181818181818, + "loss": 2.5191, + "theoretical_loss": 3.3535684919534, + "tokens_seen": 2644246528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020216161616161616, + "loss": 2.7095, + "theoretical_loss": 3.353561936401057, + "tokens_seen": 2644312064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020214141414141414, + "loss": 2.4741, + "theoretical_loss": 3.3535553810566743, + "tokens_seen": 2644377600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020212121212121213, + "loss": 2.4621, + "theoretical_loss": 3.3535488259202397, + "tokens_seen": 2644443136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002021010101010101, + "loss": 2.301, + "theoretical_loss": 3.3535422709917415, + "tokens_seen": 2644508672 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002020808080808081, + "loss": 2.641, + "theoretical_loss": 3.3535357162711676, + "tokens_seen": 2644574208 + }, + { + "debugging/Self-BLEU-5": 0.44043301378291366, + "debugging/distinct-1-grams": 0.8028592116073892, + "debugging/distinct-2-grams": 0.9709771731359748, + "debugging/entropy-1-grams": 5.991297910662675, + "debugging/entropy-2-grams": 6.773013872721587, + "debugging/length": 450.2142857142857, + "debugging/num_segments": 14, + "epoch": 0.6, + "objective/train/docs_used": 1488919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.387479543685913, + "objective/train/theoretical_loss": 3.353529161758507, + "objective/train/tokens_used": 1003498976, + "theoretical_loss": 3.353529161758507, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020206060606060605, + "loss": 2.5577, + "theoretical_loss": 3.353529161758507, + "tokens_seen": 2644639744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020204040404040406, + "loss": 2.5186, + "theoretical_loss": 3.3535226074537476, + "tokens_seen": 2644705280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020202020202020202, + "loss": 2.5868, + "theoretical_loss": 3.353516053356877, + "tokens_seen": 2644770816 + }, + { + "epoch": 0.6, + "learning_rate": 0.000202, + "loss": 2.5906, + "theoretical_loss": 3.3535094994678847, + "tokens_seen": 2644836352 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201979797979798, + "loss": 2.6755, + "theoretical_loss": 3.353502945786758, + "tokens_seen": 2644901888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020195959595959594, + "loss": 2.6454, + "theoretical_loss": 3.353496392313486, + "tokens_seen": 2644967424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020193939393939396, + "loss": 2.8038, + "theoretical_loss": 3.3534898390480556, + "tokens_seen": 2645032960 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002019191919191919, + "loss": 2.6353, + "theoretical_loss": 3.353483285990457, + "tokens_seen": 2645098496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020189898989898992, + "loss": 2.4358, + "theoretical_loss": 3.3534767331406763, + "tokens_seen": 2645164032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020187878787878788, + "loss": 2.6454, + "theoretical_loss": 3.3534701804987033, + "tokens_seen": 2645229568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020185858585858586, + "loss": 2.4914, + "theoretical_loss": 3.3534636280645254, + "tokens_seen": 2645295104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020183838383838385, + "loss": 2.6173, + "theoretical_loss": 3.353457075838132, + "tokens_seen": 2645360640 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002018181818181818, + "loss": 2.4305, + "theoretical_loss": 3.35345052381951, + "tokens_seen": 2645426176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020179797979797982, + "loss": 2.7286, + "theoretical_loss": 3.353443972008648, + "tokens_seen": 2645491712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020177777777777777, + "loss": 2.2741, + "theoretical_loss": 3.353437420405535, + "tokens_seen": 2645557248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020175757575757576, + "loss": 2.7439, + "theoretical_loss": 3.3534308690101584, + "tokens_seen": 2645622784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020173737373737374, + "loss": 2.6065, + "theoretical_loss": 3.3534243178225074, + "tokens_seen": 2645688320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020171717171717173, + "loss": 2.643, + "theoretical_loss": 3.3534177668425698, + "tokens_seen": 2645753856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016969696969697, + "loss": 2.7436, + "theoretical_loss": 3.353411216070333, + "tokens_seen": 2645819392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002016767676767677, + "loss": 2.7502, + "theoretical_loss": 3.3534046655057868, + "tokens_seen": 2645884928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020165656565656565, + "loss": 2.6484, + "theoretical_loss": 3.3533981151489187, + "tokens_seen": 2645950464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020163636363636363, + "loss": 2.812, + "theoretical_loss": 3.3533915649997166, + "tokens_seen": 2646016000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020161616161616162, + "loss": 2.4569, + "theoretical_loss": 3.3533850150581697, + "tokens_seen": 2646081536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015959595959596, + "loss": 2.5635, + "theoretical_loss": 3.3533784653242655, + "tokens_seen": 2646147072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020157575757575759, + "loss": 2.6124, + "theoretical_loss": 3.353371915797992, + "tokens_seen": 2646212608 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1489355, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.595426321029663, + "objective/train/theoretical_loss": 3.3533653664793386, + "objective/train/tokens_used": 1005137376, + "theoretical_loss": 3.3533653664793386, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020155555555555554, + "loss": 2.5675, + "theoretical_loss": 3.3533653664793386, + "tokens_seen": 2646278144 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020153535353535355, + "loss": 2.4159, + "theoretical_loss": 3.353358817368293, + "tokens_seen": 2646343680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002015151515151515, + "loss": 2.4195, + "theoretical_loss": 3.3533522684648434, + "tokens_seen": 2646409216 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020149494949494952, + "loss": 2.746, + "theoretical_loss": 3.353345719768978, + "tokens_seen": 2646474752 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020147474747474748, + "loss": 2.4875, + "theoretical_loss": 3.3533391712806857, + "tokens_seen": 2646540288 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020145454545454544, + "loss": 2.551, + "theoretical_loss": 3.353332622999954, + "tokens_seen": 2646605824 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020143434343434345, + "loss": 2.7939, + "theoretical_loss": 3.3533260749267715, + "tokens_seen": 2646671360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002014141414141414, + "loss": 2.6423, + "theoretical_loss": 3.3533195270611262, + "tokens_seen": 2646736896 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020139393939393941, + "loss": 2.4688, + "theoretical_loss": 3.353312979403007, + "tokens_seen": 2646802432 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020137373737373737, + "loss": 2.4928, + "theoretical_loss": 3.3533064319524017, + "tokens_seen": 2646867968 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020135353535353536, + "loss": 2.5253, + "theoretical_loss": 3.353299884709299, + "tokens_seen": 2646933504 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020133333333333334, + "loss": 2.5542, + "theoretical_loss": 3.3532933376736866, + "tokens_seen": 2646999040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002013131313131313, + "loss": 2.6178, + "theoretical_loss": 3.3532867908455533, + "tokens_seen": 2647064576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002012929292929293, + "loss": 2.5531, + "theoretical_loss": 3.353280244224887, + "tokens_seen": 2647130112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020127272727272726, + "loss": 2.5359, + "theoretical_loss": 3.353273697811676, + "tokens_seen": 2647195648 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020125252525252528, + "loss": 2.757, + "theoretical_loss": 3.353267151605909, + "tokens_seen": 2647261184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020123232323232323, + "loss": 2.5961, + "theoretical_loss": 3.3532606056075744, + "tokens_seen": 2647326720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020121212121212122, + "loss": 2.6251, + "theoretical_loss": 3.35325405981666, + "tokens_seen": 2647392256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002011919191919192, + "loss": 2.5091, + "theoretical_loss": 3.3532475142331535, + "tokens_seen": 2647457792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020117171717171718, + "loss": 2.6101, + "theoretical_loss": 3.353240968857045, + "tokens_seen": 2647523328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020115151515151517, + "loss": 2.6297, + "theoretical_loss": 3.353234423688321, + "tokens_seen": 2647588864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020113131313131312, + "loss": 2.5858, + "theoretical_loss": 3.3532278787269707, + "tokens_seen": 2647654400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002011111111111111, + "loss": 2.3544, + "theoretical_loss": 3.353221333972982, + "tokens_seen": 2647719936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002010909090909091, + "loss": 2.4541, + "theoretical_loss": 3.353214789426344, + "tokens_seen": 2647785472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020107070707070708, + "loss": 2.6628, + "theoretical_loss": 3.3532082450870444, + "tokens_seen": 2647851008 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1489706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.566333293914795, + "objective/train/theoretical_loss": 3.353201700955071, + "objective/train/tokens_used": 1006775776, + "theoretical_loss": 3.353201700955071, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020105050505050506, + "loss": 2.5063, + "theoretical_loss": 3.353201700955071, + "tokens_seen": 2647916544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020103030303030304, + "loss": 2.5684, + "theoretical_loss": 3.353195157030413, + "tokens_seen": 2647982080 + }, + { + "epoch": 0.6, + "learning_rate": 0.000201010101010101, + "loss": 2.3563, + "theoretical_loss": 3.3531886133130584, + "tokens_seen": 2648047616 + }, + { + "epoch": 0.6, + "learning_rate": 0.000200989898989899, + "loss": 2.3629, + "theoretical_loss": 3.3531820698029953, + "tokens_seen": 2648113152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020096969696969697, + "loss": 2.4262, + "theoretical_loss": 3.353175526500212, + "tokens_seen": 2648178688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020094949494949495, + "loss": 2.6726, + "theoretical_loss": 3.353168983404697, + "tokens_seen": 2648244224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020092929292929294, + "loss": 2.6908, + "theoretical_loss": 3.3531624405164386, + "tokens_seen": 2648309760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002009090909090909, + "loss": 2.4935, + "theoretical_loss": 3.353155897835425, + "tokens_seen": 2648375296 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002008888888888889, + "loss": 2.7062, + "theoretical_loss": 3.3531493553616447, + "tokens_seen": 2648440832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020086868686868686, + "loss": 2.3062, + "theoretical_loss": 3.3531428130950856, + "tokens_seen": 2648506368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020084848484848487, + "loss": 2.6725, + "theoretical_loss": 3.353136271035736, + "tokens_seen": 2648571904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020082828282828283, + "loss": 2.5726, + "theoretical_loss": 3.353129729183585, + "tokens_seen": 2648637440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020080808080808081, + "loss": 2.8579, + "theoretical_loss": 3.3531231875386207, + "tokens_seen": 2648702976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007878787878788, + "loss": 2.3879, + "theoretical_loss": 3.3531166461008306, + "tokens_seen": 2648768512 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020076767676767675, + "loss": 2.4863, + "theoretical_loss": 3.3531101048702037, + "tokens_seen": 2648834048 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020074747474747477, + "loss": 2.6063, + "theoretical_loss": 3.353103563846728, + "tokens_seen": 2648899584 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020072727272727272, + "loss": 2.3662, + "theoretical_loss": 3.353097023030392, + "tokens_seen": 2648965120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002007070707070707, + "loss": 2.5388, + "theoretical_loss": 3.353090482421184, + "tokens_seen": 2649030656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002006868686868687, + "loss": 2.5477, + "theoretical_loss": 3.3530839420190923, + "tokens_seen": 2649096192 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020066666666666667, + "loss": 2.5496, + "theoretical_loss": 3.353077401824105, + "tokens_seen": 2649161728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020064646464646466, + "loss": 2.4695, + "theoretical_loss": 3.353070861836211, + "tokens_seen": 2649227264 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020062626262626264, + "loss": 2.7254, + "theoretical_loss": 3.3530643220553977, + "tokens_seen": 2649292800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002006060606060606, + "loss": 2.5086, + "theoretical_loss": 3.3530577824816543, + "tokens_seen": 2649358336 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020058585858585858, + "loss": 2.5729, + "theoretical_loss": 3.353051243114969, + "tokens_seen": 2649423872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020056565656565657, + "loss": 2.4475, + "theoretical_loss": 3.3530447039553293, + "tokens_seen": 2649489408 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1491205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9683759212493896, + "objective/train/theoretical_loss": 3.3530381650027246, + "objective/train/tokens_used": 1008414176, + "theoretical_loss": 3.3530381650027246, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020054545454545455, + "loss": 2.6565, + "theoretical_loss": 3.3530381650027246, + "tokens_seen": 2649554944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020052525252525254, + "loss": 2.4266, + "theoretical_loss": 3.3530316262571427, + "tokens_seen": 2649620480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002005050505050505, + "loss": 2.6577, + "theoretical_loss": 3.3530250877185717, + "tokens_seen": 2649686016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004848484848485, + "loss": 2.5316, + "theoretical_loss": 3.3530185493870004, + "tokens_seen": 2649751552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020046464646464646, + "loss": 2.4994, + "theoretical_loss": 3.3530120112624173, + "tokens_seen": 2649817088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020044444444444444, + "loss": 2.5975, + "theoretical_loss": 3.3530054733448096, + "tokens_seen": 2649882624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020042424242424243, + "loss": 2.7333, + "theoretical_loss": 3.352998935634167, + "tokens_seen": 2649948160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002004040404040404, + "loss": 2.7093, + "theoretical_loss": 3.352992398130477, + "tokens_seen": 2650013696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002003838383838384, + "loss": 2.4037, + "theoretical_loss": 3.352985860833728, + "tokens_seen": 2650079232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020036363636363635, + "loss": 2.6057, + "theoretical_loss": 3.3529793237439085, + "tokens_seen": 2650144768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020034343434343436, + "loss": 2.6671, + "theoretical_loss": 3.352972786861007, + "tokens_seen": 2650210304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020032323232323232, + "loss": 2.5809, + "theoretical_loss": 3.3529662501850117, + "tokens_seen": 2650275840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020030303030303033, + "loss": 2.5393, + "theoretical_loss": 3.3529597137159106, + "tokens_seen": 2650341376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002828282828283, + "loss": 2.4986, + "theoretical_loss": 3.352953177453693, + "tokens_seen": 2650406912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020026262626262625, + "loss": 2.6188, + "theoretical_loss": 3.3529466413983458, + "tokens_seen": 2650472448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020024242424242426, + "loss": 2.6327, + "theoretical_loss": 3.352940105549858, + "tokens_seen": 2650537984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002002222222222222, + "loss": 2.7297, + "theoretical_loss": 3.3529335699082186, + "tokens_seen": 2650603520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020020202020202022, + "loss": 2.6606, + "theoretical_loss": 3.3529270344734154, + "tokens_seen": 2650669056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020018181818181818, + "loss": 2.5355, + "theoretical_loss": 3.3529204992454362, + "tokens_seen": 2650734592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020016161616161616, + "loss": 2.5159, + "theoretical_loss": 3.3529139642242702, + "tokens_seen": 2650800128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020014141414141415, + "loss": 2.5167, + "theoretical_loss": 3.352907429409905, + "tokens_seen": 2650865664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020012121212121213, + "loss": 2.6265, + "theoretical_loss": 3.35290089480233, + "tokens_seen": 2650931200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020010101010101012, + "loss": 2.7591, + "theoretical_loss": 3.352894360401532, + "tokens_seen": 2650996736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020008080808080807, + "loss": 2.7279, + "theoretical_loss": 3.352887826207501, + "tokens_seen": 2651062272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020006060606060606, + "loss": 2.738, + "theoretical_loss": 3.3528812922202245, + "tokens_seen": 2651127808 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1491801, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1060407161712646, + "objective/train/theoretical_loss": 3.3528747584396905, + "objective/train/tokens_used": 1010052576, + "theoretical_loss": 3.3528747584396905, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020004040404040404, + "loss": 2.4112, + "theoretical_loss": 3.3528747584396905, + "tokens_seen": 2651193344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00020002020202020203, + "loss": 2.6183, + "theoretical_loss": 3.352868224865888, + "tokens_seen": 2651258880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0002, + "loss": 2.4389, + "theoretical_loss": 3.352861691498805, + "tokens_seen": 2651324416 + }, + { + "epoch": 0.6, + "learning_rate": 0.000199979797979798, + "loss": 2.5282, + "theoretical_loss": 3.3528551583384303, + "tokens_seen": 2651389952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019995959595959595, + "loss": 2.424, + "theoretical_loss": 3.352848625384752, + "tokens_seen": 2651455488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019993939393939396, + "loss": 2.1435, + "theoretical_loss": 3.352842092637758, + "tokens_seen": 2651521024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019991919191919192, + "loss": 2.5879, + "theoretical_loss": 3.352835560097437, + "tokens_seen": 2651586560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001998989898989899, + "loss": 2.5584, + "theoretical_loss": 3.352829027763778, + "tokens_seen": 2651652096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019987878787878789, + "loss": 2.3754, + "theoretical_loss": 3.352822495636768, + "tokens_seen": 2651717632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019985858585858584, + "loss": 2.2722, + "theoretical_loss": 3.352815963716396, + "tokens_seen": 2651783168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019983838383838385, + "loss": 2.5567, + "theoretical_loss": 3.3528094320026507, + "tokens_seen": 2651848704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001998181818181818, + "loss": 2.5045, + "theoretical_loss": 3.3528029004955204, + "tokens_seen": 2651914240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019979797979797982, + "loss": 2.795, + "theoretical_loss": 3.352796369194993, + "tokens_seen": 2651979776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019977777777777778, + "loss": 2.4971, + "theoretical_loss": 3.3527898381010575, + "tokens_seen": 2652045312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019975757575757574, + "loss": 2.679, + "theoretical_loss": 3.352783307213701, + "tokens_seen": 2652110848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019973737373737375, + "loss": 2.3617, + "theoretical_loss": 3.352776776532914, + "tokens_seen": 2652176384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001997171717171717, + "loss": 2.4373, + "theoretical_loss": 3.352770246058683, + "tokens_seen": 2652241920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019969696969696971, + "loss": 2.6209, + "theoretical_loss": 3.3527637157909966, + "tokens_seen": 2652307456 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019967676767676767, + "loss": 2.5625, + "theoretical_loss": 3.352757185729844, + "tokens_seen": 2652372992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019965656565656566, + "loss": 2.7028, + "theoretical_loss": 3.3527506558752127, + "tokens_seen": 2652438528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019963636363636364, + "loss": 2.4883, + "theoretical_loss": 3.352744126227092, + "tokens_seen": 2652504064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019961616161616162, + "loss": 2.4565, + "theoretical_loss": 3.3527375967854693, + "tokens_seen": 2652569600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0001995959595959596, + "loss": 2.4017, + "theoretical_loss": 3.3527310675503337, + "tokens_seen": 2652635136 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019957575757575756, + "loss": 2.4386, + "theoretical_loss": 3.352724538521673, + "tokens_seen": 2652700672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019955555555555558, + "loss": 2.5011, + "theoretical_loss": 3.352718009699476, + "tokens_seen": 2652766208 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 1493158, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6343905925750732, + "objective/train/theoretical_loss": 3.352711481083731, + "objective/train/tokens_used": 1011690976, + "theoretical_loss": 3.352711481083731, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00019953535353535353, + "loss": 2.5465, + "theoretical_loss": 3.352711481083731, + "tokens_seen": 2652831744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019951515151515152, + "loss": 2.5914, + "theoretical_loss": 3.3527049526744257, + "tokens_seen": 2652897280 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001994949494949495, + "loss": 2.4392, + "theoretical_loss": 3.3526984244715496, + "tokens_seen": 2652962816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019947474747474748, + "loss": 2.2709, + "theoretical_loss": 3.3526918964750907, + "tokens_seen": 2653028352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019945454545454547, + "loss": 2.4115, + "theoretical_loss": 3.352685368685037, + "tokens_seen": 2653093888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019943434343434345, + "loss": 2.7514, + "theoretical_loss": 3.3526788411013766, + "tokens_seen": 2653159424 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001994141414141414, + "loss": 2.657, + "theoretical_loss": 3.352672313724099, + "tokens_seen": 2653224960 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001993939393939394, + "loss": 2.5393, + "theoretical_loss": 3.3526657865531915, + "tokens_seen": 2653290496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019937373737373738, + "loss": 2.4226, + "theoretical_loss": 3.352659259588643, + "tokens_seen": 2653356032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019935353535353536, + "loss": 2.5093, + "theoretical_loss": 3.352652732830442, + "tokens_seen": 2653421568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019933333333333334, + "loss": 2.6008, + "theoretical_loss": 3.3526462062785765, + "tokens_seen": 2653487104 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001993131313131313, + "loss": 2.5832, + "theoretical_loss": 3.352639679933035, + "tokens_seen": 2653552640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992929292929293, + "loss": 2.7859, + "theoretical_loss": 3.352633153793806, + "tokens_seen": 2653618176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019927272727272727, + "loss": 2.7814, + "theoretical_loss": 3.352626627860878, + "tokens_seen": 2653683712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019925252525252528, + "loss": 2.3467, + "theoretical_loss": 3.3526201021342388, + "tokens_seen": 2653749248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019923232323232324, + "loss": 2.3305, + "theoretical_loss": 3.3526135766138774, + "tokens_seen": 2653814784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001992121212121212, + "loss": 2.5957, + "theoretical_loss": 3.352607051299782, + "tokens_seen": 2653880320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001991919191919192, + "loss": 2.4173, + "theoretical_loss": 3.352600526191941, + "tokens_seen": 2653945856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019917171717171716, + "loss": 2.5431, + "theoretical_loss": 3.352594001290343, + "tokens_seen": 2654011392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019915151515151517, + "loss": 2.5976, + "theoretical_loss": 3.3525874765949752, + "tokens_seen": 2654076928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019913131313131313, + "loss": 2.5249, + "theoretical_loss": 3.3525809521058276, + "tokens_seen": 2654142464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019911111111111111, + "loss": 2.3668, + "theoretical_loss": 3.352574427822888, + "tokens_seen": 2654208000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001990909090909091, + "loss": 2.5079, + "theoretical_loss": 3.352567903746144, + "tokens_seen": 2654273536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019907070707070708, + "loss": 2.5722, + "theoretical_loss": 3.352561379875586, + "tokens_seen": 2654339072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019905050505050507, + "loss": 2.4793, + "theoretical_loss": 3.3525548562112, + "tokens_seen": 2654404608 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1493841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2445859909057617, + "objective/train/theoretical_loss": 3.3525483327529755, + "objective/train/tokens_used": 1013329376, + "theoretical_loss": 3.3525483327529755, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019903030303030302, + "loss": 2.6593, + "theoretical_loss": 3.3525483327529755, + "tokens_seen": 2654470144 + }, + { + "epoch": 0.61, + "learning_rate": 0.000199010101010101, + "loss": 2.4346, + "theoretical_loss": 3.3525418095009014, + "tokens_seen": 2654535680 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198989898989899, + "loss": 2.5016, + "theoretical_loss": 3.352535286454965, + "tokens_seen": 2654601216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019896969696969697, + "loss": 2.4687, + "theoretical_loss": 3.352528763615156, + "tokens_seen": 2654666752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019894949494949496, + "loss": 2.5013, + "theoretical_loss": 3.3525222409814615, + "tokens_seen": 2654732288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019892929292929294, + "loss": 2.4508, + "theoretical_loss": 3.3525157185538705, + "tokens_seen": 2654797824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001989090909090909, + "loss": 2.5435, + "theoretical_loss": 3.3525091963323717, + "tokens_seen": 2654863360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019888888888888888, + "loss": 2.499, + "theoretical_loss": 3.352502674316953, + "tokens_seen": 2654928896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019886868686868687, + "loss": 2.7157, + "theoretical_loss": 3.352496152507603, + "tokens_seen": 2654994432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019884848484848485, + "loss": 2.6212, + "theoretical_loss": 3.35248963090431, + "tokens_seen": 2655059968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019882828282828284, + "loss": 2.5758, + "theoretical_loss": 3.3524831095070624, + "tokens_seen": 2655125504 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001988080808080808, + "loss": 2.5476, + "theoretical_loss": 3.3524765883158487, + "tokens_seen": 2655191040 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001987878787878788, + "loss": 2.4787, + "theoretical_loss": 3.3524700673306573, + "tokens_seen": 2655256576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019876767676767676, + "loss": 2.4462, + "theoretical_loss": 3.3524635465514767, + "tokens_seen": 2655322112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019874747474747477, + "loss": 2.3716, + "theoretical_loss": 3.352457025978295, + "tokens_seen": 2655387648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019872727272727273, + "loss": 2.3157, + "theoretical_loss": 3.3524505056111007, + "tokens_seen": 2655453184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019870707070707068, + "loss": 2.4715, + "theoretical_loss": 3.3524439854498826, + "tokens_seen": 2655518720 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001986868686868687, + "loss": 2.5484, + "theoretical_loss": 3.352437465494629, + "tokens_seen": 2655584256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019866666666666665, + "loss": 2.5363, + "theoretical_loss": 3.352430945745328, + "tokens_seen": 2655649792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019864646464646466, + "loss": 2.5703, + "theoretical_loss": 3.352424426201968, + "tokens_seen": 2655715328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019862626262626262, + "loss": 2.5066, + "theoretical_loss": 3.3524179068645377, + "tokens_seen": 2655780864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019860606060606063, + "loss": 2.4605, + "theoretical_loss": 3.3524113877330253, + "tokens_seen": 2655846400 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985858585858586, + "loss": 2.3306, + "theoretical_loss": 3.3524048688074193, + "tokens_seen": 2655911936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019856565656565657, + "loss": 2.5908, + "theoretical_loss": 3.352398350087708, + "tokens_seen": 2655977472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019854545454545456, + "loss": 2.501, + "theoretical_loss": 3.35239183157388, + "tokens_seen": 2656043008 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1495048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3419017791748047, + "objective/train/theoretical_loss": 3.352385313265924, + "objective/train/tokens_used": 1014967776, + "theoretical_loss": 3.352385313265924, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001985252525252525, + "loss": 2.3174, + "theoretical_loss": 3.352385313265924, + "tokens_seen": 2656108544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019850505050505052, + "loss": 2.5774, + "theoretical_loss": 3.3523787951638275, + "tokens_seen": 2656174080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019848484848484848, + "loss": 2.5961, + "theoretical_loss": 3.35237227726758, + "tokens_seen": 2656239616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019846464646464647, + "loss": 2.5046, + "theoretical_loss": 3.352365759577169, + "tokens_seen": 2656305152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019844444444444445, + "loss": 2.6536, + "theoretical_loss": 3.352359242092583, + "tokens_seen": 2656370688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019842424242424243, + "loss": 2.4829, + "theoretical_loss": 3.3523527248138114, + "tokens_seen": 2656436224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019840404040404042, + "loss": 2.7001, + "theoretical_loss": 3.3523462077408417, + "tokens_seen": 2656501760 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001983838383838384, + "loss": 2.6444, + "theoretical_loss": 3.352339690873663, + "tokens_seen": 2656567296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019836363636363636, + "loss": 2.7631, + "theoretical_loss": 3.352333174212263, + "tokens_seen": 2656632832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019834343434343434, + "loss": 2.6007, + "theoretical_loss": 3.35232665775663, + "tokens_seen": 2656698368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019832323232323233, + "loss": 2.573, + "theoretical_loss": 3.3523201415067536, + "tokens_seen": 2656763904 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001983030303030303, + "loss": 2.4029, + "theoretical_loss": 3.352313625462621, + "tokens_seen": 2656829440 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001982828282828283, + "loss": 2.54, + "theoretical_loss": 3.3523071096242214, + "tokens_seen": 2656894976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019826262626262625, + "loss": 2.6602, + "theoretical_loss": 3.352300593991543, + "tokens_seen": 2656960512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019824242424242426, + "loss": 2.4029, + "theoretical_loss": 3.352294078564574, + "tokens_seen": 2657026048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019822222222222222, + "loss": 2.6486, + "theoretical_loss": 3.352287563343303, + "tokens_seen": 2657091584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019820202020202023, + "loss": 2.5979, + "theoretical_loss": 3.352281048327719, + "tokens_seen": 2657157120 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981818181818182, + "loss": 2.6269, + "theoretical_loss": 3.352274533517809, + "tokens_seen": 2657222656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019816161616161614, + "loss": 2.4998, + "theoretical_loss": 3.3522680189135627, + "tokens_seen": 2657288192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019814141414141415, + "loss": 2.4664, + "theoretical_loss": 3.3522615045149684, + "tokens_seen": 2657353728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001981212121212121, + "loss": 2.3184, + "theoretical_loss": 3.352254990322014, + "tokens_seen": 2657419264 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019810101010101012, + "loss": 2.5458, + "theoretical_loss": 3.3522484763346885, + "tokens_seen": 2657484800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019808080808080808, + "loss": 2.7486, + "theoretical_loss": 3.35224196255298, + "tokens_seen": 2657550336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019806060606060606, + "loss": 2.4651, + "theoretical_loss": 3.352235448976877, + "tokens_seen": 2657615872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019804040404040405, + "loss": 2.7647, + "theoretical_loss": 3.3522289356063677, + "tokens_seen": 2657681408 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1495638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3086888790130615, + "objective/train/theoretical_loss": 3.3522224224414408, + "objective/train/tokens_used": 1016606176, + "theoretical_loss": 3.3522224224414408, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.61, + "learning_rate": 0.000198020202020202, + "loss": 2.5061, + "theoretical_loss": 3.3522224224414408, + "tokens_seen": 2657746944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019800000000000002, + "loss": 2.4074, + "theoretical_loss": 3.352215909482085, + "tokens_seen": 2657812480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019797979797979797, + "loss": 2.3419, + "theoretical_loss": 3.352209396728288, + "tokens_seen": 2657878016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019795959595959596, + "loss": 2.429, + "theoretical_loss": 3.3522028841800395, + "tokens_seen": 2657943552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019793939393939394, + "loss": 2.7904, + "theoretical_loss": 3.3521963718373264, + "tokens_seen": 2658009088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019791919191919192, + "loss": 2.6224, + "theoretical_loss": 3.3521898597001387, + "tokens_seen": 2658074624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001978989898989899, + "loss": 2.6827, + "theoretical_loss": 3.3521833477684635, + "tokens_seen": 2658140160 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001978787878787879, + "loss": 2.6094, + "theoretical_loss": 3.3521768360422897, + "tokens_seen": 2658205696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019785858585858585, + "loss": 2.2507, + "theoretical_loss": 3.352170324521606, + "tokens_seen": 2658271232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019783838383838383, + "loss": 2.3414, + "theoretical_loss": 3.352163813206401, + "tokens_seen": 2658336768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019781818181818182, + "loss": 2.6964, + "theoretical_loss": 3.3521573020966624, + "tokens_seen": 2658402304 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977979797979798, + "loss": 2.6763, + "theoretical_loss": 3.3521507911923796, + "tokens_seen": 2658467840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019777777777777778, + "loss": 2.6573, + "theoretical_loss": 3.3521442804935404, + "tokens_seen": 2658533376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019775757575757577, + "loss": 2.6374, + "theoretical_loss": 3.352137770000133, + "tokens_seen": 2658598912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019773737373737375, + "loss": 2.5394, + "theoretical_loss": 3.352131259712147, + "tokens_seen": 2658664448 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001977171717171717, + "loss": 2.4921, + "theoretical_loss": 3.3521247496295694, + "tokens_seen": 2658729984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019769696969696972, + "loss": 2.4044, + "theoretical_loss": 3.3521182397523894, + "tokens_seen": 2658795520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019767676767676768, + "loss": 2.4398, + "theoretical_loss": 3.352111730080596, + "tokens_seen": 2658861056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019765656565656566, + "loss": 2.709, + "theoretical_loss": 3.3521052206141766, + "tokens_seen": 2658926592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019763636363636365, + "loss": 2.5744, + "theoretical_loss": 3.35209871135312, + "tokens_seen": 2658992128 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001976161616161616, + "loss": 2.499, + "theoretical_loss": 3.3520922022974156, + "tokens_seen": 2659057664 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001975959595959596, + "loss": 2.4799, + "theoretical_loss": 3.3520856934470507, + "tokens_seen": 2659123200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019757575757575757, + "loss": 2.6808, + "theoretical_loss": 3.352079184802014, + "tokens_seen": 2659188736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019755555555555558, + "loss": 2.4497, + "theoretical_loss": 3.352072676362294, + "tokens_seen": 2659254272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019753535353535354, + "loss": 2.5029, + "theoretical_loss": 3.3520661681278794, + "tokens_seen": 2659319808 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1497122, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3807220458984375, + "objective/train/theoretical_loss": 3.3520596600987584, + "objective/train/tokens_used": 1018244576, + "theoretical_loss": 3.3520596600987584, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019751515151515152, + "loss": 2.589, + "theoretical_loss": 3.3520596600987584, + "tokens_seen": 2659385344 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001974949494949495, + "loss": 2.359, + "theoretical_loss": 3.35205315227492, + "tokens_seen": 2659450880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019747474747474746, + "loss": 2.671, + "theoretical_loss": 3.352046644656352, + "tokens_seen": 2659516416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019745454545454547, + "loss": 2.7287, + "theoretical_loss": 3.352040137243043, + "tokens_seen": 2659581952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019743434343434343, + "loss": 2.6656, + "theoretical_loss": 3.3520336300349816, + "tokens_seen": 2659647488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019741414141414141, + "loss": 2.5502, + "theoretical_loss": 3.3520271230321566, + "tokens_seen": 2659713024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973939393939394, + "loss": 2.5635, + "theoretical_loss": 3.3520206162345554, + "tokens_seen": 2659778560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019737373737373738, + "loss": 2.5754, + "theoretical_loss": 3.352014109642168, + "tokens_seen": 2659844096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019735353535353537, + "loss": 2.5226, + "theoretical_loss": 3.3520076032549815, + "tokens_seen": 2659909632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019733333333333335, + "loss": 2.5691, + "theoretical_loss": 3.352001097072985, + "tokens_seen": 2659975168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001973131313131313, + "loss": 2.6796, + "theoretical_loss": 3.3519945910961675, + "tokens_seen": 2660040704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972929292929293, + "loss": 2.4798, + "theoretical_loss": 3.3519880853245163, + "tokens_seen": 2660106240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019727272727272728, + "loss": 2.3935, + "theoretical_loss": 3.3519815797580206, + "tokens_seen": 2660171776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019725252525252526, + "loss": 2.8728, + "theoretical_loss": 3.3519750743966688, + "tokens_seen": 2660237312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019723232323232324, + "loss": 2.8325, + "theoretical_loss": 3.3519685692404493, + "tokens_seen": 2660302848 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001972121212121212, + "loss": 2.6063, + "theoretical_loss": 3.3519620642893506, + "tokens_seen": 2660368384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971919191919192, + "loss": 2.6, + "theoretical_loss": 3.351955559543361, + "tokens_seen": 2660433920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019717171717171717, + "loss": 2.6448, + "theoretical_loss": 3.3519490550024695, + "tokens_seen": 2660499456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019715151515151515, + "loss": 2.3319, + "theoretical_loss": 3.3519425506666645, + "tokens_seen": 2660564992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019713131313131314, + "loss": 2.6446, + "theoretical_loss": 3.3519360465359336, + "tokens_seen": 2660630528 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001971111111111111, + "loss": 2.6313, + "theoretical_loss": 3.351929542610266, + "tokens_seen": 2660696064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001970909090909091, + "loss": 2.3787, + "theoretical_loss": 3.3519230388896504, + "tokens_seen": 2660761600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019707070707070706, + "loss": 2.8385, + "theoretical_loss": 3.351916535374075, + "tokens_seen": 2660827136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019705050505050507, + "loss": 2.6443, + "theoretical_loss": 3.351910032063528, + "tokens_seen": 2660892672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019703030303030303, + "loss": 2.4153, + "theoretical_loss": 3.351903528957998, + "tokens_seen": 2660958208 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1497834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.155775308609009, + "objective/train/theoretical_loss": 3.3518970260574736, + "objective/train/tokens_used": 1019882976, + "theoretical_loss": 3.3518970260574736, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.61, + "learning_rate": 0.000197010101010101, + "loss": 2.3751, + "theoretical_loss": 3.3518970260574736, + "tokens_seen": 2661023744 + }, + { + "epoch": 0.61, + "learning_rate": 0.000196989898989899, + "loss": 2.5545, + "theoretical_loss": 3.351890523361944, + "tokens_seen": 2661089280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019696969696969695, + "loss": 2.5546, + "theoretical_loss": 3.3518840208713967, + "tokens_seen": 2661154816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019694949494949496, + "loss": 2.4096, + "theoretical_loss": 3.35187751858582, + "tokens_seen": 2661220352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019692929292929292, + "loss": 2.493, + "theoretical_loss": 3.3518710165052035, + "tokens_seen": 2661285888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019690909090909093, + "loss": 2.5414, + "theoretical_loss": 3.351864514629535, + "tokens_seen": 2661351424 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001968888888888889, + "loss": 2.6815, + "theoretical_loss": 3.351858012958803, + "tokens_seen": 2661416960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019686868686868687, + "loss": 2.5877, + "theoretical_loss": 3.351851511492996, + "tokens_seen": 2661482496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019684848484848486, + "loss": 2.2215, + "theoretical_loss": 3.3518450102321027, + "tokens_seen": 2661548032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019682828282828284, + "loss": 2.5453, + "theoretical_loss": 3.3518385091761114, + "tokens_seen": 2661613568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019680808080808082, + "loss": 2.6484, + "theoretical_loss": 3.3518320083250104, + "tokens_seen": 2661679104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019678787878787878, + "loss": 2.5388, + "theoretical_loss": 3.3518255076787886, + "tokens_seen": 2661744640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019676767676767677, + "loss": 2.3922, + "theoretical_loss": 3.3518190072374345, + "tokens_seen": 2661810176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019674747474747475, + "loss": 2.7027, + "theoretical_loss": 3.3518125070009366, + "tokens_seen": 2661875712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019672727272727273, + "loss": 2.5223, + "theoretical_loss": 3.351806006969283, + "tokens_seen": 2661941248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019670707070707072, + "loss": 2.5915, + "theoretical_loss": 3.351799507142463, + "tokens_seen": 2662006784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966868686868687, + "loss": 2.6061, + "theoretical_loss": 3.351793007520464, + "tokens_seen": 2662072320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019666666666666666, + "loss": 2.3566, + "theoretical_loss": 3.3517865081032747, + "tokens_seen": 2662137856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019664646464646467, + "loss": 2.5973, + "theoretical_loss": 3.351780008890885, + "tokens_seen": 2662203392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019662626262626263, + "loss": 2.6056, + "theoretical_loss": 3.351773509883282, + "tokens_seen": 2662268928 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001966060606060606, + "loss": 2.4824, + "theoretical_loss": 3.351767011080454, + "tokens_seen": 2662334464 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001965858585858586, + "loss": 2.4674, + "theoretical_loss": 3.3517605124823904, + "tokens_seen": 2662400000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019656565656565655, + "loss": 2.586, + "theoretical_loss": 3.35175401408908, + "tokens_seen": 2662465536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019654545454545456, + "loss": 2.5321, + "theoretical_loss": 3.3517475159005103, + "tokens_seen": 2662531072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019652525252525252, + "loss": 2.5614, + "theoretical_loss": 3.35174101791667, + "tokens_seen": 2662596608 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1499033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.496255874633789, + "objective/train/theoretical_loss": 3.351734520137548, + "objective/train/tokens_used": 1021521376, + "theoretical_loss": 3.351734520137548, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019650505050505053, + "loss": 2.4944, + "theoretical_loss": 3.351734520137548, + "tokens_seen": 2662662144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001964848484848485, + "loss": 2.4695, + "theoretical_loss": 3.3517280225631327, + "tokens_seen": 2662727680 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019646464646464644, + "loss": 2.44, + "theoretical_loss": 3.3517215251934123, + "tokens_seen": 2662793216 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019644444444444445, + "loss": 2.7243, + "theoretical_loss": 3.351715028028376, + "tokens_seen": 2662858752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001964242424242424, + "loss": 2.569, + "theoretical_loss": 3.3517085310680117, + "tokens_seen": 2662924288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019640404040404042, + "loss": 2.535, + "theoretical_loss": 3.351702034312308, + "tokens_seen": 2662989824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019638383838383838, + "loss": 2.5151, + "theoretical_loss": 3.3516955377612536, + "tokens_seen": 2663055360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019636363636363636, + "loss": 2.6136, + "theoretical_loss": 3.351689041414837, + "tokens_seen": 2663120896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019634343434343435, + "loss": 2.3779, + "theoretical_loss": 3.3516825452730465, + "tokens_seen": 2663186432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019632323232323233, + "loss": 2.6657, + "theoretical_loss": 3.351676049335871, + "tokens_seen": 2663251968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019630303030303032, + "loss": 2.4269, + "theoretical_loss": 3.3516695536032985, + "tokens_seen": 2663317504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019628282828282827, + "loss": 2.6682, + "theoretical_loss": 3.351663058075318, + "tokens_seen": 2663383040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019626262626262626, + "loss": 2.5465, + "theoretical_loss": 3.3516565627519177, + "tokens_seen": 2663448576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019624242424242424, + "loss": 2.7041, + "theoretical_loss": 3.3516500676330865, + "tokens_seen": 2663514112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019622222222222222, + "loss": 2.4957, + "theoretical_loss": 3.3516435727188125, + "tokens_seen": 2663579648 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001962020202020202, + "loss": 2.4315, + "theoretical_loss": 3.3516370780090847, + "tokens_seen": 2663645184 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001961818181818182, + "loss": 2.5753, + "theoretical_loss": 3.3516305835038906, + "tokens_seen": 2663710720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019616161616161615, + "loss": 2.5453, + "theoretical_loss": 3.35162408920322, + "tokens_seen": 2663776256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019614141414141416, + "loss": 2.4388, + "theoretical_loss": 3.351617595107061, + "tokens_seen": 2663841792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019612121212121212, + "loss": 2.5822, + "theoretical_loss": 3.351611101215402, + "tokens_seen": 2663907328 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001961010101010101, + "loss": 2.6521, + "theoretical_loss": 3.3516046075282313, + "tokens_seen": 2663972864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019608080808080808, + "loss": 2.5068, + "theoretical_loss": 3.3515981140455375, + "tokens_seen": 2664038400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019606060606060607, + "loss": 2.4658, + "theoretical_loss": 3.35159162076731, + "tokens_seen": 2664103936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019604040404040405, + "loss": 2.3936, + "theoretical_loss": 3.3515851276935362, + "tokens_seen": 2664169472 + }, + { + "epoch": 0.61, + "learning_rate": 0.000196020202020202, + "loss": 2.3696, + "theoretical_loss": 3.351578634824205, + "tokens_seen": 2664235008 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1499572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.456423759460449, + "objective/train/theoretical_loss": 3.351572142159305, + "objective/train/tokens_used": 1023159776, + "theoretical_loss": 3.351572142159305, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019600000000000002, + "loss": 2.5511, + "theoretical_loss": 3.351572142159305, + "tokens_seen": 2664300544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019597979797979798, + "loss": 2.8039, + "theoretical_loss": 3.351565649698825, + "tokens_seen": 2664366080 + }, + { + "epoch": 0.61, + "learning_rate": 0.000195959595959596, + "loss": 2.5528, + "theoretical_loss": 3.351559157442753, + "tokens_seen": 2664431616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019593939393939395, + "loss": 2.5054, + "theoretical_loss": 3.351552665391078, + "tokens_seen": 2664497152 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001959191919191919, + "loss": 2.8125, + "theoretical_loss": 3.351546173543788, + "tokens_seen": 2664562688 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001958989898989899, + "loss": 2.5901, + "theoretical_loss": 3.3515396819008725, + "tokens_seen": 2664628224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019587878787878787, + "loss": 2.4352, + "theoretical_loss": 3.351533190462319, + "tokens_seen": 2664693760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019585858585858588, + "loss": 2.6375, + "theoretical_loss": 3.351526699228117, + "tokens_seen": 2664759296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019583838383838384, + "loss": 2.7312, + "theoretical_loss": 3.351520208198254, + "tokens_seen": 2664824832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019581818181818182, + "loss": 2.5061, + "theoretical_loss": 3.351513717372719, + "tokens_seen": 2664890368 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957979797979798, + "loss": 2.5117, + "theoretical_loss": 3.351507226751501, + "tokens_seen": 2664955904 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001957777777777778, + "loss": 2.724, + "theoretical_loss": 3.3515007363345877, + "tokens_seen": 2665021440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019575757575757577, + "loss": 2.6398, + "theoretical_loss": 3.3514942461219683, + "tokens_seen": 2665086976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019573737373737373, + "loss": 2.7609, + "theoretical_loss": 3.351487756113631, + "tokens_seen": 2665152512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019571717171717171, + "loss": 2.6772, + "theoretical_loss": 3.3514812663095648, + "tokens_seen": 2665218048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001956969696969697, + "loss": 2.6731, + "theoretical_loss": 3.3514747767097575, + "tokens_seen": 2665283584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019567676767676768, + "loss": 2.6967, + "theoretical_loss": 3.3514682873141988, + "tokens_seen": 2665349120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019565656565656567, + "loss": 2.4322, + "theoretical_loss": 3.351461798122876, + "tokens_seen": 2665414656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019563636363636365, + "loss": 2.4213, + "theoretical_loss": 3.351455309135778, + "tokens_seen": 2665480192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001956161616161616, + "loss": 2.6706, + "theoretical_loss": 3.351448820352894, + "tokens_seen": 2665545728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955959595959596, + "loss": 2.3782, + "theoretical_loss": 3.3514423317742117, + "tokens_seen": 2665611264 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019557575757575758, + "loss": 2.4778, + "theoretical_loss": 3.3514358433997202, + "tokens_seen": 2665676800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019555555555555556, + "loss": 2.3959, + "theoretical_loss": 3.351429355229408, + "tokens_seen": 2665742336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019553535353535354, + "loss": 2.6111, + "theoretical_loss": 3.3514228672632633, + "tokens_seen": 2665807872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001955151515151515, + "loss": 2.3541, + "theoretical_loss": 3.3514163795012752, + "tokens_seen": 2665873408 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1500816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9509613513946533, + "objective/train/theoretical_loss": 3.351409891943432, + "objective/train/tokens_used": 1024798176, + "theoretical_loss": 3.351409891943432, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001954949494949495, + "loss": 2.4801, + "theoretical_loss": 3.351409891943432, + "tokens_seen": 2665938944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019547474747474747, + "loss": 2.6344, + "theoretical_loss": 3.3514034045897216, + "tokens_seen": 2666004480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019545454545454548, + "loss": 2.4528, + "theoretical_loss": 3.351396917440134, + "tokens_seen": 2666070016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019543434343434344, + "loss": 2.5233, + "theoretical_loss": 3.3513904304946562, + "tokens_seen": 2666135552 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001954141414141414, + "loss": 2.4696, + "theoretical_loss": 3.351383943753278, + "tokens_seen": 2666201088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001953939393939394, + "loss": 2.6116, + "theoretical_loss": 3.351377457215987, + "tokens_seen": 2666266624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019537373737373736, + "loss": 2.6643, + "theoretical_loss": 3.351370970882773, + "tokens_seen": 2666332160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019535353535353537, + "loss": 2.6846, + "theoretical_loss": 3.3513644847536233, + "tokens_seen": 2666397696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019533333333333333, + "loss": 2.6677, + "theoretical_loss": 3.3513579988285267, + "tokens_seen": 2666463232 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001953131313131313, + "loss": 2.6459, + "theoretical_loss": 3.3513515131074723, + "tokens_seen": 2666528768 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001952929292929293, + "loss": 2.5501, + "theoretical_loss": 3.3513450275904484, + "tokens_seen": 2666594304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019527272727272728, + "loss": 2.74, + "theoretical_loss": 3.3513385422774435, + "tokens_seen": 2666659840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019525252525252526, + "loss": 2.6718, + "theoretical_loss": 3.3513320571684466, + "tokens_seen": 2666725376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019523232323232322, + "loss": 2.4956, + "theoretical_loss": 3.3513255722634456, + "tokens_seen": 2666790912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019521212121212123, + "loss": 2.4156, + "theoretical_loss": 3.351319087562429, + "tokens_seen": 2666856448 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001951919191919192, + "loss": 2.6876, + "theoretical_loss": 3.351312603065386, + "tokens_seen": 2666921984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019517171717171717, + "loss": 2.5881, + "theoretical_loss": 3.351306118772305, + "tokens_seen": 2666987520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019515151515151516, + "loss": 2.7249, + "theoretical_loss": 3.3512996346831745, + "tokens_seen": 2667053056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019513131313131314, + "loss": 2.7015, + "theoretical_loss": 3.351293150797983, + "tokens_seen": 2667118592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019511111111111113, + "loss": 2.3475, + "theoretical_loss": 3.351286667116719, + "tokens_seen": 2667184128 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001950909090909091, + "loss": 2.4122, + "theoretical_loss": 3.3512801836393713, + "tokens_seen": 2667249664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019507070707070707, + "loss": 2.6482, + "theoretical_loss": 3.3512737003659288, + "tokens_seen": 2667315200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019505050505050505, + "loss": 2.6403, + "theoretical_loss": 3.351267217296379, + "tokens_seen": 2667380736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019503030303030303, + "loss": 2.3862, + "theoretical_loss": 3.3512607344307113, + "tokens_seen": 2667446272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019501010101010102, + "loss": 2.3533, + "theoretical_loss": 3.351254251768914, + "tokens_seen": 2667511808 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1501537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3480489253997803, + "objective/train/theoretical_loss": 3.351247769310976, + "objective/train/tokens_used": 1026436576, + "theoretical_loss": 3.351247769310976, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.61, + "learning_rate": 0.000194989898989899, + "loss": 2.4167, + "theoretical_loss": 3.351247769310976, + "tokens_seen": 2667577344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019496969696969696, + "loss": 2.3824, + "theoretical_loss": 3.3512412870568857, + "tokens_seen": 2667642880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019494949494949497, + "loss": 2.6073, + "theoretical_loss": 3.351234805006632, + "tokens_seen": 2667708416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019492929292929293, + "loss": 2.7514, + "theoretical_loss": 3.3512283231602025, + "tokens_seen": 2667773952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019490909090909094, + "loss": 2.4936, + "theoretical_loss": 3.3512218415175865, + "tokens_seen": 2667839488 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001948888888888889, + "loss": 2.6473, + "theoretical_loss": 3.351215360078773, + "tokens_seen": 2667905024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019486868686868685, + "loss": 2.5471, + "theoretical_loss": 3.3512088788437495, + "tokens_seen": 2667970560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019484848484848486, + "loss": 2.5436, + "theoretical_loss": 3.3512023978125054, + "tokens_seen": 2668036096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019482828282828282, + "loss": 2.5531, + "theoretical_loss": 3.351195916985029, + "tokens_seen": 2668101632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019480808080808083, + "loss": 2.2862, + "theoretical_loss": 3.351189436361309, + "tokens_seen": 2668167168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001947878787878788, + "loss": 2.4102, + "theoretical_loss": 3.351182955941334, + "tokens_seen": 2668232704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019476767676767677, + "loss": 2.5771, + "theoretical_loss": 3.351176475725093, + "tokens_seen": 2668298240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019474747474747476, + "loss": 2.2986, + "theoretical_loss": 3.3511699957125733, + "tokens_seen": 2668363776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001947272727272727, + "loss": 2.4529, + "theoretical_loss": 3.3511635159037647, + "tokens_seen": 2668429312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019470707070707072, + "loss": 2.2897, + "theoretical_loss": 3.3511570362986554, + "tokens_seen": 2668494848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019468686868686868, + "loss": 2.4802, + "theoretical_loss": 3.3511505568972337, + "tokens_seen": 2668560384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019466666666666666, + "loss": 2.6443, + "theoretical_loss": 3.351144077699489, + "tokens_seen": 2668625920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019464646464646465, + "loss": 2.7174, + "theoretical_loss": 3.3511375987054093, + "tokens_seen": 2668691456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019462626262626263, + "loss": 2.6254, + "theoretical_loss": 3.3511311199149834, + "tokens_seen": 2668756992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019460606060606062, + "loss": 2.5148, + "theoretical_loss": 3.3511246413281994, + "tokens_seen": 2668822528 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001945858585858586, + "loss": 2.4861, + "theoretical_loss": 3.3511181629450464, + "tokens_seen": 2668888064 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019456565656565656, + "loss": 2.394, + "theoretical_loss": 3.351111684765513, + "tokens_seen": 2668953600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019454545454545454, + "loss": 2.356, + "theoretical_loss": 3.3511052067895877, + "tokens_seen": 2669019136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00019452525252525252, + "loss": 2.4455, + "theoretical_loss": 3.351098729017259, + "tokens_seen": 2669084672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001945050505050505, + "loss": 2.5161, + "theoretical_loss": 3.3510922514485157, + "tokens_seen": 2669150208 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 1503056, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.76729416847229, + "objective/train/theoretical_loss": 3.351085774083346, + "objective/train/tokens_used": 1028074976, + "theoretical_loss": 3.351085774083346, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.61, + "learning_rate": 0.0001944848484848485, + "loss": 2.5761, + "theoretical_loss": 3.351085774083346, + "tokens_seen": 2669215744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019446464646464645, + "loss": 2.7946, + "theoretical_loss": 3.351079296921739, + "tokens_seen": 2669281280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019444444444444446, + "loss": 2.5732, + "theoretical_loss": 3.3510728199636834, + "tokens_seen": 2669346816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019442424242424242, + "loss": 2.313, + "theoretical_loss": 3.351066343209167, + "tokens_seen": 2669412352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019440404040404043, + "loss": 2.6489, + "theoretical_loss": 3.3510598666581792, + "tokens_seen": 2669477888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019438383838383839, + "loss": 2.4131, + "theoretical_loss": 3.3510533903107085, + "tokens_seen": 2669543424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019436363636363637, + "loss": 2.6068, + "theoretical_loss": 3.3510469141667434, + "tokens_seen": 2669608960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019434343434343435, + "loss": 2.456, + "theoretical_loss": 3.351040438226272, + "tokens_seen": 2669674496 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001943232323232323, + "loss": 2.6451, + "theoretical_loss": 3.3510339624892835, + "tokens_seen": 2669740032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019430303030303032, + "loss": 2.4021, + "theoretical_loss": 3.3510274869557666, + "tokens_seen": 2669805568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019428282828282828, + "loss": 2.554, + "theoretical_loss": 3.3510210116257095, + "tokens_seen": 2669871104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001942626262626263, + "loss": 2.7096, + "theoretical_loss": 3.351014536499101, + "tokens_seen": 2669936640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019424242424242425, + "loss": 2.607, + "theoretical_loss": 3.3510080615759295, + "tokens_seen": 2670002176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019422222222222223, + "loss": 2.3443, + "theoretical_loss": 3.351001586856184, + "tokens_seen": 2670067712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019420202020202021, + "loss": 2.291, + "theoretical_loss": 3.350995112339853, + "tokens_seen": 2670133248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019418181818181817, + "loss": 2.3999, + "theoretical_loss": 3.3509886380269247, + "tokens_seen": 2670198784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019416161616161618, + "loss": 2.3983, + "theoretical_loss": 3.3509821639173887, + "tokens_seen": 2670264320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019414141414141414, + "loss": 2.2538, + "theoretical_loss": 3.3509756900112326, + "tokens_seen": 2670329856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019412121212121212, + "loss": 2.551, + "theoretical_loss": 3.3509692163084455, + "tokens_seen": 2670395392 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001941010101010101, + "loss": 2.4204, + "theoretical_loss": 3.350962742809016, + "tokens_seen": 2670460928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001940808080808081, + "loss": 2.5285, + "theoretical_loss": 3.3509562695129325, + "tokens_seen": 2670526464 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019406060606060607, + "loss": 2.7818, + "theoretical_loss": 3.350949796420184, + "tokens_seen": 2670592000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019404040404040406, + "loss": 2.3015, + "theoretical_loss": 3.3509433235307586, + "tokens_seen": 2670657536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019402020202020202, + "loss": 2.3477, + "theoretical_loss": 3.3509368508446453, + "tokens_seen": 2670723072 + }, + { + "epoch": 0.62, + "learning_rate": 0.000194, + "loss": 2.4528, + "theoretical_loss": 3.350930378361833, + "tokens_seen": 2670788608 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1503662, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4683971405029297, + "objective/train/theoretical_loss": 3.3509239060823095, + "objective/train/tokens_used": 1029713376, + "theoretical_loss": 3.3509239060823095, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019397979797979798, + "loss": 2.5015, + "theoretical_loss": 3.3509239060823095, + "tokens_seen": 2670854144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019395959595959597, + "loss": 2.469, + "theoretical_loss": 3.3509174340060643, + "tokens_seen": 2670919680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019393939393939395, + "loss": 2.4374, + "theoretical_loss": 3.3509109621330855, + "tokens_seen": 2670985216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001939191919191919, + "loss": 2.5136, + "theoretical_loss": 3.350904490463362, + "tokens_seen": 2671050752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019389898989898992, + "loss": 2.5206, + "theoretical_loss": 3.3508980189968818, + "tokens_seen": 2671116288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019387878787878788, + "loss": 2.7071, + "theoretical_loss": 3.3508915477336343, + "tokens_seen": 2671181824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019385858585858586, + "loss": 2.5306, + "theoretical_loss": 3.350885076673608, + "tokens_seen": 2671247360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019383838383838384, + "loss": 2.5063, + "theoretical_loss": 3.350878605816791, + "tokens_seen": 2671312896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001938181818181818, + "loss": 2.4523, + "theoretical_loss": 3.350872135163173, + "tokens_seen": 2671378432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001937979797979798, + "loss": 2.5712, + "theoretical_loss": 3.3508656647127415, + "tokens_seen": 2671443968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019377777777777777, + "loss": 2.5187, + "theoretical_loss": 3.3508591944654853, + "tokens_seen": 2671509504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019375757575757578, + "loss": 2.5454, + "theoretical_loss": 3.3508527244213937, + "tokens_seen": 2671575040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019373737373737374, + "loss": 2.5599, + "theoretical_loss": 3.350846254580455, + "tokens_seen": 2671640576 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019371717171717172, + "loss": 2.7312, + "theoretical_loss": 3.3508397849426577, + "tokens_seen": 2671706112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936969696969697, + "loss": 2.3382, + "theoretical_loss": 3.3508333155079906, + "tokens_seen": 2671771648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019367676767676766, + "loss": 2.6713, + "theoretical_loss": 3.3508268462764423, + "tokens_seen": 2671837184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019365656565656567, + "loss": 2.2998, + "theoretical_loss": 3.3508203772480014, + "tokens_seen": 2671902720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019363636363636363, + "loss": 2.7183, + "theoretical_loss": 3.3508139084226563, + "tokens_seen": 2671968256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001936161616161616, + "loss": 2.7112, + "theoretical_loss": 3.350807439800396, + "tokens_seen": 2672033792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001935959595959596, + "loss": 2.464, + "theoretical_loss": 3.3508009713812092, + "tokens_seen": 2672099328 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019357575757575758, + "loss": 2.5141, + "theoretical_loss": 3.3507945031650843, + "tokens_seen": 2672164864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019355555555555557, + "loss": 2.692, + "theoretical_loss": 3.3507880351520103, + "tokens_seen": 2672230400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019353535353535355, + "loss": 2.5518, + "theoretical_loss": 3.3507815673419756, + "tokens_seen": 2672295936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019351515151515153, + "loss": 2.4673, + "theoretical_loss": 3.3507750997349683, + "tokens_seen": 2672361472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001934949494949495, + "loss": 2.4511, + "theoretical_loss": 3.350768632330978, + "tokens_seen": 2672427008 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1505160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6477866172790527, + "objective/train/theoretical_loss": 3.350762165129993, + "objective/train/tokens_used": 1031351776, + "theoretical_loss": 3.350762165129993, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019347474747474747, + "loss": 2.5228, + "theoretical_loss": 3.350762165129993, + "tokens_seen": 2672492544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019345454545454546, + "loss": 2.5105, + "theoretical_loss": 3.3507556981320015, + "tokens_seen": 2672558080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019343434343434344, + "loss": 2.4682, + "theoretical_loss": 3.3507492313369927, + "tokens_seen": 2672623616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019341414141414143, + "loss": 2.4098, + "theoretical_loss": 3.3507427647449552, + "tokens_seen": 2672689152 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001933939393939394, + "loss": 2.5818, + "theoretical_loss": 3.3507362983558773, + "tokens_seen": 2672754688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019337373737373737, + "loss": 2.2549, + "theoretical_loss": 3.350729832169748, + "tokens_seen": 2672820224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019335353535353538, + "loss": 2.6363, + "theoretical_loss": 3.350723366186556, + "tokens_seen": 2672885760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019333333333333333, + "loss": 2.2632, + "theoretical_loss": 3.3507169004062893, + "tokens_seen": 2672951296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019331313131313132, + "loss": 2.443, + "theoretical_loss": 3.3507104348289376, + "tokens_seen": 2673016832 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001932929292929293, + "loss": 2.5915, + "theoretical_loss": 3.350703969454489, + "tokens_seen": 2673082368 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019327272727272726, + "loss": 2.4265, + "theoretical_loss": 3.350697504282932, + "tokens_seen": 2673147904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019325252525252527, + "loss": 2.5295, + "theoretical_loss": 3.3506910393142553, + "tokens_seen": 2673213440 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019323232323232323, + "loss": 2.7675, + "theoretical_loss": 3.3506845745484477, + "tokens_seen": 2673278976 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019321212121212124, + "loss": 2.6112, + "theoretical_loss": 3.3506781099854983, + "tokens_seen": 2673344512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001931919191919192, + "loss": 2.5162, + "theoretical_loss": 3.3506716456253947, + "tokens_seen": 2673410048 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019317171717171715, + "loss": 2.3789, + "theoretical_loss": 3.3506651814681265, + "tokens_seen": 2673475584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019315151515151516, + "loss": 2.4292, + "theoretical_loss": 3.350658717513682, + "tokens_seen": 2673541120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019313131313131312, + "loss": 2.6162, + "theoretical_loss": 3.35065225376205, + "tokens_seen": 2673606656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019311111111111113, + "loss": 2.7269, + "theoretical_loss": 3.3506457902132185, + "tokens_seen": 2673672192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001930909090909091, + "loss": 2.6016, + "theoretical_loss": 3.3506393268671775, + "tokens_seen": 2673737728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019307070707070707, + "loss": 2.7098, + "theoretical_loss": 3.3506328637239147, + "tokens_seen": 2673803264 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019305050505050506, + "loss": 2.5747, + "theoretical_loss": 3.3506264007834186, + "tokens_seen": 2673868800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019303030303030304, + "loss": 2.6594, + "theoretical_loss": 3.3506199380456785, + "tokens_seen": 2673934336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019301010101010102, + "loss": 2.6136, + "theoretical_loss": 3.350613475510683, + "tokens_seen": 2673999872 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019298989898989898, + "loss": 2.4956, + "theoretical_loss": 3.3506070131784202, + "tokens_seen": 2674065408 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1505818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8669445514678955, + "objective/train/theoretical_loss": 3.3506005510488794, + "objective/train/tokens_used": 1032990176, + "theoretical_loss": 3.3506005510488794, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019296969696969696, + "loss": 2.435, + "theoretical_loss": 3.3506005510488794, + "tokens_seen": 2674130944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019294949494949495, + "loss": 2.2508, + "theoretical_loss": 3.350594089122049, + "tokens_seen": 2674196480 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019292929292929293, + "loss": 2.6159, + "theoretical_loss": 3.3505876273979176, + "tokens_seen": 2674262016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019290909090909092, + "loss": 2.4743, + "theoretical_loss": 3.350581165876474, + "tokens_seen": 2674327552 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928888888888889, + "loss": 2.6403, + "theoretical_loss": 3.350574704557707, + "tokens_seen": 2674393088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019286868686868686, + "loss": 2.3333, + "theoretical_loss": 3.3505682434416046, + "tokens_seen": 2674458624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019284848484848487, + "loss": 2.6541, + "theoretical_loss": 3.3505617825281564, + "tokens_seen": 2674524160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019282828282828283, + "loss": 2.7564, + "theoretical_loss": 3.3505553218173505, + "tokens_seen": 2674589696 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001928080808080808, + "loss": 2.7812, + "theoretical_loss": 3.350548861309176, + "tokens_seen": 2674655232 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001927878787878788, + "loss": 2.4448, + "theoretical_loss": 3.3505424010036213, + "tokens_seen": 2674720768 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019276767676767675, + "loss": 2.4431, + "theoretical_loss": 3.350535940900675, + "tokens_seen": 2674786304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019274747474747476, + "loss": 2.5631, + "theoretical_loss": 3.350529481000326, + "tokens_seen": 2674851840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019272727272727272, + "loss": 2.7181, + "theoretical_loss": 3.3505230213025627, + "tokens_seen": 2674917376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019270707070707073, + "loss": 2.4104, + "theoretical_loss": 3.3505165618073742, + "tokens_seen": 2674982912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019268686868686869, + "loss": 2.4574, + "theoretical_loss": 3.350510102514749, + "tokens_seen": 2675048448 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001926666666666667, + "loss": 2.6936, + "theoretical_loss": 3.3505036434246755, + "tokens_seen": 2675113984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019264646464646465, + "loss": 2.3155, + "theoretical_loss": 3.3504971845371427, + "tokens_seen": 2675179520 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001926262626262626, + "loss": 2.5033, + "theoretical_loss": 3.350490725852139, + "tokens_seen": 2675245056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019260606060606062, + "loss": 2.4031, + "theoretical_loss": 3.3504842673696538, + "tokens_seen": 2675310592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019258585858585858, + "loss": 2.5714, + "theoretical_loss": 3.350477809089675, + "tokens_seen": 2675376128 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001925656565656566, + "loss": 2.8698, + "theoretical_loss": 3.3504713510121915, + "tokens_seen": 2675441664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019254545454545455, + "loss": 2.5447, + "theoretical_loss": 3.3504648931371923, + "tokens_seen": 2675507200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019252525252525253, + "loss": 2.6156, + "theoretical_loss": 3.350458435464666, + "tokens_seen": 2675572736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019250505050505051, + "loss": 2.5776, + "theoretical_loss": 3.3504519779946005, + "tokens_seen": 2675638272 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001924848484848485, + "loss": 2.516, + "theoretical_loss": 3.3504455207269856, + "tokens_seen": 2675703808 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1507167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1715915203094482, + "objective/train/theoretical_loss": 3.3504390636618093, + "objective/train/tokens_used": 1034628576, + "theoretical_loss": 3.3504390636618093, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019246464646464648, + "loss": 2.5117, + "theoretical_loss": 3.3504390636618093, + "tokens_seen": 2675769344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019244444444444444, + "loss": 2.4379, + "theoretical_loss": 3.350432606799061, + "tokens_seen": 2675834880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019242424242424242, + "loss": 2.651, + "theoretical_loss": 3.3504261501387282, + "tokens_seen": 2675900416 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001924040404040404, + "loss": 2.6164, + "theoretical_loss": 3.350419693680801, + "tokens_seen": 2675965952 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001923838383838384, + "loss": 2.5808, + "theoretical_loss": 3.350413237425267, + "tokens_seen": 2676031488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019236363636363637, + "loss": 2.6627, + "theoretical_loss": 3.350406781372116, + "tokens_seen": 2676097024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019234343434343436, + "loss": 2.6115, + "theoretical_loss": 3.3504003255213353, + "tokens_seen": 2676162560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019232323232323232, + "loss": 2.7699, + "theoretical_loss": 3.3503938698729145, + "tokens_seen": 2676228096 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001923030303030303, + "loss": 2.5488, + "theoretical_loss": 3.350387414426842, + "tokens_seen": 2676293632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019228282828282828, + "loss": 2.4661, + "theoretical_loss": 3.350380959183107, + "tokens_seen": 2676359168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019226262626262627, + "loss": 2.6931, + "theoretical_loss": 3.3503745041416977, + "tokens_seen": 2676424704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019224242424242425, + "loss": 2.771, + "theoretical_loss": 3.350368049302603, + "tokens_seen": 2676490240 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001922222222222222, + "loss": 2.3211, + "theoretical_loss": 3.350361594665811, + "tokens_seen": 2676555776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019220202020202022, + "loss": 2.422, + "theoretical_loss": 3.350355140231312, + "tokens_seen": 2676621312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019218181818181818, + "loss": 2.3997, + "theoretical_loss": 3.3503486859990925, + "tokens_seen": 2676686848 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921616161616162, + "loss": 2.5512, + "theoretical_loss": 3.350342231969143, + "tokens_seen": 2676752384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019214141414141414, + "loss": 2.2546, + "theoretical_loss": 3.3503357781414516, + "tokens_seen": 2676817920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921212121212121, + "loss": 2.4833, + "theoretical_loss": 3.350329324516007, + "tokens_seen": 2676883456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001921010101010101, + "loss": 2.7316, + "theoretical_loss": 3.3503228710927977, + "tokens_seen": 2676948992 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019208080808080807, + "loss": 2.5543, + "theoretical_loss": 3.350316417871813, + "tokens_seen": 2677014528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019206060606060608, + "loss": 2.6765, + "theoretical_loss": 3.350309964853041, + "tokens_seen": 2677080064 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019204040404040404, + "loss": 2.4669, + "theoretical_loss": 3.3503035120364704, + "tokens_seen": 2677145600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019202020202020202, + "loss": 2.4105, + "theoretical_loss": 3.35029705942209, + "tokens_seen": 2677211136 + }, + { + "epoch": 0.62, + "learning_rate": 0.000192, + "loss": 2.6351, + "theoretical_loss": 3.350290607009889, + "tokens_seen": 2677276672 + }, + { + "epoch": 0.62, + "learning_rate": 0.000191979797979798, + "loss": 2.593, + "theoretical_loss": 3.350284154799856, + "tokens_seen": 2677342208 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1507786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.567622661590576, + "objective/train/theoretical_loss": 3.350277702791979, + "objective/train/tokens_used": 1036266976, + "theoretical_loss": 3.350277702791979, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019195959595959597, + "loss": 2.4607, + "theoretical_loss": 3.350277702791979, + "tokens_seen": 2677407744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019193939393939393, + "loss": 2.5503, + "theoretical_loss": 3.3502712509862476, + "tokens_seen": 2677473280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019191919191919191, + "loss": 2.6415, + "theoretical_loss": 3.35026479938265, + "tokens_seen": 2677538816 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001918989898989899, + "loss": 2.6276, + "theoretical_loss": 3.3502583479811747, + "tokens_seen": 2677604352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019187878787878788, + "loss": 2.7926, + "theoretical_loss": 3.3502518967818116, + "tokens_seen": 2677669888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019185858585858587, + "loss": 2.4728, + "theoretical_loss": 3.350245445784548, + "tokens_seen": 2677735424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019183838383838385, + "loss": 2.6089, + "theoretical_loss": 3.3502389949893736, + "tokens_seen": 2677800960 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001918181818181818, + "loss": 2.5288, + "theoretical_loss": 3.3502325443962766, + "tokens_seen": 2677866496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019179797979797982, + "loss": 2.4177, + "theoretical_loss": 3.3502260940052455, + "tokens_seen": 2677932032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019177777777777777, + "loss": 2.7567, + "theoretical_loss": 3.3502196438162697, + "tokens_seen": 2677997568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019175757575757576, + "loss": 2.6903, + "theoretical_loss": 3.3502131938293376, + "tokens_seen": 2678063104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019173737373737374, + "loss": 2.5229, + "theoretical_loss": 3.3502067440444376, + "tokens_seen": 2678128640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019171717171717173, + "loss": 2.608, + "theoretical_loss": 3.350200294461559, + "tokens_seen": 2678194176 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001916969696969697, + "loss": 2.5998, + "theoretical_loss": 3.3501938450806907, + "tokens_seen": 2678259712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019167676767676767, + "loss": 2.6987, + "theoretical_loss": 3.3501873959018207, + "tokens_seen": 2678325248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019165656565656568, + "loss": 2.4594, + "theoretical_loss": 3.3501809469249384, + "tokens_seen": 2678390784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019163636363636363, + "loss": 2.6161, + "theoretical_loss": 3.350174498150032, + "tokens_seen": 2678456320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019161616161616165, + "loss": 2.6906, + "theoretical_loss": 3.35016804957709, + "tokens_seen": 2678521856 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001915959595959596, + "loss": 2.3811, + "theoretical_loss": 3.350161601206102, + "tokens_seen": 2678587392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019157575757575756, + "loss": 2.5593, + "theoretical_loss": 3.3501551530370564, + "tokens_seen": 2678652928 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019155555555555557, + "loss": 2.5458, + "theoretical_loss": 3.3501487050699414, + "tokens_seen": 2678718464 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019153535353535353, + "loss": 2.6269, + "theoretical_loss": 3.3501422573047464, + "tokens_seen": 2678784000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019151515151515154, + "loss": 2.5516, + "theoretical_loss": 3.35013580974146, + "tokens_seen": 2678849536 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001914949494949495, + "loss": 2.5219, + "theoretical_loss": 3.3501293623800708, + "tokens_seen": 2678915072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019147474747474748, + "loss": 2.6542, + "theoretical_loss": 3.3501229152205676, + "tokens_seen": 2678980608 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1509162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4454691410064697, + "objective/train/theoretical_loss": 3.350116468262939, + "objective/train/tokens_used": 1037905376, + "theoretical_loss": 3.350116468262939, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019145454545454546, + "loss": 2.5091, + "theoretical_loss": 3.350116468262939, + "tokens_seen": 2679046144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019143434343434342, + "loss": 2.463, + "theoretical_loss": 3.350110021507174, + "tokens_seen": 2679111680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019141414141414143, + "loss": 2.5742, + "theoretical_loss": 3.3501035749532613, + "tokens_seen": 2679177216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001913939393939394, + "loss": 2.6119, + "theoretical_loss": 3.350097128601189, + "tokens_seen": 2679242752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019137373737373737, + "loss": 2.583, + "theoretical_loss": 3.350090682450947, + "tokens_seen": 2679308288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019135353535353536, + "loss": 2.5353, + "theoretical_loss": 3.350084236502523, + "tokens_seen": 2679373824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019133333333333334, + "loss": 2.6649, + "theoretical_loss": 3.3500777907559067, + "tokens_seen": 2679439360 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019131313131313132, + "loss": 2.5131, + "theoretical_loss": 3.350071345211086, + "tokens_seen": 2679504896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001912929292929293, + "loss": 2.5368, + "theoretical_loss": 3.35006489986805, + "tokens_seen": 2679570432 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019127272727272726, + "loss": 2.5393, + "theoretical_loss": 3.3500584547267875, + "tokens_seen": 2679635968 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019125252525252525, + "loss": 2.7813, + "theoretical_loss": 3.3500520097872872, + "tokens_seen": 2679701504 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019123232323232323, + "loss": 2.4726, + "theoretical_loss": 3.3500455650495375, + "tokens_seen": 2679767040 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019121212121212122, + "loss": 2.6738, + "theoretical_loss": 3.350039120513528, + "tokens_seen": 2679832576 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001911919191919192, + "loss": 2.588, + "theoretical_loss": 3.3500326761792465, + "tokens_seen": 2679898112 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019117171717171716, + "loss": 2.7065, + "theoretical_loss": 3.350026232046682, + "tokens_seen": 2679963648 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019115151515151517, + "loss": 2.6132, + "theoretical_loss": 3.3500197881158242, + "tokens_seen": 2680029184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019113131313131313, + "loss": 2.5778, + "theoretical_loss": 3.35001334438666, + "tokens_seen": 2680094720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019111111111111114, + "loss": 2.5794, + "theoretical_loss": 3.35000690085918, + "tokens_seen": 2680160256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001910909090909091, + "loss": 2.6199, + "theoretical_loss": 3.3500004575333726, + "tokens_seen": 2680225792 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019107070707070705, + "loss": 2.67, + "theoretical_loss": 3.3499940144092255, + "tokens_seen": 2680291328 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019105050505050506, + "loss": 2.6696, + "theoretical_loss": 3.349987571486728, + "tokens_seen": 2680356864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019103030303030302, + "loss": 2.5315, + "theoretical_loss": 3.349981128765869, + "tokens_seen": 2680422400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019101010101010103, + "loss": 2.7719, + "theoretical_loss": 3.3499746862466377, + "tokens_seen": 2680487936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019098989898989899, + "loss": 2.7342, + "theoretical_loss": 3.3499682439290224, + "tokens_seen": 2680553472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019096969696969697, + "loss": 2.284, + "theoretical_loss": 3.349961801813011, + "tokens_seen": 2680619008 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1509790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7762537002563477, + "objective/train/theoretical_loss": 3.349955359898594, + "objective/train/tokens_used": 1039543776, + "theoretical_loss": 3.349955359898594, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019094949494949495, + "loss": 2.739, + "theoretical_loss": 3.349955359898594, + "tokens_seen": 2680684544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019092929292929294, + "loss": 2.4975, + "theoretical_loss": 3.349948918185759, + "tokens_seen": 2680750080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019090909090909092, + "loss": 2.6303, + "theoretical_loss": 3.349942476674495, + "tokens_seen": 2680815616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019088888888888888, + "loss": 2.5636, + "theoretical_loss": 3.3499360353647907, + "tokens_seen": 2680881152 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001908686868686869, + "loss": 2.5713, + "theoretical_loss": 3.349929594256635, + "tokens_seen": 2680946688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019084848484848485, + "loss": 2.2742, + "theoretical_loss": 3.3499231533500167, + "tokens_seen": 2681012224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019082828282828283, + "loss": 2.7588, + "theoretical_loss": 3.3499167126449243, + "tokens_seen": 2681077760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019080808080808081, + "loss": 2.7317, + "theoretical_loss": 3.349910272141347, + "tokens_seen": 2681143296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907878787878788, + "loss": 2.5896, + "theoretical_loss": 3.3499038318392733, + "tokens_seen": 2681208832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019076767676767678, + "loss": 2.3542, + "theoretical_loss": 3.3498973917386925, + "tokens_seen": 2681274368 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019074747474747474, + "loss": 2.7827, + "theoretical_loss": 3.3498909518395923, + "tokens_seen": 2681339904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019072727272727272, + "loss": 2.5406, + "theoretical_loss": 3.349884512141962, + "tokens_seen": 2681405440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001907070707070707, + "loss": 2.6637, + "theoretical_loss": 3.3498780726457906, + "tokens_seen": 2681470976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001906868686868687, + "loss": 2.3685, + "theoretical_loss": 3.349871633351067, + "tokens_seen": 2681536512 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019066666666666668, + "loss": 2.4265, + "theoretical_loss": 3.349865194257779, + "tokens_seen": 2681602048 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019064646464646466, + "loss": 2.5149, + "theoretical_loss": 3.3498587553659167, + "tokens_seen": 2681667584 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019062626262626262, + "loss": 2.7259, + "theoretical_loss": 3.349852316675468, + "tokens_seen": 2681733120 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019060606060606063, + "loss": 2.518, + "theoretical_loss": 3.349845878186422, + "tokens_seen": 2681798656 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019058585858585858, + "loss": 2.5833, + "theoretical_loss": 3.3498394398987674, + "tokens_seen": 2681864192 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019056565656565657, + "loss": 2.6882, + "theoretical_loss": 3.3498330018124927, + "tokens_seen": 2681929728 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019054545454545455, + "loss": 2.4731, + "theoretical_loss": 3.349826563927587, + "tokens_seen": 2681995264 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001905252525252525, + "loss": 2.5538, + "theoretical_loss": 3.3498201262440395, + "tokens_seen": 2682060800 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019050505050505052, + "loss": 2.697, + "theoretical_loss": 3.349813688761838, + "tokens_seen": 2682126336 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019048484848484848, + "loss": 2.581, + "theoretical_loss": 3.349807251480972, + "tokens_seen": 2682191872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904646464646465, + "loss": 2.6001, + "theoretical_loss": 3.34980081440143, + "tokens_seen": 2682257408 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1510817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7443935871124268, + "objective/train/theoretical_loss": 3.349794377523201, + "objective/train/tokens_used": 1041182176, + "theoretical_loss": 3.349794377523201, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019044444444444444, + "loss": 2.9328, + "theoretical_loss": 3.349794377523201, + "tokens_seen": 2682322944 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019042424242424243, + "loss": 2.4631, + "theoretical_loss": 3.3497879408462734, + "tokens_seen": 2682388480 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001904040404040404, + "loss": 2.5554, + "theoretical_loss": 3.3497815043706365, + "tokens_seen": 2682454016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019038383838383837, + "loss": 2.6724, + "theoretical_loss": 3.3497750680962786, + "tokens_seen": 2682519552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019036363636363638, + "loss": 2.7489, + "theoretical_loss": 3.349768632023189, + "tokens_seen": 2682585088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019034343434343434, + "loss": 3.0285, + "theoretical_loss": 3.3497621961513557, + "tokens_seen": 2682650624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019032323232323232, + "loss": 2.5633, + "theoretical_loss": 3.3497557604807686, + "tokens_seen": 2682716160 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001903030303030303, + "loss": 2.4215, + "theoretical_loss": 3.3497493250114156, + "tokens_seen": 2682781696 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001902828282828283, + "loss": 2.7301, + "theoretical_loss": 3.3497428897432857, + "tokens_seen": 2682847232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019026262626262627, + "loss": 2.6636, + "theoretical_loss": 3.3497364546763677, + "tokens_seen": 2682912768 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019024242424242426, + "loss": 2.6754, + "theoretical_loss": 3.349730019810651, + "tokens_seen": 2682978304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019022222222222221, + "loss": 2.4477, + "theoretical_loss": 3.3497235851461236, + "tokens_seen": 2683043840 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001902020202020202, + "loss": 2.7296, + "theoretical_loss": 3.3497171506827743, + "tokens_seen": 2683109376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019018181818181818, + "loss": 2.577, + "theoretical_loss": 3.349710716420592, + "tokens_seen": 2683174912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019016161616161617, + "loss": 2.6425, + "theoretical_loss": 3.349704282359566, + "tokens_seen": 2683240448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019014141414141415, + "loss": 2.444, + "theoretical_loss": 3.3496978484996847, + "tokens_seen": 2683305984 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001901212121212121, + "loss": 2.7121, + "theoretical_loss": 3.349691414840937, + "tokens_seen": 2683371520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019010101010101012, + "loss": 2.6593, + "theoretical_loss": 3.349684981383312, + "tokens_seen": 2683437056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019008080808080807, + "loss": 2.6483, + "theoretical_loss": 3.349678548126797, + "tokens_seen": 2683502592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019006060606060609, + "loss": 2.631, + "theoretical_loss": 3.349672115071383, + "tokens_seen": 2683568128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019004040404040404, + "loss": 2.2801, + "theoretical_loss": 3.349665682217058, + "tokens_seen": 2683633664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019002020202020203, + "loss": 2.5319, + "theoretical_loss": 3.3496592495638096, + "tokens_seen": 2683699200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00019, + "loss": 2.668, + "theoretical_loss": 3.349652817111628, + "tokens_seen": 2683764736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018997979797979797, + "loss": 2.8224, + "theoretical_loss": 3.3496463848605016, + "tokens_seen": 2683830272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018995959595959598, + "loss": 2.521, + "theoretical_loss": 3.349639952810419, + "tokens_seen": 2683895808 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1511447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615917444229126, + "objective/train/theoretical_loss": 3.3496335209613695, + "objective/train/tokens_used": 1042820576, + "theoretical_loss": 3.3496335209613695, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018993939393939394, + "loss": 2.6358, + "theoretical_loss": 3.3496335209613695, + "tokens_seen": 2683961344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018991919191919195, + "loss": 2.3965, + "theoretical_loss": 3.349627089313341, + "tokens_seen": 2684026880 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001898989898989899, + "loss": 2.7613, + "theoretical_loss": 3.3496206578663235, + "tokens_seen": 2684092416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018987878787878786, + "loss": 2.5125, + "theoretical_loss": 3.349614226620305, + "tokens_seen": 2684157952 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018985858585858587, + "loss": 2.6405, + "theoretical_loss": 3.3496077955752743, + "tokens_seen": 2684223488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018983838383838383, + "loss": 2.252, + "theoretical_loss": 3.3496013647312206, + "tokens_seen": 2684289024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018981818181818184, + "loss": 2.2868, + "theoretical_loss": 3.349594934088133, + "tokens_seen": 2684354560 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001897979797979798, + "loss": 2.615, + "theoretical_loss": 3.349588503645999, + "tokens_seen": 2684420096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018977777777777778, + "loss": 2.4308, + "theoretical_loss": 3.3495820734048087, + "tokens_seen": 2684485632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018975757575757576, + "loss": 2.3135, + "theoretical_loss": 3.3495756433645507, + "tokens_seen": 2684551168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018973737373737375, + "loss": 2.7574, + "theoretical_loss": 3.3495692135252133, + "tokens_seen": 2684616704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018971717171717173, + "loss": 2.6799, + "theoretical_loss": 3.3495627838867854, + "tokens_seen": 2684682240 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001896969696969697, + "loss": 2.4387, + "theoretical_loss": 3.3495563544492564, + "tokens_seen": 2684747776 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018967676767676767, + "loss": 2.5947, + "theoretical_loss": 3.3495499252126146, + "tokens_seen": 2684813312 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018965656565656566, + "loss": 2.7872, + "theoretical_loss": 3.3495434961768495, + "tokens_seen": 2684878848 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018963636363636364, + "loss": 2.7473, + "theoretical_loss": 3.3495370673419487, + "tokens_seen": 2684944384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018961616161616162, + "loss": 2.377, + "theoretical_loss": 3.3495306387079022, + "tokens_seen": 2685009920 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001895959595959596, + "loss": 2.4916, + "theoretical_loss": 3.349524210274698, + "tokens_seen": 2685075456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018957575757575757, + "loss": 2.5453, + "theoretical_loss": 3.3495177820423256, + "tokens_seen": 2685140992 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018955555555555558, + "loss": 2.7854, + "theoretical_loss": 3.3495113540107733, + "tokens_seen": 2685206528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018953535353535353, + "loss": 2.6818, + "theoretical_loss": 3.34950492618003, + "tokens_seen": 2685272064 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018951515151515152, + "loss": 2.6732, + "theoretical_loss": 3.3494984985500844, + "tokens_seen": 2685337600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0001894949494949495, + "loss": 2.5192, + "theoretical_loss": 3.349492071120926, + "tokens_seen": 2685403136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018947474747474746, + "loss": 2.526, + "theoretical_loss": 3.3494856438925433, + "tokens_seen": 2685468672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018945454545454547, + "loss": 2.6078, + "theoretical_loss": 3.349479216864925, + "tokens_seen": 2685534208 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 1512730, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.91302490234375, + "objective/train/theoretical_loss": 3.3494727900380594, + "objective/train/tokens_used": 1044458976, + "theoretical_loss": 3.3494727900380594, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00018943434343434343, + "loss": 2.5406, + "theoretical_loss": 3.3494727900380594, + "tokens_seen": 2685599744 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018941414141414144, + "loss": 2.3657, + "theoretical_loss": 3.3494663634119366, + "tokens_seen": 2685665280 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001893939393939394, + "loss": 2.677, + "theoretical_loss": 3.3494599369865443, + "tokens_seen": 2685730816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018937373737373738, + "loss": 2.6871, + "theoretical_loss": 3.349453510761872, + "tokens_seen": 2685796352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018935353535353536, + "loss": 2.4652, + "theoretical_loss": 3.349447084737908, + "tokens_seen": 2685861888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018933333333333332, + "loss": 2.849, + "theoretical_loss": 3.3494406589146415, + "tokens_seen": 2685927424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018931313131313133, + "loss": 2.602, + "theoretical_loss": 3.3494342332920612, + "tokens_seen": 2685992960 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001892929292929293, + "loss": 2.8607, + "theoretical_loss": 3.349427807870156, + "tokens_seen": 2686058496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018927272727272727, + "loss": 2.8069, + "theoretical_loss": 3.3494213826489148, + "tokens_seen": 2686124032 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018925252525252525, + "loss": 2.5046, + "theoretical_loss": 3.3494149576283263, + "tokens_seen": 2686189568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018923232323232324, + "loss": 2.8973, + "theoretical_loss": 3.3494085328083796, + "tokens_seen": 2686255104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018921212121212122, + "loss": 2.6644, + "theoretical_loss": 3.3494021081890635, + "tokens_seen": 2686320640 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891919191919192, + "loss": 2.4421, + "theoretical_loss": 3.349395683770366, + "tokens_seen": 2686386176 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001891717171717172, + "loss": 2.6052, + "theoretical_loss": 3.349389259552277, + "tokens_seen": 2686451712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018915151515151515, + "loss": 2.5202, + "theoretical_loss": 3.349382835534785, + "tokens_seen": 2686517248 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018913131313131313, + "loss": 2.4242, + "theoretical_loss": 3.349376411717879, + "tokens_seen": 2686582784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018911111111111112, + "loss": 2.5368, + "theoretical_loss": 3.3493699881015475, + "tokens_seen": 2686648320 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001890909090909091, + "loss": 2.4214, + "theoretical_loss": 3.3493635646857793, + "tokens_seen": 2686713856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018907070707070708, + "loss": 2.5159, + "theoretical_loss": 3.3493571414705636, + "tokens_seen": 2686779392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018905050505050507, + "loss": 2.6364, + "theoretical_loss": 3.349350718455889, + "tokens_seen": 2686844928 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018903030303030302, + "loss": 2.7727, + "theoretical_loss": 3.3493442956417443, + "tokens_seen": 2686910464 + }, + { + "epoch": 0.63, + "learning_rate": 0.000189010101010101, + "loss": 2.6196, + "theoretical_loss": 3.349337873028119, + "tokens_seen": 2686976000 + }, + { + "epoch": 0.63, + "learning_rate": 0.000188989898989899, + "loss": 2.6099, + "theoretical_loss": 3.349331450615001, + "tokens_seen": 2687041536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018896969696969698, + "loss": 2.5803, + "theoretical_loss": 3.3493250284023794, + "tokens_seen": 2687107072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018894949494949496, + "loss": 2.6623, + "theoretical_loss": 3.349318606390244, + "tokens_seen": 2687172608 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1513440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.467771530151367, + "objective/train/theoretical_loss": 3.349312184578582, + "objective/train/tokens_used": 1046097376, + "theoretical_loss": 3.349312184578582, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018892929292929292, + "loss": 2.4254, + "theoretical_loss": 3.349312184578582, + "tokens_seen": 2687238144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018890909090909093, + "loss": 2.6838, + "theoretical_loss": 3.3493057629673837, + "tokens_seen": 2687303680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018888888888888888, + "loss": 2.3594, + "theoretical_loss": 3.3492993415566374, + "tokens_seen": 2687369216 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001888686868686869, + "loss": 2.7433, + "theoretical_loss": 3.3492929203463317, + "tokens_seen": 2687434752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018884848484848485, + "loss": 2.6869, + "theoretical_loss": 3.349286499336456, + "tokens_seen": 2687500288 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001888282828282828, + "loss": 2.5123, + "theoretical_loss": 3.3492800785269985, + "tokens_seen": 2687565824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018880808080808082, + "loss": 2.5001, + "theoretical_loss": 3.3492736579179487, + "tokens_seen": 2687631360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018878787878787878, + "loss": 2.7291, + "theoretical_loss": 3.349267237509295, + "tokens_seen": 2687696896 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001887676767676768, + "loss": 2.6364, + "theoretical_loss": 3.3492608173010265, + "tokens_seen": 2687762432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018874747474747475, + "loss": 2.6798, + "theoretical_loss": 3.349254397293132, + "tokens_seen": 2687827968 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018872727272727273, + "loss": 2.7449, + "theoretical_loss": 3.3492479774856005, + "tokens_seen": 2687893504 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001887070707070707, + "loss": 2.4842, + "theoretical_loss": 3.3492415578784205, + "tokens_seen": 2687959040 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001886868686868687, + "loss": 2.3552, + "theoretical_loss": 3.349235138471581, + "tokens_seen": 2688024576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018866666666666668, + "loss": 2.5213, + "theoretical_loss": 3.3492287192650716, + "tokens_seen": 2688090112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018864646464646464, + "loss": 2.7242, + "theoretical_loss": 3.3492223002588797, + "tokens_seen": 2688155648 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018862626262626262, + "loss": 2.4533, + "theoretical_loss": 3.349215881452995, + "tokens_seen": 2688221184 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001886060606060606, + "loss": 2.6013, + "theoretical_loss": 3.3492094628474067, + "tokens_seen": 2688286720 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001885858585858586, + "loss": 2.7879, + "theoretical_loss": 3.3492030444421035, + "tokens_seen": 2688352256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018856565656565657, + "loss": 2.5888, + "theoretical_loss": 3.349196626237074, + "tokens_seen": 2688417792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018854545454545456, + "loss": 2.5575, + "theoretical_loss": 3.349190208232307, + "tokens_seen": 2688483328 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018852525252525251, + "loss": 2.595, + "theoretical_loss": 3.3491837904277912, + "tokens_seen": 2688548864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018850505050505053, + "loss": 2.53, + "theoretical_loss": 3.349177372823516, + "tokens_seen": 2688614400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018848484848484848, + "loss": 2.5543, + "theoretical_loss": 3.34917095541947, + "tokens_seen": 2688679936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018846464646464647, + "loss": 2.5713, + "theoretical_loss": 3.3491645382156428, + "tokens_seen": 2688745472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018844444444444445, + "loss": 2.449, + "theoretical_loss": 3.349158121212022, + "tokens_seen": 2688811008 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1514443, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5168426036834717, + "objective/train/theoretical_loss": 3.349151704408597, + "objective/train/tokens_used": 1047735776, + "theoretical_loss": 3.349151704408597, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001884242424242424, + "loss": 2.5755, + "theoretical_loss": 3.349151704408597, + "tokens_seen": 2688876544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018840404040404042, + "loss": 2.4745, + "theoretical_loss": 3.349145287805357, + "tokens_seen": 2688942080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018838383838383838, + "loss": 2.5431, + "theoretical_loss": 3.3491388714022907, + "tokens_seen": 2689007616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018836363636363639, + "loss": 2.6047, + "theoretical_loss": 3.3491324551993866, + "tokens_seen": 2689073152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018834343434343434, + "loss": 2.5339, + "theoretical_loss": 3.349126039196634, + "tokens_seen": 2689138688 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018832323232323235, + "loss": 2.7611, + "theoretical_loss": 3.3491196233940217, + "tokens_seen": 2689204224 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001883030303030303, + "loss": 2.5039, + "theoretical_loss": 3.349113207791538, + "tokens_seen": 2689269760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018828282828282827, + "loss": 2.467, + "theoretical_loss": 3.3491067923891733, + "tokens_seen": 2689335296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018826262626262628, + "loss": 2.3426, + "theoretical_loss": 3.349100377186915, + "tokens_seen": 2689400832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018824242424242424, + "loss": 2.6465, + "theoretical_loss": 3.3490939621847526, + "tokens_seen": 2689466368 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018822222222222225, + "loss": 2.7191, + "theoretical_loss": 3.3490875473826747, + "tokens_seen": 2689531904 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001882020202020202, + "loss": 2.4797, + "theoretical_loss": 3.3490811327806704, + "tokens_seen": 2689597440 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001881818181818182, + "loss": 2.7592, + "theoretical_loss": 3.349074718378729, + "tokens_seen": 2689662976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018816161616161617, + "loss": 2.6166, + "theoretical_loss": 3.3490683041768383, + "tokens_seen": 2689728512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018814141414141413, + "loss": 2.5879, + "theoretical_loss": 3.349061890174988, + "tokens_seen": 2689794048 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018812121212121214, + "loss": 2.5066, + "theoretical_loss": 3.3490554763731666, + "tokens_seen": 2689859584 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001881010101010101, + "loss": 2.8284, + "theoretical_loss": 3.3490490627713636, + "tokens_seen": 2689925120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018808080808080808, + "loss": 2.5418, + "theoretical_loss": 3.3490426493695673, + "tokens_seen": 2689990656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018806060606060606, + "loss": 2.6204, + "theoretical_loss": 3.349036236167767, + "tokens_seen": 2690056192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018804040404040405, + "loss": 2.7119, + "theoretical_loss": 3.349029823165951, + "tokens_seen": 2690121728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018802020202020203, + "loss": 2.661, + "theoretical_loss": 3.3490234103641088, + "tokens_seen": 2690187264 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018800000000000002, + "loss": 2.679, + "theoretical_loss": 3.349016997762229, + "tokens_seen": 2690252800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018797979797979797, + "loss": 2.4439, + "theoretical_loss": 3.3490105853603005, + "tokens_seen": 2690318336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018795959595959596, + "loss": 2.5767, + "theoretical_loss": 3.349004173158312, + "tokens_seen": 2690383872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018793939393939394, + "loss": 2.5373, + "theoretical_loss": 3.348997761156253, + "tokens_seen": 2690449408 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1514886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.462597608566284, + "objective/train/theoretical_loss": 3.3489913493541117, + "objective/train/tokens_used": 1049374176, + "theoretical_loss": 3.3489913493541117, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018791919191919192, + "loss": 2.4871, + "theoretical_loss": 3.3489913493541117, + "tokens_seen": 2690514944 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001878989898989899, + "loss": 2.5471, + "theoretical_loss": 3.3489849377518777, + "tokens_seen": 2690580480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018787878787878787, + "loss": 2.6205, + "theoretical_loss": 3.3489785263495393, + "tokens_seen": 2690646016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018785858585858588, + "loss": 2.5838, + "theoretical_loss": 3.3489721151470855, + "tokens_seen": 2690711552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018783838383838383, + "loss": 2.5336, + "theoretical_loss": 3.3489657041445056, + "tokens_seen": 2690777088 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018781818181818184, + "loss": 2.6553, + "theoretical_loss": 3.348959293341788, + "tokens_seen": 2690842624 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001877979797979798, + "loss": 2.66, + "theoretical_loss": 3.3489528827389217, + "tokens_seen": 2690908160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018777777777777776, + "loss": 2.8179, + "theoretical_loss": 3.348946472335896, + "tokens_seen": 2690973696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018775757575757577, + "loss": 2.3985, + "theoretical_loss": 3.3489400621326997, + "tokens_seen": 2691039232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018773737373737373, + "loss": 2.5578, + "theoretical_loss": 3.348933652129321, + "tokens_seen": 2691104768 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018771717171717174, + "loss": 2.8249, + "theoretical_loss": 3.3489272423257495, + "tokens_seen": 2691170304 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001876969696969697, + "loss": 2.4738, + "theoretical_loss": 3.348920832721974, + "tokens_seen": 2691235840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018767676767676768, + "loss": 2.7213, + "theoretical_loss": 3.3489144233179835, + "tokens_seen": 2691301376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018765656565656566, + "loss": 2.6503, + "theoretical_loss": 3.348908014113767, + "tokens_seen": 2691366912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018763636363636365, + "loss": 2.8588, + "theoretical_loss": 3.3489016051093126, + "tokens_seen": 2691432448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018761616161616163, + "loss": 2.6995, + "theoretical_loss": 3.34889519630461, + "tokens_seen": 2691497984 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001875959595959596, + "loss": 2.5105, + "theoretical_loss": 3.3488887876996483, + "tokens_seen": 2691563520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018757575757575757, + "loss": 2.6234, + "theoretical_loss": 3.348882379294415, + "tokens_seen": 2691629056 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018755555555555555, + "loss": 2.4247, + "theoretical_loss": 3.348875971088901, + "tokens_seen": 2691694592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018753535353535354, + "loss": 2.5934, + "theoretical_loss": 3.3488695630830936, + "tokens_seen": 2691760128 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018751515151515152, + "loss": 2.627, + "theoretical_loss": 3.348863155276983, + "tokens_seen": 2691825664 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001874949494949495, + "loss": 2.7985, + "theoretical_loss": 3.348856747670557, + "tokens_seen": 2691891200 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001874747474747475, + "loss": 2.6946, + "theoretical_loss": 3.3488503402638052, + "tokens_seen": 2691956736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018745454545454545, + "loss": 2.8376, + "theoretical_loss": 3.348843933056716, + "tokens_seen": 2692022272 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018743434343434343, + "loss": 2.5617, + "theoretical_loss": 3.3488375260492784, + "tokens_seen": 2692087808 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1515566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8839523792266846, + "objective/train/theoretical_loss": 3.348831119241482, + "objective/train/tokens_used": 1051012576, + "theoretical_loss": 3.348831119241482, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018741414141414142, + "loss": 2.7195, + "theoretical_loss": 3.348831119241482, + "tokens_seen": 2692153344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001873939393939394, + "loss": 2.4855, + "theoretical_loss": 3.348824712633315, + "tokens_seen": 2692218880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018737373737373738, + "loss": 2.6346, + "theoretical_loss": 3.348818306224767, + "tokens_seen": 2692284416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018735353535353537, + "loss": 2.4737, + "theoretical_loss": 3.348811900015826, + "tokens_seen": 2692349952 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018733333333333332, + "loss": 2.509, + "theoretical_loss": 3.3488054940064815, + "tokens_seen": 2692415488 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018731313131313134, + "loss": 2.841, + "theoretical_loss": 3.3487990881967225, + "tokens_seen": 2692481024 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001872929292929293, + "loss": 2.5387, + "theoretical_loss": 3.3487926825865375, + "tokens_seen": 2692546560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018727272727272728, + "loss": 2.6835, + "theoretical_loss": 3.348786277175916, + "tokens_seen": 2692612096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018725252525252526, + "loss": 2.6204, + "theoretical_loss": 3.3487798719648465, + "tokens_seen": 2692677632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018723232323232322, + "loss": 2.4249, + "theoretical_loss": 3.348773466953318, + "tokens_seen": 2692743168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018721212121212123, + "loss": 2.6819, + "theoretical_loss": 3.3487670621413193, + "tokens_seen": 2692808704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018719191919191918, + "loss": 2.4055, + "theoretical_loss": 3.34876065752884, + "tokens_seen": 2692874240 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001871717171717172, + "loss": 2.5353, + "theoretical_loss": 3.348754253115868, + "tokens_seen": 2692939776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018715151515151515, + "loss": 2.6887, + "theoretical_loss": 3.3487478489023927, + "tokens_seen": 2693005312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018713131313131314, + "loss": 2.7253, + "theoretical_loss": 3.348741444888403, + "tokens_seen": 2693070848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018711111111111112, + "loss": 2.4593, + "theoretical_loss": 3.3487350410738883, + "tokens_seen": 2693136384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018709090909090908, + "loss": 2.6215, + "theoretical_loss": 3.348728637458837, + "tokens_seen": 2693201920 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001870707070707071, + "loss": 2.5837, + "theoretical_loss": 3.3487222340432385, + "tokens_seen": 2693267456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018705050505050505, + "loss": 2.7576, + "theoretical_loss": 3.3487158308270812, + "tokens_seen": 2693332992 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018703030303030303, + "loss": 2.7729, + "theoretical_loss": 3.3487094278103537, + "tokens_seen": 2693398528 + }, + { + "epoch": 0.63, + "learning_rate": 0.000187010101010101, + "loss": 2.6721, + "theoretical_loss": 3.3487030249930463, + "tokens_seen": 2693464064 + }, + { + "epoch": 0.63, + "learning_rate": 0.000186989898989899, + "loss": 2.6149, + "theoretical_loss": 3.348696622375147, + "tokens_seen": 2693529600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018696969696969698, + "loss": 2.5404, + "theoretical_loss": 3.3486902199566444, + "tokens_seen": 2693595136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018694949494949497, + "loss": 2.539, + "theoretical_loss": 3.348683817737528, + "tokens_seen": 2693660672 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018692929292929292, + "loss": 2.7503, + "theoretical_loss": 3.3486774157177868, + "tokens_seen": 2693726208 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1516840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.709155559539795, + "objective/train/theoretical_loss": 3.34867101389741, + "objective/train/tokens_used": 1052650976, + "theoretical_loss": 3.34867101389741, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001869090909090909, + "loss": 2.5917, + "theoretical_loss": 3.34867101389741, + "tokens_seen": 2693791744 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001868888888888889, + "loss": 2.6546, + "theoretical_loss": 3.3486646122763855, + "tokens_seen": 2693857280 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018686868686868687, + "loss": 2.682, + "theoretical_loss": 3.348658210854703, + "tokens_seen": 2693922816 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018684848484848486, + "loss": 2.6461, + "theoretical_loss": 3.348651809632351, + "tokens_seen": 2693988352 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018682828282828281, + "loss": 2.8869, + "theoretical_loss": 3.348645408609319, + "tokens_seen": 2694053888 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018680808080808083, + "loss": 2.3684, + "theoretical_loss": 3.348639007785596, + "tokens_seen": 2694119424 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018678787878787878, + "loss": 2.5326, + "theoretical_loss": 3.3486326071611705, + "tokens_seen": 2694184960 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867676767676768, + "loss": 2.6235, + "theoretical_loss": 3.3486262067360317, + "tokens_seen": 2694250496 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018674747474747475, + "loss": 2.7401, + "theoretical_loss": 3.348619806510168, + "tokens_seen": 2694316032 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001867272727272727, + "loss": 2.6899, + "theoretical_loss": 3.3486134064835693, + "tokens_seen": 2694381568 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018670707070707072, + "loss": 2.801, + "theoretical_loss": 3.348607006656224, + "tokens_seen": 2694447104 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018668686868686868, + "loss": 2.659, + "theoretical_loss": 3.3486006070281205, + "tokens_seen": 2694512640 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866666666666667, + "loss": 2.748, + "theoretical_loss": 3.3485942075992488, + "tokens_seen": 2694578176 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018664646464646464, + "loss": 2.572, + "theoretical_loss": 3.3485878083695972, + "tokens_seen": 2694643712 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018662626262626265, + "loss": 2.7604, + "theoretical_loss": 3.3485814093391553, + "tokens_seen": 2694709248 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001866060606060606, + "loss": 2.5449, + "theoretical_loss": 3.3485750105079113, + "tokens_seen": 2694774784 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018658585858585857, + "loss": 2.5478, + "theoretical_loss": 3.348568611875854, + "tokens_seen": 2694840320 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018656565656565658, + "loss": 2.5156, + "theoretical_loss": 3.3485622134429733, + "tokens_seen": 2694905856 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018654545454545454, + "loss": 2.6568, + "theoretical_loss": 3.3485558152092576, + "tokens_seen": 2694971392 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018652525252525255, + "loss": 2.5948, + "theoretical_loss": 3.348549417174696, + "tokens_seen": 2695036928 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001865050505050505, + "loss": 2.5584, + "theoretical_loss": 3.348543019339277, + "tokens_seen": 2695102464 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001864848484848485, + "loss": 2.5884, + "theoretical_loss": 3.3485366217029906, + "tokens_seen": 2695168000 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018646464646464647, + "loss": 2.589, + "theoretical_loss": 3.348530224265825, + "tokens_seen": 2695233536 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018644444444444446, + "loss": 2.5259, + "theoretical_loss": 3.3485238270277686, + "tokens_seen": 2695299072 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018642424242424244, + "loss": 2.709, + "theoretical_loss": 3.3485174299888114, + "tokens_seen": 2695364608 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1517438, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8408963680267334, + "objective/train/theoretical_loss": 3.348511033148942, + "objective/train/tokens_used": 1054289376, + "theoretical_loss": 3.348511033148942, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001864040404040404, + "loss": 2.4817, + "theoretical_loss": 3.348511033148942, + "tokens_seen": 2695430144 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018638383838383838, + "loss": 2.5966, + "theoretical_loss": 3.3485046365081494, + "tokens_seen": 2695495680 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018636363636363636, + "loss": 2.5711, + "theoretical_loss": 3.3484982400664225, + "tokens_seen": 2695561216 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018634343434343435, + "loss": 2.4162, + "theoretical_loss": 3.34849184382375, + "tokens_seen": 2695626752 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018632323232323233, + "loss": 2.2215, + "theoretical_loss": 3.3484854477801216, + "tokens_seen": 2695692288 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018630303030303032, + "loss": 2.6324, + "theoretical_loss": 3.3484790519355254, + "tokens_seen": 2695757824 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018628282828282827, + "loss": 2.6899, + "theoretical_loss": 3.3484726562899514, + "tokens_seen": 2695823360 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018626262626262628, + "loss": 2.6938, + "theoretical_loss": 3.348466260843387, + "tokens_seen": 2695888896 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018624242424242424, + "loss": 2.43, + "theoretical_loss": 3.3484598655958226, + "tokens_seen": 2695954432 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018622222222222223, + "loss": 2.2669, + "theoretical_loss": 3.348453470547247, + "tokens_seen": 2696019968 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001862020202020202, + "loss": 2.8737, + "theoretical_loss": 3.3484470756976488, + "tokens_seen": 2696085504 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018618181818181817, + "loss": 2.5126, + "theoretical_loss": 3.3484406810470166, + "tokens_seen": 2696151040 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018616161616161618, + "loss": 2.5449, + "theoretical_loss": 3.34843428659534, + "tokens_seen": 2696216576 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018614141414141413, + "loss": 2.6403, + "theoretical_loss": 3.348427892342608, + "tokens_seen": 2696282112 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018612121212121214, + "loss": 2.6687, + "theoretical_loss": 3.348421498288809, + "tokens_seen": 2696347648 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001861010101010101, + "loss": 2.689, + "theoretical_loss": 3.3484151044339328, + "tokens_seen": 2696413184 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018608080808080809, + "loss": 2.8117, + "theoretical_loss": 3.348408710777967, + "tokens_seen": 2696478720 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018606060606060607, + "loss": 2.5631, + "theoretical_loss": 3.3484023173209025, + "tokens_seen": 2696544256 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018604040404040403, + "loss": 2.3323, + "theoretical_loss": 3.348395924062727, + "tokens_seen": 2696609792 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018602020202020204, + "loss": 2.4724, + "theoretical_loss": 3.3483895310034297, + "tokens_seen": 2696675328 + }, + { + "epoch": 0.63, + "learning_rate": 0.000186, + "loss": 2.4777, + "theoretical_loss": 3.3483831381429994, + "tokens_seen": 2696740864 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018597979797979798, + "loss": 2.7964, + "theoretical_loss": 3.3483767454814255, + "tokens_seen": 2696806400 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018595959595959596, + "loss": 2.6029, + "theoretical_loss": 3.3483703530186966, + "tokens_seen": 2696871936 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018593939393939395, + "loss": 2.5394, + "theoretical_loss": 3.348363960754802, + "tokens_seen": 2696937472 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018591919191919193, + "loss": 2.6565, + "theoretical_loss": 3.348357568689731, + "tokens_seen": 2697003008 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1518738, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7018232345581055, + "objective/train/theoretical_loss": 3.3483511768234715, + "objective/train/tokens_used": 1055927776, + "theoretical_loss": 3.3483511768234715, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018589898989898991, + "loss": 2.4956, + "theoretical_loss": 3.3483511768234715, + "tokens_seen": 2697068544 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018587878787878787, + "loss": 2.6616, + "theoretical_loss": 3.348344785156013, + "tokens_seen": 2697134080 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018585858585858586, + "loss": 2.5265, + "theoretical_loss": 3.348338393687345, + "tokens_seen": 2697199616 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018583838383838384, + "loss": 2.4433, + "theoretical_loss": 3.3483320024174565, + "tokens_seen": 2697265152 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018581818181818182, + "loss": 2.5986, + "theoretical_loss": 3.3483256113463353, + "tokens_seen": 2697330688 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001857979797979798, + "loss": 2.6738, + "theoretical_loss": 3.3483192204739716, + "tokens_seen": 2697396224 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018577777777777776, + "loss": 2.5898, + "theoretical_loss": 3.3483128298003537, + "tokens_seen": 2697461760 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018575757575757577, + "loss": 2.573, + "theoretical_loss": 3.348306439325471, + "tokens_seen": 2697527296 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018573737373737373, + "loss": 2.4156, + "theoretical_loss": 3.3483000490493127, + "tokens_seen": 2697592832 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018571717171717172, + "loss": 2.6374, + "theoretical_loss": 3.3482936589718673, + "tokens_seen": 2697658368 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001856969696969697, + "loss": 2.6149, + "theoretical_loss": 3.348287269093124, + "tokens_seen": 2697723904 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018567676767676768, + "loss": 2.5457, + "theoretical_loss": 3.348280879413071, + "tokens_seen": 2697789440 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018565656565656567, + "loss": 2.6961, + "theoretical_loss": 3.3482744899316987, + "tokens_seen": 2697854976 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018563636363636362, + "loss": 2.801, + "theoretical_loss": 3.3482681006489954, + "tokens_seen": 2697920512 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018561616161616164, + "loss": 2.4739, + "theoretical_loss": 3.34826171156495, + "tokens_seen": 2697986048 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001855959595959596, + "loss": 2.3506, + "theoretical_loss": 3.3482553226795515, + "tokens_seen": 2698051584 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001855757575757576, + "loss": 2.8089, + "theoretical_loss": 3.348248933992789, + "tokens_seen": 2698117120 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018555555555555556, + "loss": 2.5989, + "theoretical_loss": 3.3482425455046516, + "tokens_seen": 2698182656 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018553535353535352, + "loss": 2.5149, + "theoretical_loss": 3.3482361572151285, + "tokens_seen": 2698248192 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018551515151515153, + "loss": 2.4749, + "theoretical_loss": 3.3482297691242078, + "tokens_seen": 2698313728 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018549494949494949, + "loss": 2.6245, + "theoretical_loss": 3.34822338123188, + "tokens_seen": 2698379264 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001854747474747475, + "loss": 2.3438, + "theoretical_loss": 3.3482169935381325, + "tokens_seen": 2698444800 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018545454545454545, + "loss": 2.6608, + "theoretical_loss": 3.3482106060429553, + "tokens_seen": 2698510336 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018543434343434344, + "loss": 2.7145, + "theoretical_loss": 3.348204218746337, + "tokens_seen": 2698575872 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018541414141414142, + "loss": 2.7103, + "theoretical_loss": 3.348197831648267, + "tokens_seen": 2698641408 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1519063, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4194154739379883, + "objective/train/theoretical_loss": 3.3481914447487338, + "objective/train/tokens_used": 1057566176, + "theoretical_loss": 3.3481914447487338, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853939393939394, + "loss": 2.4357, + "theoretical_loss": 3.3481914447487338, + "tokens_seen": 2698706944 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001853737373737374, + "loss": 2.6214, + "theoretical_loss": 3.3481850580477266, + "tokens_seen": 2698772480 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018535353535353535, + "loss": 2.7215, + "theoretical_loss": 3.3481786715452344, + "tokens_seen": 2698838016 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018533333333333333, + "loss": 2.3841, + "theoretical_loss": 3.3481722852412465, + "tokens_seen": 2698903552 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018531313131313131, + "loss": 2.8113, + "theoretical_loss": 3.348165899135752, + "tokens_seen": 2698969088 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001852929292929293, + "loss": 2.6973, + "theoretical_loss": 3.348159513228739, + "tokens_seen": 2699034624 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018527272727272728, + "loss": 2.6011, + "theoretical_loss": 3.348153127520197, + "tokens_seen": 2699100160 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018525252525252527, + "loss": 2.5087, + "theoretical_loss": 3.348146742010116, + "tokens_seen": 2699165696 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018523232323232322, + "loss": 2.4608, + "theoretical_loss": 3.348140356698483, + "tokens_seen": 2699231232 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018521212121212123, + "loss": 2.8851, + "theoretical_loss": 3.348133971585289, + "tokens_seen": 2699296768 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001851919191919192, + "loss": 2.7126, + "theoretical_loss": 3.348127586670522, + "tokens_seen": 2699362304 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018517171717171717, + "loss": 2.684, + "theoretical_loss": 3.348121201954171, + "tokens_seen": 2699427840 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018515151515151516, + "loss": 2.5202, + "theoretical_loss": 3.348114817436225, + "tokens_seen": 2699493376 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018513131313131312, + "loss": 2.716, + "theoretical_loss": 3.3481084331166735, + "tokens_seen": 2699558912 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018511111111111113, + "loss": 2.5323, + "theoretical_loss": 3.3481020489955053, + "tokens_seen": 2699624448 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018509090909090908, + "loss": 2.6032, + "theoretical_loss": 3.3480956650727096, + "tokens_seen": 2699689984 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001850707070707071, + "loss": 2.77, + "theoretical_loss": 3.3480892813482743, + "tokens_seen": 2699755520 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018505050505050505, + "loss": 2.8415, + "theoretical_loss": 3.34808289782219, + "tokens_seen": 2699821056 + }, + { + "epoch": 0.63, + "learning_rate": 0.000185030303030303, + "loss": 2.5888, + "theoretical_loss": 3.348076514494445, + "tokens_seen": 2699886592 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018501010101010102, + "loss": 2.6039, + "theoretical_loss": 3.348070131365028, + "tokens_seen": 2699952128 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018498989898989898, + "loss": 2.6769, + "theoretical_loss": 3.348063748433929, + "tokens_seen": 2700017664 + }, + { + "epoch": 0.63, + "learning_rate": 0.000184969696969697, + "loss": 2.6663, + "theoretical_loss": 3.3480573657011354, + "tokens_seen": 2700083200 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018494949494949494, + "loss": 2.6881, + "theoretical_loss": 3.348050983166638, + "tokens_seen": 2700148736 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018492929292929293, + "loss": 2.5537, + "theoretical_loss": 3.3480446008304248, + "tokens_seen": 2700214272 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001849090909090909, + "loss": 2.6118, + "theoretical_loss": 3.348038218692485, + "tokens_seen": 2700279808 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1519691, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2327558994293213, + "objective/train/theoretical_loss": 3.3480318367528077, + "objective/train/tokens_used": 1059204576, + "theoretical_loss": 3.3480318367528077, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001848888888888889, + "loss": 2.4724, + "theoretical_loss": 3.3480318367528077, + "tokens_seen": 2700345344 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018486868686868688, + "loss": 2.7245, + "theoretical_loss": 3.3480254550113817, + "tokens_seen": 2700410880 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018484848484848484, + "loss": 2.8274, + "theoretical_loss": 3.3480190734681967, + "tokens_seen": 2700476416 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018482828282828285, + "loss": 2.8145, + "theoretical_loss": 3.348012692123241, + "tokens_seen": 2700541952 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001848080808080808, + "loss": 2.6703, + "theoretical_loss": 3.348006310976504, + "tokens_seen": 2700607488 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001847878787878788, + "loss": 2.6975, + "theoretical_loss": 3.347999930027975, + "tokens_seen": 2700673024 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018476767676767677, + "loss": 2.53, + "theoretical_loss": 3.3479935492776423, + "tokens_seen": 2700738560 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018474747474747476, + "loss": 2.6282, + "theoretical_loss": 3.347987168725495, + "tokens_seen": 2700804096 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018472727272727274, + "loss": 2.8317, + "theoretical_loss": 3.3479807883715234, + "tokens_seen": 2700869632 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018470707070707072, + "loss": 2.573, + "theoretical_loss": 3.347974408215715, + "tokens_seen": 2700935168 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018468686868686868, + "loss": 2.6501, + "theoretical_loss": 3.3479680282580593, + "tokens_seen": 2701000704 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018466666666666666, + "loss": 2.6805, + "theoretical_loss": 3.3479616484985457, + "tokens_seen": 2701066240 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018464646464646465, + "loss": 2.6262, + "theoretical_loss": 3.347955268937163, + "tokens_seen": 2701131776 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018462626262626263, + "loss": 2.4554, + "theoretical_loss": 3.3479488895739, + "tokens_seen": 2701197312 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018460606060606062, + "loss": 2.8706, + "theoretical_loss": 3.347942510408746, + "tokens_seen": 2701262848 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018458585858585857, + "loss": 2.7055, + "theoretical_loss": 3.3479361314416907, + "tokens_seen": 2701328384 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018456565656565658, + "loss": 2.5955, + "theoretical_loss": 3.347929752672722, + "tokens_seen": 2701393920 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018454545454545454, + "loss": 2.6614, + "theoretical_loss": 3.3479233741018293, + "tokens_seen": 2701459456 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018452525252525255, + "loss": 2.6484, + "theoretical_loss": 3.347916995729002, + "tokens_seen": 2701524992 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001845050505050505, + "loss": 2.5144, + "theoretical_loss": 3.347910617554229, + "tokens_seen": 2701590528 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018448484848484847, + "loss": 2.5729, + "theoretical_loss": 3.3479042395774985, + "tokens_seen": 2701656064 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018446464646464648, + "loss": 2.529, + "theoretical_loss": 3.347897861798801, + "tokens_seen": 2701721600 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018444444444444443, + "loss": 2.7664, + "theoretical_loss": 3.347891484218125, + "tokens_seen": 2701787136 + }, + { + "epoch": 0.63, + "learning_rate": 0.00018442424242424245, + "loss": 2.6695, + "theoretical_loss": 3.3478851068354585, + "tokens_seen": 2701852672 + }, + { + "epoch": 0.63, + "learning_rate": 0.0001844040404040404, + "loss": 2.5959, + "theoretical_loss": 3.3478787296507924, + "tokens_seen": 2701918208 + }, + { + "epoch": 0.63, + "objective/train/docs_used": 1520269, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.024388313293457, + "objective/train/theoretical_loss": 3.347872352664114, + "objective/train/tokens_used": 1060842976, + "theoretical_loss": 3.347872352664114, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018438383838383839, + "loss": 2.8268, + "theoretical_loss": 3.347872352664114, + "tokens_seen": 2701983744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018436363636363637, + "loss": 2.7091, + "theoretical_loss": 3.347865975875414, + "tokens_seen": 2702049280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018434343434343435, + "loss": 2.8673, + "theoretical_loss": 3.34785959928468, + "tokens_seen": 2702114816 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018432323232323234, + "loss": 2.6644, + "theoretical_loss": 3.3478532228919016, + "tokens_seen": 2702180352 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001843030303030303, + "loss": 2.7297, + "theoretical_loss": 3.347846846697068, + "tokens_seen": 2702245888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018428282828282828, + "loss": 2.578, + "theoretical_loss": 3.3478404707001683, + "tokens_seen": 2702311424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018426262626262626, + "loss": 2.6762, + "theoretical_loss": 3.3478340949011915, + "tokens_seen": 2702376960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018424242424242425, + "loss": 2.845, + "theoretical_loss": 3.3478277193001267, + "tokens_seen": 2702442496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018422222222222223, + "loss": 2.4354, + "theoretical_loss": 3.347821343896962, + "tokens_seen": 2702508032 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018420202020202021, + "loss": 2.735, + "theoretical_loss": 3.3478149686916883, + "tokens_seen": 2702573568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018418181818181817, + "loss": 2.6821, + "theoretical_loss": 3.3478085936842934, + "tokens_seen": 2702639104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018416161616161616, + "loss": 2.4392, + "theoretical_loss": 3.347802218874766, + "tokens_seen": 2702704640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018414141414141414, + "loss": 2.8934, + "theoretical_loss": 3.3477958442630964, + "tokens_seen": 2702770176 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018412121212121212, + "loss": 2.6483, + "theoretical_loss": 3.347789469849273, + "tokens_seen": 2702835712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001841010101010101, + "loss": 2.4378, + "theoretical_loss": 3.3477830956332846, + "tokens_seen": 2702901248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018408080808080806, + "loss": 2.642, + "theoretical_loss": 3.3477767216151206, + "tokens_seen": 2702966784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018406060606060608, + "loss": 2.6657, + "theoretical_loss": 3.34777034779477, + "tokens_seen": 2703032320 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018404040404040403, + "loss": 2.5204, + "theoretical_loss": 3.347763974172222, + "tokens_seen": 2703097856 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018402020202020204, + "loss": 2.6923, + "theoretical_loss": 3.3477576007474656, + "tokens_seen": 2703163392 + }, + { + "epoch": 0.64, + "learning_rate": 0.000184, + "loss": 2.491, + "theoretical_loss": 3.3477512275204897, + "tokens_seen": 2703228928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018397979797979798, + "loss": 2.6024, + "theoretical_loss": 3.3477448544912836, + "tokens_seen": 2703294464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018395959595959597, + "loss": 2.5975, + "theoretical_loss": 3.347738481659836, + "tokens_seen": 2703360000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018393939393939392, + "loss": 2.466, + "theoretical_loss": 3.3477321090261363, + "tokens_seen": 2703425536 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018391919191919194, + "loss": 2.4724, + "theoretical_loss": 3.347725736590174, + "tokens_seen": 2703491072 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001838989898989899, + "loss": 2.3406, + "theoretical_loss": 3.347719364351937, + "tokens_seen": 2703556608 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1521568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.494840621948242, + "objective/train/theoretical_loss": 3.3477129923114153, + "objective/train/tokens_used": 1062481376, + "theoretical_loss": 3.3477129923114153, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001838787878787879, + "loss": 2.424, + "theoretical_loss": 3.3477129923114153, + "tokens_seen": 2703622144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018385858585858586, + "loss": 2.6127, + "theoretical_loss": 3.347706620468598, + "tokens_seen": 2703687680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018383838383838384, + "loss": 2.7342, + "theoretical_loss": 3.3477002488234735, + "tokens_seen": 2703753216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018381818181818183, + "loss": 2.5359, + "theoretical_loss": 3.347693877376031, + "tokens_seen": 2703818752 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018379797979797979, + "loss": 2.8505, + "theoretical_loss": 3.3476875061262605, + "tokens_seen": 2703884288 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001837777777777778, + "loss": 2.674, + "theoretical_loss": 3.3476811350741498, + "tokens_seen": 2703949824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018375757575757575, + "loss": 2.3084, + "theoretical_loss": 3.347674764219689, + "tokens_seen": 2704015360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018373737373737374, + "loss": 2.422, + "theoretical_loss": 3.3476683935628664, + "tokens_seen": 2704080896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018371717171717172, + "loss": 2.5229, + "theoretical_loss": 3.3476620231036716, + "tokens_seen": 2704146432 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836969696969697, + "loss": 2.3097, + "theoretical_loss": 3.3476556528420938, + "tokens_seen": 2704211968 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001836767676767677, + "loss": 2.7267, + "theoretical_loss": 3.3476492827781215, + "tokens_seen": 2704277504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018365656565656567, + "loss": 2.5939, + "theoretical_loss": 3.3476429129117444, + "tokens_seen": 2704343040 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018363636363636363, + "loss": 2.5805, + "theoretical_loss": 3.347636543242951, + "tokens_seen": 2704408576 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018361616161616161, + "loss": 2.7607, + "theoretical_loss": 3.347630173771731, + "tokens_seen": 2704474112 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001835959595959596, + "loss": 2.6447, + "theoretical_loss": 3.3476238044980726, + "tokens_seen": 2704539648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018357575757575758, + "loss": 2.5767, + "theoretical_loss": 3.3476174354219657, + "tokens_seen": 2704605184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018355555555555557, + "loss": 2.5843, + "theoretical_loss": 3.347611066543399, + "tokens_seen": 2704670720 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018353535353535352, + "loss": 2.4304, + "theoretical_loss": 3.347604697862362, + "tokens_seen": 2704736256 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018351515151515153, + "loss": 2.571, + "theoretical_loss": 3.347598329378843, + "tokens_seen": 2704801792 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001834949494949495, + "loss": 2.6495, + "theoretical_loss": 3.347591961092832, + "tokens_seen": 2704867328 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001834747474747475, + "loss": 2.5595, + "theoretical_loss": 3.3475855930043172, + "tokens_seen": 2704932864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018345454545454546, + "loss": 2.6233, + "theoretical_loss": 3.3475792251132885, + "tokens_seen": 2704998400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018343434343434342, + "loss": 2.609, + "theoretical_loss": 3.3475728574197348, + "tokens_seen": 2705063936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018341414141414143, + "loss": 2.2552, + "theoretical_loss": 3.347566489923645, + "tokens_seen": 2705129472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018339393939393938, + "loss": 2.5372, + "theoretical_loss": 3.347560122625008, + "tokens_seen": 2705195008 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1522374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5427770614624023, + "objective/train/theoretical_loss": 3.3475537555238133, + "objective/train/tokens_used": 1064119776, + "theoretical_loss": 3.3475537555238133, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001833737373737374, + "loss": 2.6784, + "theoretical_loss": 3.3475537555238133, + "tokens_seen": 2705260544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018335353535353535, + "loss": 2.6821, + "theoretical_loss": 3.3475473886200495, + "tokens_seen": 2705326080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018333333333333334, + "loss": 2.6126, + "theoretical_loss": 3.347541021913706, + "tokens_seen": 2705391616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018331313131313132, + "loss": 2.7621, + "theoretical_loss": 3.3475346554047722, + "tokens_seen": 2705457152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018329292929292928, + "loss": 2.7536, + "theoretical_loss": 3.347528289093237, + "tokens_seen": 2705522688 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832727272727273, + "loss": 2.6668, + "theoretical_loss": 3.347521922979089, + "tokens_seen": 2705588224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018325252525252524, + "loss": 2.5133, + "theoretical_loss": 3.347515557062318, + "tokens_seen": 2705653760 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018323232323232323, + "loss": 2.575, + "theoretical_loss": 3.3475091913429127, + "tokens_seen": 2705719296 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001832121212121212, + "loss": 2.5177, + "theoretical_loss": 3.347502825820863, + "tokens_seen": 2705784832 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001831919191919192, + "loss": 2.553, + "theoretical_loss": 3.3474964604961563, + "tokens_seen": 2705850368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018317171717171718, + "loss": 2.8401, + "theoretical_loss": 3.3474900953687827, + "tokens_seen": 2705915904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018315151515151516, + "loss": 2.5866, + "theoretical_loss": 3.3474837304387317, + "tokens_seen": 2705981440 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018313131313131315, + "loss": 2.8389, + "theoretical_loss": 3.347477365705992, + "tokens_seen": 2706046976 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001831111111111111, + "loss": 2.5233, + "theoretical_loss": 3.3474710011705526, + "tokens_seen": 2706112512 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001830909090909091, + "loss": 2.5091, + "theoretical_loss": 3.3474646368324033, + "tokens_seen": 2706178048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018307070707070707, + "loss": 2.4786, + "theoretical_loss": 3.347458272691532, + "tokens_seen": 2706243584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018305050505050506, + "loss": 2.6455, + "theoretical_loss": 3.3474519087479284, + "tokens_seen": 2706309120 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018303030303030304, + "loss": 2.5152, + "theoretical_loss": 3.347445545001582, + "tokens_seen": 2706374656 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018301010101010102, + "loss": 2.4542, + "theoretical_loss": 3.347439181452481, + "tokens_seen": 2706440192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018298989898989898, + "loss": 2.4541, + "theoretical_loss": 3.3474328181006157, + "tokens_seen": 2706505728 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182969696969697, + "loss": 2.741, + "theoretical_loss": 3.3474264549459742, + "tokens_seen": 2706571264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018294949494949495, + "loss": 2.2504, + "theoretical_loss": 3.347420091988546, + "tokens_seen": 2706636800 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018292929292929293, + "loss": 2.5658, + "theoretical_loss": 3.347413729228321, + "tokens_seen": 2706702336 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018290909090909092, + "loss": 2.6866, + "theoretical_loss": 3.3474073666652866, + "tokens_seen": 2706767872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018288888888888887, + "loss": 2.7382, + "theoretical_loss": 3.3474010042994333, + "tokens_seen": 2706833408 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1523555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6162068843841553, + "objective/train/theoretical_loss": 3.347394642130749, + "objective/train/tokens_used": 1065758176, + "theoretical_loss": 3.347394642130749, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018286868686868689, + "loss": 2.3889, + "theoretical_loss": 3.347394642130749, + "tokens_seen": 2706898944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018284848484848484, + "loss": 2.7553, + "theoretical_loss": 3.3473882801592243, + "tokens_seen": 2706964480 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018282828282828285, + "loss": 2.7308, + "theoretical_loss": 3.3473819183848477, + "tokens_seen": 2707030016 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001828080808080808, + "loss": 2.6367, + "theoretical_loss": 3.3473755568076076, + "tokens_seen": 2707095552 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001827878787878788, + "loss": 2.4337, + "theoretical_loss": 3.3473691954274942, + "tokens_seen": 2707161088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018276767676767678, + "loss": 2.5889, + "theoretical_loss": 3.3473628342444957, + "tokens_seen": 2707226624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018274747474747473, + "loss": 2.4032, + "theoretical_loss": 3.347356473258602, + "tokens_seen": 2707292160 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018272727272727275, + "loss": 2.7051, + "theoretical_loss": 3.347350112469802, + "tokens_seen": 2707357696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001827070707070707, + "loss": 2.5007, + "theoretical_loss": 3.347343751878084, + "tokens_seen": 2707423232 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001826868686868687, + "loss": 2.693, + "theoretical_loss": 3.3473373914834386, + "tokens_seen": 2707488768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018266666666666667, + "loss": 2.4702, + "theoretical_loss": 3.3473310312858535, + "tokens_seen": 2707554304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018264646464646465, + "loss": 2.5545, + "theoretical_loss": 3.3473246712853193, + "tokens_seen": 2707619840 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018262626262626264, + "loss": 2.5202, + "theoretical_loss": 3.347318311481824, + "tokens_seen": 2707685376 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018260606060606062, + "loss": 2.4567, + "theoretical_loss": 3.3473119518753567, + "tokens_seen": 2707750912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018258585858585858, + "loss": 2.5155, + "theoretical_loss": 3.347305592465907, + "tokens_seen": 2707816448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018256565656565656, + "loss": 2.7778, + "theoretical_loss": 3.3472992332534637, + "tokens_seen": 2707881984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018254545454545455, + "loss": 2.4171, + "theoretical_loss": 3.347292874238016, + "tokens_seen": 2707947520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018252525252525253, + "loss": 2.7939, + "theoretical_loss": 3.3472865154195537, + "tokens_seen": 2708013056 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018250505050505052, + "loss": 2.8638, + "theoretical_loss": 3.3472801567980652, + "tokens_seen": 2708078592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018248484848484847, + "loss": 2.5725, + "theoretical_loss": 3.34727379837354, + "tokens_seen": 2708144128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018246464646464648, + "loss": 2.5823, + "theoretical_loss": 3.3472674401459663, + "tokens_seen": 2708209664 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018244444444444444, + "loss": 2.4917, + "theoretical_loss": 3.3472610821153346, + "tokens_seen": 2708275200 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018242424242424242, + "loss": 2.3957, + "theoretical_loss": 3.3472547242816333, + "tokens_seen": 2708340736 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001824040404040404, + "loss": 2.6222, + "theoretical_loss": 3.347248366644852, + "tokens_seen": 2708406272 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018238383838383836, + "loss": 2.8078, + "theoretical_loss": 3.3472420092049786, + "tokens_seen": 2708471808 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1524121, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815988063812256, + "objective/train/theoretical_loss": 3.347235651962004, + "objective/train/tokens_used": 1067396576, + "theoretical_loss": 3.347235651962004, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018236363636363638, + "loss": 2.5752, + "theoretical_loss": 3.347235651962004, + "tokens_seen": 2708537344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018234343434343433, + "loss": 2.5931, + "theoretical_loss": 3.3472292949159157, + "tokens_seen": 2708602880 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018232323232323234, + "loss": 2.7454, + "theoretical_loss": 3.347222938066704, + "tokens_seen": 2708668416 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001823030303030303, + "loss": 2.4505, + "theoretical_loss": 3.3472165814143575, + "tokens_seen": 2708733952 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001822828282828283, + "loss": 2.7081, + "theoretical_loss": 3.3472102249588653, + "tokens_seen": 2708799488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018226262626262627, + "loss": 2.601, + "theoretical_loss": 3.347203868700217, + "tokens_seen": 2708865024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018224242424242423, + "loss": 2.4889, + "theoretical_loss": 3.3471975126384015, + "tokens_seen": 2708930560 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018222222222222224, + "loss": 2.5379, + "theoretical_loss": 3.347191156773408, + "tokens_seen": 2708996096 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001822020202020202, + "loss": 2.7589, + "theoretical_loss": 3.347184801105225, + "tokens_seen": 2709061632 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821818181818182, + "loss": 2.493, + "theoretical_loss": 3.3471784456338427, + "tokens_seen": 2709127168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018216161616161616, + "loss": 2.5968, + "theoretical_loss": 3.3471720903592495, + "tokens_seen": 2709192704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018214141414141415, + "loss": 2.6025, + "theoretical_loss": 3.3471657352814352, + "tokens_seen": 2709258240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018212121212121213, + "loss": 2.8367, + "theoretical_loss": 3.347159380400388, + "tokens_seen": 2709323776 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001821010101010101, + "loss": 2.5342, + "theoretical_loss": 3.3471530257160977, + "tokens_seen": 2709389312 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001820808080808081, + "loss": 2.7671, + "theoretical_loss": 3.347146671228553, + "tokens_seen": 2709454848 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018206060606060605, + "loss": 2.542, + "theoretical_loss": 3.347140316937744, + "tokens_seen": 2709520384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018204040404040404, + "loss": 2.7356, + "theoretical_loss": 3.347133962843659, + "tokens_seen": 2709585920 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018202020202020202, + "loss": 2.6187, + "theoretical_loss": 3.3471276089462876, + "tokens_seen": 2709651456 + }, + { + "epoch": 0.64, + "learning_rate": 0.000182, + "loss": 2.7284, + "theoretical_loss": 3.3471212552456184, + "tokens_seen": 2709716992 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181979797979798, + "loss": 2.7042, + "theoretical_loss": 3.347114901741641, + "tokens_seen": 2709782528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018195959595959597, + "loss": 2.7634, + "theoretical_loss": 3.347108548434344, + "tokens_seen": 2709848064 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018193939393939393, + "loss": 2.6456, + "theoretical_loss": 3.3471021953237177, + "tokens_seen": 2709913600 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018191919191919194, + "loss": 2.4433, + "theoretical_loss": 3.34709584240975, + "tokens_seen": 2709979136 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001818989898989899, + "loss": 2.5443, + "theoretical_loss": 3.347089489692431, + "tokens_seen": 2710044672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018187878787878788, + "loss": 2.6313, + "theoretical_loss": 3.347083137171749, + "tokens_seen": 2710110208 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1525702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2911176681518555, + "objective/train/theoretical_loss": 3.347076784847694, + "objective/train/tokens_used": 1069034976, + "theoretical_loss": 3.347076784847694, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018185858585858587, + "loss": 2.4136, + "theoretical_loss": 3.347076784847694, + "tokens_seen": 2710175744 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018183838383838382, + "loss": 2.5631, + "theoretical_loss": 3.3470704327202547, + "tokens_seen": 2710241280 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018181818181818183, + "loss": 2.5589, + "theoretical_loss": 3.3470640807894205, + "tokens_seen": 2710306816 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001817979797979798, + "loss": 2.559, + "theoretical_loss": 3.34705772905518, + "tokens_seen": 2710372352 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001817777777777778, + "loss": 2.5986, + "theoretical_loss": 3.347051377517523, + "tokens_seen": 2710437888 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018175757575757576, + "loss": 2.6749, + "theoretical_loss": 3.3470450261764384, + "tokens_seen": 2710503424 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018173737373737372, + "loss": 2.7142, + "theoretical_loss": 3.3470386750319157, + "tokens_seen": 2710568960 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018171717171717173, + "loss": 2.5826, + "theoretical_loss": 3.3470323240839432, + "tokens_seen": 2710634496 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018169696969696968, + "loss": 2.6258, + "theoretical_loss": 3.347025973332511, + "tokens_seen": 2710700032 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001816767676767677, + "loss": 2.7941, + "theoretical_loss": 3.3470196227776077, + "tokens_seen": 2710765568 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018165656565656565, + "loss": 2.6547, + "theoretical_loss": 3.3470132724192228, + "tokens_seen": 2710831104 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018163636363636364, + "loss": 2.4917, + "theoretical_loss": 3.3470069222573455, + "tokens_seen": 2710896640 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018161616161616162, + "loss": 2.571, + "theoretical_loss": 3.3470005722919645, + "tokens_seen": 2710962176 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001815959595959596, + "loss": 2.5283, + "theoretical_loss": 3.346994222523069, + "tokens_seen": 2711027712 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001815757575757576, + "loss": 2.6309, + "theoretical_loss": 3.346987872950649, + "tokens_seen": 2711093248 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018155555555555554, + "loss": 2.7787, + "theoretical_loss": 3.3469815235746925, + "tokens_seen": 2711158784 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018153535353535353, + "loss": 2.6253, + "theoretical_loss": 3.34697517439519, + "tokens_seen": 2711224320 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001815151515151515, + "loss": 2.5931, + "theoretical_loss": 3.3469688254121293, + "tokens_seen": 2711289856 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001814949494949495, + "loss": 2.7125, + "theoretical_loss": 3.3469624766255004, + "tokens_seen": 2711355392 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018147474747474748, + "loss": 2.6195, + "theoretical_loss": 3.346956128035292, + "tokens_seen": 2711420928 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018145454545454546, + "loss": 2.3652, + "theoretical_loss": 3.3469497796414944, + "tokens_seen": 2711486464 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018143434343434345, + "loss": 2.5474, + "theoretical_loss": 3.3469434314440956, + "tokens_seen": 2711552000 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018141414141414143, + "loss": 2.7577, + "theoretical_loss": 3.346937083443085, + "tokens_seen": 2711617536 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001813939393939394, + "loss": 2.6503, + "theoretical_loss": 3.346930735638452, + "tokens_seen": 2711683072 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018137373737373737, + "loss": 2.6497, + "theoretical_loss": 3.3469243880301853, + "tokens_seen": 2711748608 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1526439, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.650019407272339, + "objective/train/theoretical_loss": 3.3469180406182746, + "objective/train/tokens_used": 1070673376, + "theoretical_loss": 3.3469180406182746, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018135353535353536, + "loss": 2.6302, + "theoretical_loss": 3.3469180406182746, + "tokens_seen": 2711814144 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018133333333333334, + "loss": 2.7455, + "theoretical_loss": 3.3469116934027094, + "tokens_seen": 2711879680 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018131313131313132, + "loss": 2.7392, + "theoretical_loss": 3.346905346383478, + "tokens_seen": 2711945216 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018129292929292928, + "loss": 2.6802, + "theoretical_loss": 3.3468989995605702, + "tokens_seen": 2712010752 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001812727272727273, + "loss": 2.6624, + "theoretical_loss": 3.346892652933975, + "tokens_seen": 2712076288 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018125252525252525, + "loss": 2.7247, + "theoretical_loss": 3.3468863065036816, + "tokens_seen": 2712141824 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018123232323232326, + "loss": 2.5522, + "theoretical_loss": 3.346879960269679, + "tokens_seen": 2712207360 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018121212121212122, + "loss": 2.8344, + "theoretical_loss": 3.3468736142319564, + "tokens_seen": 2712272896 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018119191919191917, + "loss": 2.4204, + "theoretical_loss": 3.346867268390503, + "tokens_seen": 2712338432 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018117171717171719, + "loss": 3.0145, + "theoretical_loss": 3.346860922745309, + "tokens_seen": 2712403968 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018115151515151514, + "loss": 2.681, + "theoretical_loss": 3.3468545772963623, + "tokens_seen": 2712469504 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018113131313131315, + "loss": 2.6055, + "theoretical_loss": 3.3468482320436523, + "tokens_seen": 2712535040 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001811111111111111, + "loss": 2.5812, + "theoretical_loss": 3.3468418869871686, + "tokens_seen": 2712600576 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001810909090909091, + "loss": 2.3789, + "theoretical_loss": 3.3468355421269, + "tokens_seen": 2712666112 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018107070707070708, + "loss": 2.6857, + "theoretical_loss": 3.346829197462836, + "tokens_seen": 2712731648 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018105050505050506, + "loss": 2.6439, + "theoretical_loss": 3.3468228529949653, + "tokens_seen": 2712797184 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018103030303030305, + "loss": 2.7321, + "theoretical_loss": 3.3468165087232777, + "tokens_seen": 2712862720 + }, + { + "epoch": 0.64, + "learning_rate": 0.000181010101010101, + "loss": 2.7598, + "theoretical_loss": 3.3468101646477626, + "tokens_seen": 2712928256 + }, + { + "epoch": 0.64, + "learning_rate": 0.000180989898989899, + "loss": 2.5969, + "theoretical_loss": 3.3468038207684083, + "tokens_seen": 2712993792 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018096969696969697, + "loss": 2.515, + "theoretical_loss": 3.3467974770852043, + "tokens_seen": 2713059328 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018094949494949495, + "loss": 2.5531, + "theoretical_loss": 3.34679113359814, + "tokens_seen": 2713124864 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018092929292929294, + "loss": 2.5488, + "theoretical_loss": 3.346784790307205, + "tokens_seen": 2713190400 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018090909090909092, + "loss": 2.7071, + "theoretical_loss": 3.3467784472123876, + "tokens_seen": 2713255936 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018088888888888888, + "loss": 2.5754, + "theoretical_loss": 3.3467721043136773, + "tokens_seen": 2713321472 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018086868686868686, + "loss": 2.4793, + "theoretical_loss": 3.346765761611064, + "tokens_seen": 2713387008 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1527722, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8723199367523193, + "objective/train/theoretical_loss": 3.346759419104536, + "objective/train/tokens_used": 1072311776, + "theoretical_loss": 3.346759419104536, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018084848484848485, + "loss": 2.7227, + "theoretical_loss": 3.346759419104536, + "tokens_seen": 2713452544 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018082828282828283, + "loss": 2.7205, + "theoretical_loss": 3.3467530767940827, + "tokens_seen": 2713518080 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018080808080808082, + "loss": 2.7518, + "theoretical_loss": 3.346746734679694, + "tokens_seen": 2713583616 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018078787878787877, + "loss": 2.5883, + "theoretical_loss": 3.346740392761358, + "tokens_seen": 2713649152 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018076767676767678, + "loss": 2.346, + "theoretical_loss": 3.3467340510390646, + "tokens_seen": 2713714688 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018074747474747474, + "loss": 2.7637, + "theoretical_loss": 3.346727709512803, + "tokens_seen": 2713780224 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018072727272727275, + "loss": 2.7248, + "theoretical_loss": 3.3467213681825623, + "tokens_seen": 2713845760 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001807070707070707, + "loss": 2.8039, + "theoretical_loss": 3.3467150270483312, + "tokens_seen": 2713911296 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018068686868686867, + "loss": 2.4567, + "theoretical_loss": 3.3467086861101, + "tokens_seen": 2713976832 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018066666666666668, + "loss": 2.5921, + "theoretical_loss": 3.346702345367857, + "tokens_seen": 2714042368 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018064646464646463, + "loss": 2.4849, + "theoretical_loss": 3.3466960048215917, + "tokens_seen": 2714107904 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018062626262626264, + "loss": 2.4925, + "theoretical_loss": 3.3466896644712936, + "tokens_seen": 2714173440 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001806060606060606, + "loss": 2.696, + "theoretical_loss": 3.3466833243169516, + "tokens_seen": 2714238976 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805858585858586, + "loss": 2.6867, + "theoretical_loss": 3.3466769843585547, + "tokens_seen": 2714304512 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018056565656565657, + "loss": 2.6684, + "theoretical_loss": 3.346670644596092, + "tokens_seen": 2714370048 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018054545454545455, + "loss": 2.5516, + "theoretical_loss": 3.3466643050295537, + "tokens_seen": 2714435584 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018052525252525254, + "loss": 2.6636, + "theoretical_loss": 3.346657965658928, + "tokens_seen": 2714501120 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001805050505050505, + "loss": 2.4989, + "theoretical_loss": 3.3466516264842046, + "tokens_seen": 2714566656 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001804848484848485, + "loss": 2.4147, + "theoretical_loss": 3.346645287505373, + "tokens_seen": 2714632192 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018046464646464646, + "loss": 2.4126, + "theoretical_loss": 3.3466389487224215, + "tokens_seen": 2714697728 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018044444444444445, + "loss": 2.6729, + "theoretical_loss": 3.34663261013534, + "tokens_seen": 2714763264 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018042424242424243, + "loss": 2.5466, + "theoretical_loss": 3.3466262717441175, + "tokens_seen": 2714828800 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001804040404040404, + "loss": 2.784, + "theoretical_loss": 3.346619933548743, + "tokens_seen": 2714894336 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803838383838384, + "loss": 2.5255, + "theoretical_loss": 3.346613595549207, + "tokens_seen": 2714959872 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018036363636363638, + "loss": 2.6728, + "theoretical_loss": 3.3466072577454966, + "tokens_seen": 2715025408 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1528322, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7192542552948, + "objective/train/theoretical_loss": 3.346600920137603, + "objective/train/tokens_used": 1073950176, + "theoretical_loss": 3.346600920137603, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018034343434343434, + "loss": 2.5891, + "theoretical_loss": 3.346600920137603, + "tokens_seen": 2715090944 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018032323232323232, + "loss": 2.6888, + "theoretical_loss": 3.346594582725514, + "tokens_seen": 2715156480 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001803030303030303, + "loss": 2.6435, + "theoretical_loss": 3.3465882455092197, + "tokens_seen": 2715222016 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802828282828283, + "loss": 2.4234, + "theoretical_loss": 3.346581908488709, + "tokens_seen": 2715287552 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018026262626262627, + "loss": 2.7232, + "theoretical_loss": 3.3465755716639713, + "tokens_seen": 2715353088 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018024242424242423, + "loss": 2.6788, + "theoretical_loss": 3.3465692350349956, + "tokens_seen": 2715418624 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018022222222222224, + "loss": 2.6494, + "theoretical_loss": 3.3465628986017713, + "tokens_seen": 2715484160 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001802020202020202, + "loss": 2.5337, + "theoretical_loss": 3.3465565623642872, + "tokens_seen": 2715549696 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001801818181818182, + "loss": 2.4798, + "theoretical_loss": 3.3465502263225333, + "tokens_seen": 2715615232 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018016161616161617, + "loss": 2.729, + "theoretical_loss": 3.346543890476498, + "tokens_seen": 2715680768 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018014141414141412, + "loss": 2.5909, + "theoretical_loss": 3.346537554826171, + "tokens_seen": 2715746304 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018012121212121213, + "loss": 2.4227, + "theoretical_loss": 3.3465312193715415, + "tokens_seen": 2715811840 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001801010101010101, + "loss": 2.4335, + "theoretical_loss": 3.3465248841125987, + "tokens_seen": 2715877376 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001800808080808081, + "loss": 2.7959, + "theoretical_loss": 3.346518549049332, + "tokens_seen": 2715942912 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018006060606060606, + "loss": 2.792, + "theoretical_loss": 3.3465122141817303, + "tokens_seen": 2716008448 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018004040404040404, + "loss": 2.4213, + "theoretical_loss": 3.3465058795097833, + "tokens_seen": 2716073984 + }, + { + "epoch": 0.64, + "learning_rate": 0.00018002020202020203, + "loss": 2.4384, + "theoretical_loss": 3.3464995450334794, + "tokens_seen": 2716139520 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017999999999999998, + "loss": 2.6433, + "theoretical_loss": 3.3464932107528087, + "tokens_seen": 2716205056 + }, + { + "epoch": 0.64, + "learning_rate": 0.000179979797979798, + "loss": 2.7945, + "theoretical_loss": 3.3464868766677602, + "tokens_seen": 2716270592 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017995959595959595, + "loss": 2.6017, + "theoretical_loss": 3.346480542778323, + "tokens_seen": 2716336128 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017993939393939394, + "loss": 2.7188, + "theoretical_loss": 3.3464742090844863, + "tokens_seen": 2716401664 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017991919191919192, + "loss": 2.4667, + "theoretical_loss": 3.346467875586239, + "tokens_seen": 2716467200 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798989898989899, + "loss": 2.3768, + "theoretical_loss": 3.3464615422835715, + "tokens_seen": 2716532736 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798787878787879, + "loss": 2.5586, + "theoretical_loss": 3.346455209176472, + "tokens_seen": 2716598272 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017985858585858587, + "loss": 2.4814, + "theoretical_loss": 3.34644887626493, + "tokens_seen": 2716663808 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1529677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.125514268875122, + "objective/train/theoretical_loss": 3.346442543548935, + "objective/train/tokens_used": 1075588576, + "theoretical_loss": 3.346442543548935, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017983838383838383, + "loss": 2.2826, + "theoretical_loss": 3.346442543548935, + "tokens_seen": 2716729344 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001798181818181818, + "loss": 2.7132, + "theoretical_loss": 3.346436211028476, + "tokens_seen": 2716794880 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001797979797979798, + "loss": 2.6059, + "theoretical_loss": 3.346429878703542, + "tokens_seen": 2716860416 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017977777777777778, + "loss": 2.5245, + "theoretical_loss": 3.346423546574123, + "tokens_seen": 2716925952 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017975757575757576, + "loss": 2.4781, + "theoretical_loss": 3.3464172146402076, + "tokens_seen": 2716991488 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017973737373737375, + "loss": 2.6194, + "theoretical_loss": 3.346410882901785, + "tokens_seen": 2717057024 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017971717171717173, + "loss": 2.447, + "theoretical_loss": 3.346404551358845, + "tokens_seen": 2717122560 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001796969696969697, + "loss": 2.5621, + "theoretical_loss": 3.3463982200113764, + "tokens_seen": 2717188096 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001796767676767677, + "loss": 2.415, + "theoretical_loss": 3.3463918888593684, + "tokens_seen": 2717253632 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017965656565656566, + "loss": 2.7768, + "theoretical_loss": 3.346385557902811, + "tokens_seen": 2717319168 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017963636363636364, + "loss": 2.6953, + "theoretical_loss": 3.3463792271416923, + "tokens_seen": 2717384704 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017961616161616163, + "loss": 2.6317, + "theoretical_loss": 3.3463728965760025, + "tokens_seen": 2717450240 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017959595959595958, + "loss": 2.699, + "theoretical_loss": 3.34636656620573, + "tokens_seen": 2717515776 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001795757575757576, + "loss": 2.7071, + "theoretical_loss": 3.3463602360308653, + "tokens_seen": 2717581312 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017955555555555555, + "loss": 2.3763, + "theoretical_loss": 3.3463539060513963, + "tokens_seen": 2717646848 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017953535353535356, + "loss": 2.8181, + "theoretical_loss": 3.346347576267313, + "tokens_seen": 2717712384 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017951515151515152, + "loss": 2.7357, + "theoretical_loss": 3.346341246678605, + "tokens_seen": 2717777920 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794949494949495, + "loss": 2.6382, + "theoretical_loss": 3.346334917285261, + "tokens_seen": 2717843456 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017947474747474749, + "loss": 2.6105, + "theoretical_loss": 3.3463285880872697, + "tokens_seen": 2717908992 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017945454545454544, + "loss": 2.561, + "theoretical_loss": 3.346322259084621, + "tokens_seen": 2717974528 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017943434343434345, + "loss": 2.6691, + "theoretical_loss": 3.3463159302773047, + "tokens_seen": 2718040064 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001794141414141414, + "loss": 2.3302, + "theoretical_loss": 3.3463096016653093, + "tokens_seen": 2718105600 + }, + { + "epoch": 0.64, + "learning_rate": 0.0001793939393939394, + "loss": 2.6935, + "theoretical_loss": 3.3463032732486244, + "tokens_seen": 2718171136 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017937373737373738, + "loss": 2.5593, + "theoretical_loss": 3.3462969450272393, + "tokens_seen": 2718236672 + }, + { + "epoch": 0.64, + "learning_rate": 0.00017935353535353536, + "loss": 2.7442, + "theoretical_loss": 3.346290617001143, + "tokens_seen": 2718302208 + }, + { + "epoch": 0.64, + "objective/train/docs_used": 1530384, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0346944332122803, + "objective/train/theoretical_loss": 3.3462842891703244, + "objective/train/tokens_used": 1077226976, + "theoretical_loss": 3.3462842891703244, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017933333333333335, + "loss": 2.6172, + "theoretical_loss": 3.3462842891703244, + "tokens_seen": 2718367744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001793131313131313, + "loss": 2.6448, + "theoretical_loss": 3.3462779615347737, + "tokens_seen": 2718433280 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001792929292929293, + "loss": 2.5979, + "theoretical_loss": 3.34627163409448, + "tokens_seen": 2718498816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017927272727272727, + "loss": 2.6301, + "theoretical_loss": 3.3462653068494315, + "tokens_seen": 2718564352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017925252525252526, + "loss": 2.6051, + "theoretical_loss": 3.346258979799619, + "tokens_seen": 2718629888 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017923232323232324, + "loss": 2.5613, + "theoretical_loss": 3.3462526529450303, + "tokens_seen": 2718695424 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017921212121212122, + "loss": 2.5515, + "theoretical_loss": 3.346246326285656, + "tokens_seen": 2718760960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017919191919191918, + "loss": 2.6481, + "theoretical_loss": 3.3462399998214845, + "tokens_seen": 2718826496 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001791717171717172, + "loss": 2.6036, + "theoretical_loss": 3.3462336735525056, + "tokens_seen": 2718892032 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017915151515151515, + "loss": 2.6832, + "theoretical_loss": 3.346227347478708, + "tokens_seen": 2718957568 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017913131313131313, + "loss": 2.619, + "theoretical_loss": 3.3462210216000816, + "tokens_seen": 2719023104 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017911111111111112, + "loss": 2.5264, + "theoretical_loss": 3.346214695916615, + "tokens_seen": 2719088640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017909090909090907, + "loss": 2.3429, + "theoretical_loss": 3.3462083704282977, + "tokens_seen": 2719154176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017907070707070708, + "loss": 2.6247, + "theoretical_loss": 3.3462020451351195, + "tokens_seen": 2719219712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017905050505050504, + "loss": 2.4011, + "theoretical_loss": 3.3461957200370693, + "tokens_seen": 2719285248 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017903030303030305, + "loss": 2.4128, + "theoretical_loss": 3.346189395134136, + "tokens_seen": 2719350784 + }, + { + "epoch": 0.65, + "learning_rate": 0.000179010101010101, + "loss": 2.7148, + "theoretical_loss": 3.3461830704263096, + "tokens_seen": 2719416320 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178989898989899, + "loss": 2.5069, + "theoretical_loss": 3.3461767459135787, + "tokens_seen": 2719481856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017896969696969698, + "loss": 2.503, + "theoretical_loss": 3.346170421595933, + "tokens_seen": 2719547392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017894949494949493, + "loss": 2.6227, + "theoretical_loss": 3.346164097473362, + "tokens_seen": 2719612928 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017892929292929294, + "loss": 2.389, + "theoretical_loss": 3.3461577735458548, + "tokens_seen": 2719678464 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001789090909090909, + "loss": 2.4908, + "theoretical_loss": 3.3461514498133997, + "tokens_seen": 2719744000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017888888888888889, + "loss": 2.5992, + "theoretical_loss": 3.346145126275988, + "tokens_seen": 2719809536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017886868686868687, + "loss": 2.7344, + "theoretical_loss": 3.3461388029336065, + "tokens_seen": 2719875072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017884848484848485, + "loss": 2.5363, + "theoretical_loss": 3.3461324797862466, + "tokens_seen": 2719940608 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1531036, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4944186210632324, + "objective/train/theoretical_loss": 3.346126156833897, + "objective/train/tokens_used": 1078865376, + "theoretical_loss": 3.346126156833897, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017882828282828284, + "loss": 2.4942, + "theoretical_loss": 3.346126156833897, + "tokens_seen": 2720006144 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017880808080808082, + "loss": 2.5573, + "theoretical_loss": 3.3461198340765463, + "tokens_seen": 2720071680 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001787878787878788, + "loss": 2.6103, + "theoretical_loss": 3.3461135115141842, + "tokens_seen": 2720137216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017876767676767676, + "loss": 2.6675, + "theoretical_loss": 3.3461071891468004, + "tokens_seen": 2720202752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017874747474747475, + "loss": 2.6248, + "theoretical_loss": 3.3461008669743837, + "tokens_seen": 2720268288 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017872727272727273, + "loss": 2.5194, + "theoretical_loss": 3.3460945449969235, + "tokens_seen": 2720333824 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017870707070707071, + "loss": 2.6074, + "theoretical_loss": 3.346088223214409, + "tokens_seen": 2720399360 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786868686868687, + "loss": 2.6642, + "theoretical_loss": 3.34608190162683, + "tokens_seen": 2720464896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017866666666666668, + "loss": 2.5939, + "theoretical_loss": 3.3460755802341753, + "tokens_seen": 2720530432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017864646464646464, + "loss": 2.7182, + "theoretical_loss": 3.346069259036434, + "tokens_seen": 2720595968 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017862626262626265, + "loss": 2.7521, + "theoretical_loss": 3.346062938033596, + "tokens_seen": 2720661504 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001786060606060606, + "loss": 2.5542, + "theoretical_loss": 3.34605661722565, + "tokens_seen": 2720727040 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001785858585858586, + "loss": 2.6045, + "theoretical_loss": 3.346050296612586, + "tokens_seen": 2720792576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017856565656565657, + "loss": 2.4934, + "theoretical_loss": 3.3460439761943928, + "tokens_seen": 2720858112 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017854545454545453, + "loss": 2.8202, + "theoretical_loss": 3.3460376559710596, + "tokens_seen": 2720923648 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017852525252525254, + "loss": 2.6011, + "theoretical_loss": 3.346031335942576, + "tokens_seen": 2720989184 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001785050505050505, + "loss": 2.4672, + "theoretical_loss": 3.346025016108931, + "tokens_seen": 2721054720 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001784848484848485, + "loss": 2.5832, + "theoretical_loss": 3.3460186964701144, + "tokens_seen": 2721120256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017846464646464647, + "loss": 2.6986, + "theoretical_loss": 3.3460123770261148, + "tokens_seen": 2721185792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017844444444444442, + "loss": 2.4801, + "theoretical_loss": 3.3460060577769224, + "tokens_seen": 2721251328 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017842424242424244, + "loss": 2.4772, + "theoretical_loss": 3.3459997387225258, + "tokens_seen": 2721316864 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001784040404040404, + "loss": 2.3979, + "theoretical_loss": 3.345993419862914, + "tokens_seen": 2721382400 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783838383838384, + "loss": 2.5649, + "theoretical_loss": 3.3459871011980775, + "tokens_seen": 2721447936 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017836363636363636, + "loss": 2.4002, + "theoretical_loss": 3.3459807827280046, + "tokens_seen": 2721513472 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017834343434343434, + "loss": 2.5921, + "theoretical_loss": 3.345974464452685, + "tokens_seen": 2721579008 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1532195, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6007792949676514, + "objective/train/theoretical_loss": 3.3459681463721083, + "objective/train/tokens_used": 1080503776, + "theoretical_loss": 3.3459681463721083, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017832323232323233, + "loss": 2.7873, + "theoretical_loss": 3.3459681463721083, + "tokens_seen": 2721644544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001783030303030303, + "loss": 2.4629, + "theoretical_loss": 3.3459618284862627, + "tokens_seen": 2721710080 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782828282828283, + "loss": 2.6564, + "theoretical_loss": 3.3459555107951386, + "tokens_seen": 2721775616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017826262626262625, + "loss": 2.6652, + "theoretical_loss": 3.345949193298725, + "tokens_seen": 2721841152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017824242424242424, + "loss": 2.5586, + "theoretical_loss": 3.3459428759970113, + "tokens_seen": 2721906688 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017822222222222222, + "loss": 2.5931, + "theoretical_loss": 3.3459365588899863, + "tokens_seen": 2721972224 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001782020202020202, + "loss": 2.3858, + "theoretical_loss": 3.34593024197764, + "tokens_seen": 2722037760 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001781818181818182, + "loss": 2.6381, + "theoretical_loss": 3.3459239252599615, + "tokens_seen": 2722103296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017816161616161617, + "loss": 2.6755, + "theoretical_loss": 3.34591760873694, + "tokens_seen": 2722168832 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017814141414141413, + "loss": 2.7783, + "theoretical_loss": 3.3459112924085646, + "tokens_seen": 2722234368 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017812121212121214, + "loss": 2.3325, + "theoretical_loss": 3.345904976274825, + "tokens_seen": 2722299904 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001781010101010101, + "loss": 2.584, + "theoretical_loss": 3.34589866033571, + "tokens_seen": 2722365440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017808080808080808, + "loss": 2.7913, + "theoretical_loss": 3.34589234459121, + "tokens_seen": 2722430976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017806060606060607, + "loss": 2.3995, + "theoretical_loss": 3.345886029041313, + "tokens_seen": 2722496512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017804040404040402, + "loss": 2.6519, + "theoretical_loss": 3.345879713686009, + "tokens_seen": 2722562048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017802020202020203, + "loss": 2.5513, + "theoretical_loss": 3.3458733985252875, + "tokens_seen": 2722627584 + }, + { + "epoch": 0.65, + "learning_rate": 0.000178, + "loss": 2.4782, + "theoretical_loss": 3.3458670835591375, + "tokens_seen": 2722693120 + }, + { + "epoch": 0.65, + "learning_rate": 0.000177979797979798, + "loss": 2.5113, + "theoretical_loss": 3.3458607687875483, + "tokens_seen": 2722758656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017795959595959596, + "loss": 2.6745, + "theoretical_loss": 3.3458544542105093, + "tokens_seen": 2722824192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017793939393939397, + "loss": 2.719, + "theoretical_loss": 3.3458481398280098, + "tokens_seen": 2722889728 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017791919191919193, + "loss": 2.3759, + "theoretical_loss": 3.345841825640039, + "tokens_seen": 2722955264 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017789898989898988, + "loss": 2.5675, + "theoretical_loss": 3.345835511646587, + "tokens_seen": 2723020800 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001778787878787879, + "loss": 2.4883, + "theoretical_loss": 3.345829197847642, + "tokens_seen": 2723086336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017785858585858585, + "loss": 2.5825, + "theoretical_loss": 3.345822884243194, + "tokens_seen": 2723151872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017783838383838386, + "loss": 2.4593, + "theoretical_loss": 3.3458165708332324, + "tokens_seen": 2723217408 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1532891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0827062129974365, + "objective/train/theoretical_loss": 3.345810257617746, + "objective/train/tokens_used": 1082142176, + "theoretical_loss": 3.345810257617746, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017781818181818182, + "loss": 2.4514, + "theoretical_loss": 3.345810257617746, + "tokens_seen": 2723282944 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001777979797979798, + "loss": 2.6241, + "theoretical_loss": 3.3458039445967245, + "tokens_seen": 2723348480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017777777777777779, + "loss": 2.3651, + "theoretical_loss": 3.3457976317701568, + "tokens_seen": 2723414016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017775757575757577, + "loss": 2.5027, + "theoretical_loss": 3.3457913191380326, + "tokens_seen": 2723479552 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017773737373737375, + "loss": 2.3628, + "theoretical_loss": 3.345785006700342, + "tokens_seen": 2723545088 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001777171717171717, + "loss": 2.5569, + "theoretical_loss": 3.345778694457073, + "tokens_seen": 2723610624 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001776969696969697, + "loss": 2.5516, + "theoretical_loss": 3.3457723824082155, + "tokens_seen": 2723676160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017767676767676768, + "loss": 2.6346, + "theoretical_loss": 3.3457660705537586, + "tokens_seen": 2723741696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017765656565656566, + "loss": 2.5167, + "theoretical_loss": 3.3457597588936925, + "tokens_seen": 2723807232 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017763636363636365, + "loss": 2.4808, + "theoretical_loss": 3.3457534474280055, + "tokens_seen": 2723872768 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017761616161616163, + "loss": 2.5118, + "theoretical_loss": 3.3457471361566875, + "tokens_seen": 2723938304 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001775959595959596, + "loss": 2.514, + "theoretical_loss": 3.3457408250797274, + "tokens_seen": 2724003840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017757575757575757, + "loss": 2.6324, + "theoretical_loss": 3.345734514197115, + "tokens_seen": 2724069376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017755555555555556, + "loss": 2.5499, + "theoretical_loss": 3.3457282035088394, + "tokens_seen": 2724134912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017753535353535354, + "loss": 2.6839, + "theoretical_loss": 3.3457218930148898, + "tokens_seen": 2724200448 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017751515151515152, + "loss": 2.6134, + "theoretical_loss": 3.345715582715256, + "tokens_seen": 2724265984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017749494949494948, + "loss": 2.6295, + "theoretical_loss": 3.345709272609927, + "tokens_seen": 2724331520 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001774747474747475, + "loss": 2.4603, + "theoretical_loss": 3.3457029626988923, + "tokens_seen": 2724397056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017745454545454545, + "loss": 2.616, + "theoretical_loss": 3.345696652982141, + "tokens_seen": 2724462592 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017743434343434346, + "loss": 2.6753, + "theoretical_loss": 3.3456903434596628, + "tokens_seen": 2724528128 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017741414141414142, + "loss": 2.6805, + "theoretical_loss": 3.3456840341314464, + "tokens_seen": 2724593664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017739393939393937, + "loss": 2.9134, + "theoretical_loss": 3.345677724997482, + "tokens_seen": 2724659200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017737373737373738, + "loss": 2.5446, + "theoretical_loss": 3.3456714160577583, + "tokens_seen": 2724724736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017735353535353534, + "loss": 2.5592, + "theoretical_loss": 3.345665107312265, + "tokens_seen": 2724790272 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017733333333333335, + "loss": 2.5851, + "theoretical_loss": 3.3456587987609914, + "tokens_seen": 2724855808 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1534156, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.641813039779663, + "objective/train/theoretical_loss": 3.3456524904039266, + "objective/train/tokens_used": 1083780576, + "theoretical_loss": 3.3456524904039266, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001773131313131313, + "loss": 2.7055, + "theoretical_loss": 3.3456524904039266, + "tokens_seen": 2724921344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001772929292929293, + "loss": 2.5863, + "theoretical_loss": 3.3456461822410604, + "tokens_seen": 2724986880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017727272727272728, + "loss": 2.6298, + "theoretical_loss": 3.345639874272382, + "tokens_seen": 2725052416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017725252525252526, + "loss": 2.7789, + "theoretical_loss": 3.3456335664978805, + "tokens_seen": 2725117952 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017723232323232324, + "loss": 2.4926, + "theoretical_loss": 3.345627258917545, + "tokens_seen": 2725183488 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001772121212121212, + "loss": 2.6051, + "theoretical_loss": 3.345620951531366, + "tokens_seen": 2725249024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017719191919191919, + "loss": 2.6702, + "theoretical_loss": 3.3456146443393315, + "tokens_seen": 2725314560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017717171717171717, + "loss": 2.7227, + "theoretical_loss": 3.3456083373414316, + "tokens_seen": 2725380096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017715151515151515, + "loss": 2.7672, + "theoretical_loss": 3.3456020305376555, + "tokens_seen": 2725445632 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017713131313131314, + "loss": 2.7317, + "theoretical_loss": 3.3455957239279925, + "tokens_seen": 2725511168 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017711111111111112, + "loss": 2.6097, + "theoretical_loss": 3.345589417512432, + "tokens_seen": 2725576704 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770909090909091, + "loss": 2.6687, + "theoretical_loss": 3.345583111290964, + "tokens_seen": 2725642240 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001770707070707071, + "loss": 2.6508, + "theoretical_loss": 3.3455768052635766, + "tokens_seen": 2725707776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017705050505050505, + "loss": 2.6335, + "theoretical_loss": 3.34557049943026, + "tokens_seen": 2725773312 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017703030303030303, + "loss": 2.3611, + "theoretical_loss": 3.3455641937910037, + "tokens_seen": 2725838848 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017701010101010101, + "loss": 2.5213, + "theoretical_loss": 3.3455578883457964, + "tokens_seen": 2725904384 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176989898989899, + "loss": 2.7967, + "theoretical_loss": 3.3455515830946276, + "tokens_seen": 2725969920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017696969696969698, + "loss": 2.5097, + "theoretical_loss": 3.3455452780374872, + "tokens_seen": 2726035456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017694949494949494, + "loss": 2.4741, + "theoretical_loss": 3.345538973174364, + "tokens_seen": 2726100992 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017692929292929295, + "loss": 2.3151, + "theoretical_loss": 3.345532668505248, + "tokens_seen": 2726166528 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001769090909090909, + "loss": 2.5747, + "theoretical_loss": 3.345526364030128, + "tokens_seen": 2726232064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017688888888888892, + "loss": 2.7041, + "theoretical_loss": 3.345520059748993, + "tokens_seen": 2726297600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017686868686868687, + "loss": 2.6668, + "theoretical_loss": 3.3455137556618335, + "tokens_seen": 2726363136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017684848484848483, + "loss": 2.6414, + "theoretical_loss": 3.345507451768638, + "tokens_seen": 2726428672 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017682828282828284, + "loss": 2.5422, + "theoretical_loss": 3.3455011480693964, + "tokens_seen": 2726494208 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1534846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.791715621948242, + "objective/train/theoretical_loss": 3.3454948445640973, + "objective/train/tokens_used": 1085418976, + "theoretical_loss": 3.3454948445640973, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001768080808080808, + "loss": 2.695, + "theoretical_loss": 3.3454948445640973, + "tokens_seen": 2726559744 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001767878787878788, + "loss": 2.3782, + "theoretical_loss": 3.345488541252731, + "tokens_seen": 2726625280 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017676767676767677, + "loss": 2.6466, + "theoretical_loss": 3.3454822381352862, + "tokens_seen": 2726690816 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017674747474747475, + "loss": 2.5308, + "theoretical_loss": 3.345475935211753, + "tokens_seen": 2726756352 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017672727272727274, + "loss": 2.7702, + "theoretical_loss": 3.3454696324821196, + "tokens_seen": 2726821888 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001767070707070707, + "loss": 2.4083, + "theoretical_loss": 3.3454633299463765, + "tokens_seen": 2726887424 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766868686868687, + "loss": 2.5391, + "theoretical_loss": 3.345457027604513, + "tokens_seen": 2726952960 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017666666666666666, + "loss": 2.7046, + "theoretical_loss": 3.3454507254565176, + "tokens_seen": 2727018496 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017664646464646464, + "loss": 2.7499, + "theoretical_loss": 3.3454444235023804, + "tokens_seen": 2727084032 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017662626262626263, + "loss": 2.6037, + "theoretical_loss": 3.3454381217420903, + "tokens_seen": 2727149568 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001766060606060606, + "loss": 2.4631, + "theoretical_loss": 3.3454318201756372, + "tokens_seen": 2727215104 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765858585858586, + "loss": 2.5859, + "theoretical_loss": 3.34542551880301, + "tokens_seen": 2727280640 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017656565656565658, + "loss": 2.4369, + "theoretical_loss": 3.345419217624199, + "tokens_seen": 2727346176 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017654545454545454, + "loss": 2.503, + "theoretical_loss": 3.345412916639192, + "tokens_seen": 2727411712 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017652525252525252, + "loss": 2.7498, + "theoretical_loss": 3.34540661584798, + "tokens_seen": 2727477248 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001765050505050505, + "loss": 2.4314, + "theoretical_loss": 3.345400315250551, + "tokens_seen": 2727542784 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001764848484848485, + "loss": 2.5348, + "theoretical_loss": 3.3453940148468955, + "tokens_seen": 2727608320 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017646464646464647, + "loss": 2.7673, + "theoretical_loss": 3.3453877146370026, + "tokens_seen": 2727673856 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017644444444444443, + "loss": 2.6819, + "theoretical_loss": 3.345381414620861, + "tokens_seen": 2727739392 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017642424242424244, + "loss": 2.5132, + "theoretical_loss": 3.345375114798461, + "tokens_seen": 2727804928 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001764040404040404, + "loss": 2.4936, + "theoretical_loss": 3.345368815169792, + "tokens_seen": 2727870464 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001763838383838384, + "loss": 2.5909, + "theoretical_loss": 3.345362515734842, + "tokens_seen": 2727936000 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017636363636363637, + "loss": 2.3729, + "theoretical_loss": 3.345356216493602, + "tokens_seen": 2728001536 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017634343434343432, + "loss": 2.4448, + "theoretical_loss": 3.3453499174460606, + "tokens_seen": 2728067072 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017632323232323233, + "loss": 2.6747, + "theoretical_loss": 3.3453436185922074, + "tokens_seen": 2728132608 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1535777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.71197247505188, + "objective/train/theoretical_loss": 3.3453373199320318, + "objective/train/tokens_used": 1087057376, + "theoretical_loss": 3.3453373199320318, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001763030303030303, + "loss": 2.6606, + "theoretical_loss": 3.3453373199320318, + "tokens_seen": 2728198144 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001762828282828283, + "loss": 2.5449, + "theoretical_loss": 3.345331021465523, + "tokens_seen": 2728263680 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017626262626262626, + "loss": 2.7809, + "theoretical_loss": 3.3453247231926704, + "tokens_seen": 2728329216 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017624242424242427, + "loss": 2.7805, + "theoretical_loss": 3.345318425113464, + "tokens_seen": 2728394752 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017622222222222223, + "loss": 2.6313, + "theoretical_loss": 3.345312127227892, + "tokens_seen": 2728460288 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001762020202020202, + "loss": 2.6753, + "theoretical_loss": 3.3453058295359455, + "tokens_seen": 2728525824 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761818181818182, + "loss": 2.6021, + "theoretical_loss": 3.345299532037612, + "tokens_seen": 2728591360 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017616161616161615, + "loss": 2.5543, + "theoretical_loss": 3.3452932347328823, + "tokens_seen": 2728656896 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017614141414141416, + "loss": 2.8184, + "theoretical_loss": 3.3452869376217453, + "tokens_seen": 2728722432 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017612121212121212, + "loss": 2.6247, + "theoretical_loss": 3.3452806407041904, + "tokens_seen": 2728787968 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001761010101010101, + "loss": 2.2766, + "theoretical_loss": 3.345274343980207, + "tokens_seen": 2728853504 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001760808080808081, + "loss": 2.6327, + "theoretical_loss": 3.3452680474497845, + "tokens_seen": 2728919040 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017606060606060607, + "loss": 2.5347, + "theoretical_loss": 3.345261751112912, + "tokens_seen": 2728984576 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017604040404040405, + "loss": 2.5409, + "theoretical_loss": 3.3452554549695797, + "tokens_seen": 2729050112 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176020202020202, + "loss": 2.5113, + "theoretical_loss": 3.345249159019776, + "tokens_seen": 2729115648 + }, + { + "epoch": 0.65, + "learning_rate": 0.000176, + "loss": 2.708, + "theoretical_loss": 3.3452428632634916, + "tokens_seen": 2729181184 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017597979797979798, + "loss": 2.6431, + "theoretical_loss": 3.3452365677007143, + "tokens_seen": 2729246720 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017595959595959596, + "loss": 2.7648, + "theoretical_loss": 3.3452302723314347, + "tokens_seen": 2729312256 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017593939393939395, + "loss": 2.464, + "theoretical_loss": 3.345223977155642, + "tokens_seen": 2729377792 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017591919191919193, + "loss": 2.5857, + "theoretical_loss": 3.3452176821733253, + "tokens_seen": 2729443328 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758989898989899, + "loss": 2.581, + "theoretical_loss": 3.345211387384474, + "tokens_seen": 2729508864 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001758787878787879, + "loss": 2.5657, + "theoretical_loss": 3.3452050927890777, + "tokens_seen": 2729574400 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017585858585858586, + "loss": 2.7291, + "theoretical_loss": 3.345198798387126, + "tokens_seen": 2729639936 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017583838383838384, + "loss": 2.5709, + "theoretical_loss": 3.345192504178608, + "tokens_seen": 2729705472 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017581818181818182, + "loss": 2.4635, + "theoretical_loss": 3.3451862101635133, + "tokens_seen": 2729771008 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1536637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.034106731414795, + "objective/train/theoretical_loss": 3.345179916341831, + "objective/train/tokens_used": 1088695776, + "theoretical_loss": 3.345179916341831, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017579797979797978, + "loss": 2.8196, + "theoretical_loss": 3.345179916341831, + "tokens_seen": 2729836544 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001757777777777778, + "loss": 2.756, + "theoretical_loss": 3.345173622713551, + "tokens_seen": 2729902080 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017575757575757575, + "loss": 2.6102, + "theoretical_loss": 3.345167329278662, + "tokens_seen": 2729967616 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017573737373737376, + "loss": 2.5604, + "theoretical_loss": 3.345161036037154, + "tokens_seen": 2730033152 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017571717171717172, + "loss": 2.5978, + "theoretical_loss": 3.3451547429890165, + "tokens_seen": 2730098688 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001756969696969697, + "loss": 2.6844, + "theoretical_loss": 3.3451484501342383, + "tokens_seen": 2730164224 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017567676767676768, + "loss": 2.4824, + "theoretical_loss": 3.34514215747281, + "tokens_seen": 2730229760 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017565656565656564, + "loss": 2.4743, + "theoretical_loss": 3.3451358650047194, + "tokens_seen": 2730295296 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017563636363636365, + "loss": 2.5969, + "theoretical_loss": 3.3451295727299573, + "tokens_seen": 2730360832 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001756161616161616, + "loss": 2.3673, + "theoretical_loss": 3.345123280648512, + "tokens_seen": 2730426368 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001755959595959596, + "loss": 2.6293, + "theoretical_loss": 3.345116988760374, + "tokens_seen": 2730491904 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017557575757575758, + "loss": 2.5637, + "theoretical_loss": 3.345110697065532, + "tokens_seen": 2730557440 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017555555555555556, + "loss": 2.5869, + "theoretical_loss": 3.3451044055639754, + "tokens_seen": 2730622976 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017553535353535355, + "loss": 2.4159, + "theoretical_loss": 3.345098114255694, + "tokens_seen": 2730688512 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017551515151515153, + "loss": 2.6595, + "theoretical_loss": 3.3450918231406774, + "tokens_seen": 2730754048 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017549494949494949, + "loss": 2.527, + "theoretical_loss": 3.3450855322189144, + "tokens_seen": 2730819584 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017547474747474747, + "loss": 2.593, + "theoretical_loss": 3.345079241490395, + "tokens_seen": 2730885120 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017545454545454545, + "loss": 2.5549, + "theoretical_loss": 3.345072950955108, + "tokens_seen": 2730950656 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017543434343434344, + "loss": 2.589, + "theoretical_loss": 3.345066660613043, + "tokens_seen": 2731016192 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017541414141414142, + "loss": 2.6333, + "theoretical_loss": 3.34506037046419, + "tokens_seen": 2731081728 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753939393939394, + "loss": 2.7065, + "theoretical_loss": 3.345054080508538, + "tokens_seen": 2731147264 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001753737373737374, + "loss": 2.4986, + "theoretical_loss": 3.3450477907460767, + "tokens_seen": 2731212800 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017535353535353535, + "loss": 2.4757, + "theoretical_loss": 3.345041501176795, + "tokens_seen": 2731278336 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017533333333333336, + "loss": 2.695, + "theoretical_loss": 3.3450352118006825, + "tokens_seen": 2731343872 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017531313131313131, + "loss": 2.6056, + "theoretical_loss": 3.345028922617729, + "tokens_seen": 2731409408 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1538108, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.700874090194702, + "objective/train/theoretical_loss": 3.3450226336279236, + "objective/train/tokens_used": 1090334176, + "theoretical_loss": 3.3450226336279236, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752929292929293, + "loss": 2.7573, + "theoretical_loss": 3.3450226336279236, + "tokens_seen": 2731474944 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017527272727272728, + "loss": 2.6718, + "theoretical_loss": 3.3450163448312558, + "tokens_seen": 2731540480 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017525252525252524, + "loss": 2.5583, + "theoretical_loss": 3.345010056227715, + "tokens_seen": 2731606016 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017523232323232325, + "loss": 2.7239, + "theoretical_loss": 3.345003767817291, + "tokens_seen": 2731671552 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001752121212121212, + "loss": 2.5671, + "theoretical_loss": 3.3449974795999724, + "tokens_seen": 2731737088 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017519191919191922, + "loss": 2.6069, + "theoretical_loss": 3.3449911915757498, + "tokens_seen": 2731802624 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017517171717171718, + "loss": 2.6537, + "theoretical_loss": 3.3449849037446118, + "tokens_seen": 2731868160 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017515151515151513, + "loss": 2.5632, + "theoretical_loss": 3.344978616106548, + "tokens_seen": 2731933696 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017513131313131314, + "loss": 2.9292, + "theoretical_loss": 3.344972328661548, + "tokens_seen": 2731999232 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001751111111111111, + "loss": 2.6601, + "theoretical_loss": 3.344966041409601, + "tokens_seen": 2732064768 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001750909090909091, + "loss": 2.5808, + "theoretical_loss": 3.3449597543506964, + "tokens_seen": 2732130304 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017507070707070707, + "loss": 2.5452, + "theoretical_loss": 3.344953467484824, + "tokens_seen": 2732195840 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017505050505050505, + "loss": 2.5687, + "theoretical_loss": 3.3449471808119733, + "tokens_seen": 2732261376 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017503030303030304, + "loss": 2.4525, + "theoretical_loss": 3.344940894332133, + "tokens_seen": 2732326912 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017501010101010102, + "loss": 2.6365, + "theoretical_loss": 3.3449346080452935, + "tokens_seen": 2732392448 + }, + { + "epoch": 0.65, + "learning_rate": 0.000174989898989899, + "loss": 2.6612, + "theoretical_loss": 3.3449283219514436, + "tokens_seen": 2732457984 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017496969696969696, + "loss": 2.5083, + "theoretical_loss": 3.344922036050573, + "tokens_seen": 2732523520 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017494949494949494, + "loss": 2.6089, + "theoretical_loss": 3.344915750342671, + "tokens_seen": 2732589056 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017492929292929293, + "loss": 2.643, + "theoretical_loss": 3.3449094648277273, + "tokens_seen": 2732654592 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001749090909090909, + "loss": 2.622, + "theoretical_loss": 3.344903179505731, + "tokens_seen": 2732720128 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001748888888888889, + "loss": 2.6918, + "theoretical_loss": 3.344896894376672, + "tokens_seen": 2732785664 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017486868686868688, + "loss": 2.6132, + "theoretical_loss": 3.3448906094405393, + "tokens_seen": 2732851200 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017484848484848484, + "loss": 2.8488, + "theoretical_loss": 3.3448843246973228, + "tokens_seen": 2732916736 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017482828282828285, + "loss": 2.4549, + "theoretical_loss": 3.3448780401470115, + "tokens_seen": 2732982272 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001748080808080808, + "loss": 2.417, + "theoretical_loss": 3.344871755789595, + "tokens_seen": 2733047808 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1538743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0219013690948486, + "objective/train/theoretical_loss": 3.344865471625063, + "objective/train/tokens_used": 1091972576, + "theoretical_loss": 3.344865471625063, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001747878787878788, + "loss": 2.4718, + "theoretical_loss": 3.344865471625063, + "tokens_seen": 2733113344 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017476767676767677, + "loss": 2.689, + "theoretical_loss": 3.3448591876534044, + "tokens_seen": 2733178880 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017474747474747473, + "loss": 2.6701, + "theoretical_loss": 3.3448529038746093, + "tokens_seen": 2733244416 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017472727272727274, + "loss": 2.6146, + "theoretical_loss": 3.3448466202886666, + "tokens_seen": 2733309952 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001747070707070707, + "loss": 2.4505, + "theoretical_loss": 3.344840336895566, + "tokens_seen": 2733375488 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746868686868687, + "loss": 2.4545, + "theoretical_loss": 3.3448340536952976, + "tokens_seen": 2733441024 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017466666666666667, + "loss": 2.5015, + "theoretical_loss": 3.3448277706878495, + "tokens_seen": 2733506560 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017464646464646465, + "loss": 2.8671, + "theoretical_loss": 3.3448214878732125, + "tokens_seen": 2733572096 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017462626262626263, + "loss": 2.628, + "theoretical_loss": 3.344815205251375, + "tokens_seen": 2733637632 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001746060606060606, + "loss": 2.4954, + "theoretical_loss": 3.3448089228223274, + "tokens_seen": 2733703168 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745858585858586, + "loss": 2.675, + "theoretical_loss": 3.344802640586058, + "tokens_seen": 2733768704 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017456565656565656, + "loss": 2.6325, + "theoretical_loss": 3.3447963585425575, + "tokens_seen": 2733834240 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017454545454545457, + "loss": 2.476, + "theoretical_loss": 3.344790076691815, + "tokens_seen": 2733899776 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017452525252525253, + "loss": 2.3113, + "theoretical_loss": 3.3447837950338193, + "tokens_seen": 2733965312 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001745050505050505, + "loss": 2.5244, + "theoretical_loss": 3.3447775135685607, + "tokens_seen": 2734030848 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744848484848485, + "loss": 2.3154, + "theoretical_loss": 3.344771232296028, + "tokens_seen": 2734096384 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017446464646464648, + "loss": 2.7361, + "theoretical_loss": 3.3447649512162108, + "tokens_seen": 2734161920 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017444444444444446, + "loss": 2.5634, + "theoretical_loss": 3.3447586703290995, + "tokens_seen": 2734227456 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017442424242424242, + "loss": 2.476, + "theoretical_loss": 3.344752389634682, + "tokens_seen": 2734292992 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001744040404040404, + "loss": 2.5918, + "theoretical_loss": 3.344746109132949, + "tokens_seen": 2734358528 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001743838383838384, + "loss": 2.6297, + "theoretical_loss": 3.3447398288238896, + "tokens_seen": 2734424064 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017436363636363637, + "loss": 2.6094, + "theoretical_loss": 3.344733548707493, + "tokens_seen": 2734489600 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017434343434343435, + "loss": 2.4947, + "theoretical_loss": 3.3447272687837493, + "tokens_seen": 2734555136 + }, + { + "epoch": 0.65, + "learning_rate": 0.00017432323232323234, + "loss": 2.6949, + "theoretical_loss": 3.344720989052647, + "tokens_seen": 2734620672 + }, + { + "epoch": 0.65, + "learning_rate": 0.0001743030303030303, + "loss": 2.4573, + "theoretical_loss": 3.344714709514177, + "tokens_seen": 2734686208 + }, + { + "epoch": 0.65, + "objective/train/docs_used": 1539437, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2974483966827393, + "objective/train/theoretical_loss": 3.344708430168327, + "objective/train/tokens_used": 1093610976, + "theoretical_loss": 3.344708430168327, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017428282828282828, + "loss": 2.403, + "theoretical_loss": 3.344708430168327, + "tokens_seen": 2734751744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017426262626262626, + "loss": 2.5606, + "theoretical_loss": 3.344702151015088, + "tokens_seen": 2734817280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017424242424242425, + "loss": 2.7045, + "theoretical_loss": 3.3446958720544484, + "tokens_seen": 2734882816 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017422222222222223, + "loss": 2.5537, + "theoretical_loss": 3.3446895932863985, + "tokens_seen": 2734948352 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001742020202020202, + "loss": 2.3335, + "theoretical_loss": 3.3446833147109274, + "tokens_seen": 2735013888 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001741818181818182, + "loss": 2.5346, + "theoretical_loss": 3.3446770363280245, + "tokens_seen": 2735079424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017416161616161616, + "loss": 2.505, + "theoretical_loss": 3.3446707581376796, + "tokens_seen": 2735144960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017414141414141417, + "loss": 2.6215, + "theoretical_loss": 3.344664480139882, + "tokens_seen": 2735210496 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017412121212121212, + "loss": 2.4224, + "theoretical_loss": 3.344658202334621, + "tokens_seen": 2735276032 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017410101010101008, + "loss": 2.8152, + "theoretical_loss": 3.344651924721886, + "tokens_seen": 2735341568 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001740808080808081, + "loss": 2.4886, + "theoretical_loss": 3.3446456473016672, + "tokens_seen": 2735407104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017406060606060605, + "loss": 2.6231, + "theoretical_loss": 3.344639370073953, + "tokens_seen": 2735472640 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017404040404040406, + "loss": 2.4923, + "theoretical_loss": 3.344633093038734, + "tokens_seen": 2735538176 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017402020202020202, + "loss": 2.7118, + "theoretical_loss": 3.344626816195999, + "tokens_seen": 2735603712 + }, + { + "epoch": 0.66, + "learning_rate": 0.000174, + "loss": 2.5832, + "theoretical_loss": 3.344620539545738, + "tokens_seen": 2735669248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017397979797979798, + "loss": 2.7314, + "theoretical_loss": 3.3446142630879394, + "tokens_seen": 2735734784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017395959595959597, + "loss": 2.6804, + "theoretical_loss": 3.344607986822594, + "tokens_seen": 2735800320 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017393939393939395, + "loss": 2.3437, + "theoretical_loss": 3.3446017107496906, + "tokens_seen": 2735865856 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001739191919191919, + "loss": 2.6403, + "theoretical_loss": 3.344595434869219, + "tokens_seen": 2735931392 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001738989898989899, + "loss": 2.6607, + "theoretical_loss": 3.3445891591811683, + "tokens_seen": 2735996928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017387878787878788, + "loss": 2.5726, + "theoretical_loss": 3.3445828836855283, + "tokens_seen": 2736062464 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017385858585858586, + "loss": 2.5743, + "theoretical_loss": 3.344576608382288, + "tokens_seen": 2736128000 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017383838383838385, + "loss": 2.3649, + "theoretical_loss": 3.3445703332714376, + "tokens_seen": 2736193536 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017381818181818183, + "loss": 2.5998, + "theoretical_loss": 3.344564058352966, + "tokens_seen": 2736259072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017379797979797979, + "loss": 2.4275, + "theoretical_loss": 3.3445577836268634, + "tokens_seen": 2736324608 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1540841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.446636199951172, + "objective/train/theoretical_loss": 3.344551509093119, + "objective/train/tokens_used": 1095249376, + "theoretical_loss": 3.344551509093119, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001737777777777778, + "loss": 2.4962, + "theoretical_loss": 3.344551509093119, + "tokens_seen": 2736390144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017375757575757575, + "loss": 2.6924, + "theoretical_loss": 3.3445452347517217, + "tokens_seen": 2736455680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017373737373737374, + "loss": 2.5887, + "theoretical_loss": 3.3445389606026614, + "tokens_seen": 2736521216 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017371717171717172, + "loss": 2.5511, + "theoretical_loss": 3.3445326866459277, + "tokens_seen": 2736586752 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736969696969697, + "loss": 2.5297, + "theoretical_loss": 3.3445264128815104, + "tokens_seen": 2736652288 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001736767676767677, + "loss": 2.7693, + "theoretical_loss": 3.3445201393093984, + "tokens_seen": 2736717824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017365656565656565, + "loss": 2.6839, + "theoretical_loss": 3.344513865929582, + "tokens_seen": 2736783360 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017363636363636366, + "loss": 2.681, + "theoretical_loss": 3.3445075927420493, + "tokens_seen": 2736848896 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017361616161616161, + "loss": 2.4679, + "theoretical_loss": 3.344501319746791, + "tokens_seen": 2736914432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017359595959595963, + "loss": 2.4919, + "theoretical_loss": 3.3444950469437966, + "tokens_seen": 2736979968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017357575757575758, + "loss": 2.577, + "theoretical_loss": 3.3444887743330547, + "tokens_seen": 2737045504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017355555555555554, + "loss": 2.2513, + "theoretical_loss": 3.344482501914556, + "tokens_seen": 2737111040 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017353535353535355, + "loss": 2.5726, + "theoretical_loss": 3.344476229688289, + "tokens_seen": 2737176576 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001735151515151515, + "loss": 2.6941, + "theoretical_loss": 3.3444699576542436, + "tokens_seen": 2737242112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017349494949494952, + "loss": 2.7834, + "theoretical_loss": 3.3444636858124093, + "tokens_seen": 2737307648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017347474747474748, + "loss": 2.6916, + "theoretical_loss": 3.3444574141627754, + "tokens_seen": 2737373184 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017345454545454546, + "loss": 2.5725, + "theoretical_loss": 3.344451142705332, + "tokens_seen": 2737438720 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017343434343434344, + "loss": 2.6174, + "theoretical_loss": 3.344444871440068, + "tokens_seen": 2737504256 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001734141414141414, + "loss": 2.4087, + "theoretical_loss": 3.344438600366973, + "tokens_seen": 2737569792 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001733939393939394, + "loss": 2.4183, + "theoretical_loss": 3.3444323294860365, + "tokens_seen": 2737635328 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017337373737373737, + "loss": 2.3662, + "theoretical_loss": 3.3444260587972487, + "tokens_seen": 2737700864 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017335353535353535, + "loss": 2.2618, + "theoretical_loss": 3.3444197883005984, + "tokens_seen": 2737766400 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017333333333333334, + "loss": 2.5618, + "theoretical_loss": 3.344413517996075, + "tokens_seen": 2737831936 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017331313131313132, + "loss": 2.6099, + "theoretical_loss": 3.3444072478836686, + "tokens_seen": 2737897472 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732929292929293, + "loss": 2.6191, + "theoretical_loss": 3.3444009779633683, + "tokens_seen": 2737963008 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1541498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1691482067108154, + "objective/train/theoretical_loss": 3.3443947082351633, + "objective/train/tokens_used": 1096887776, + "theoretical_loss": 3.3443947082351633, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732727272727273, + "loss": 2.6336, + "theoretical_loss": 3.3443947082351633, + "tokens_seen": 2738028544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017325252525252524, + "loss": 2.4114, + "theoretical_loss": 3.344388438699044, + "tokens_seen": 2738094080 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017323232323232323, + "loss": 2.4465, + "theoretical_loss": 3.3443821693549993, + "tokens_seen": 2738159616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001732121212121212, + "loss": 2.5813, + "theoretical_loss": 3.344375900203019, + "tokens_seen": 2738225152 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731919191919192, + "loss": 2.5687, + "theoretical_loss": 3.3443696312430924, + "tokens_seen": 2738290688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017317171717171718, + "loss": 2.3104, + "theoretical_loss": 3.3443633624752094, + "tokens_seen": 2738356224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017315151515151514, + "loss": 2.5115, + "theoretical_loss": 3.344357093899359, + "tokens_seen": 2738421760 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017313131313131315, + "loss": 2.71, + "theoretical_loss": 3.344350825515531, + "tokens_seen": 2738487296 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001731111111111111, + "loss": 2.5731, + "theoretical_loss": 3.3443445573237147, + "tokens_seen": 2738552832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017309090909090912, + "loss": 2.5981, + "theoretical_loss": 3.3443382893238995, + "tokens_seen": 2738618368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017307070707070707, + "loss": 2.6356, + "theoretical_loss": 3.3443320215160757, + "tokens_seen": 2738683904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017305050505050503, + "loss": 2.6777, + "theoretical_loss": 3.3443257539002325, + "tokens_seen": 2738749440 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017303030303030304, + "loss": 2.5313, + "theoretical_loss": 3.3443194864763592, + "tokens_seen": 2738814976 + }, + { + "epoch": 0.66, + "learning_rate": 0.000173010101010101, + "loss": 2.7116, + "theoretical_loss": 3.3443132192444454, + "tokens_seen": 2738880512 + }, + { + "epoch": 0.66, + "learning_rate": 0.000172989898989899, + "loss": 2.5997, + "theoretical_loss": 3.34430695220448, + "tokens_seen": 2738946048 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017296969696969697, + "loss": 2.4214, + "theoretical_loss": 3.344300685356454, + "tokens_seen": 2739011584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017294949494949495, + "loss": 2.3985, + "theoretical_loss": 3.344294418700356, + "tokens_seen": 2739077120 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017292929292929293, + "loss": 2.3348, + "theoretical_loss": 3.3442881522361754, + "tokens_seen": 2739142656 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017290909090909092, + "loss": 2.575, + "theoretical_loss": 3.344281885963902, + "tokens_seen": 2739208192 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728888888888889, + "loss": 2.8112, + "theoretical_loss": 3.344275619883525, + "tokens_seen": 2739273728 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017286868686868686, + "loss": 2.5746, + "theoretical_loss": 3.3442693539950343, + "tokens_seen": 2739339264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017284848484848484, + "loss": 2.723, + "theoretical_loss": 3.34426308829842, + "tokens_seen": 2739404800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017282828282828283, + "loss": 2.7748, + "theoretical_loss": 3.34425682279367, + "tokens_seen": 2739470336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001728080808080808, + "loss": 2.5612, + "theoretical_loss": 3.3442505574807755, + "tokens_seen": 2739535872 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727878787878788, + "loss": 2.4352, + "theoretical_loss": 3.344244292359725, + "tokens_seen": 2739601408 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1542857, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8372130393981934, + "objective/train/theoretical_loss": 3.3442380274305084, + "objective/train/tokens_used": 1098526176, + "theoretical_loss": 3.3442380274305084, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017276767676767678, + "loss": 2.5493, + "theoretical_loss": 3.3442380274305084, + "tokens_seen": 2739666944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017274747474747476, + "loss": 2.5244, + "theoretical_loss": 3.3442317626931155, + "tokens_seen": 2739732480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017272727272727272, + "loss": 2.3526, + "theoretical_loss": 3.3442254981475354, + "tokens_seen": 2739798016 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001727070707070707, + "loss": 2.5991, + "theoretical_loss": 3.344219233793758, + "tokens_seen": 2739863552 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001726868686868687, + "loss": 2.5107, + "theoretical_loss": 3.344212969631773, + "tokens_seen": 2739929088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017266666666666667, + "loss": 2.6682, + "theoretical_loss": 3.3442067056615685, + "tokens_seen": 2739994624 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017264646464646466, + "loss": 2.5141, + "theoretical_loss": 3.3442004418831357, + "tokens_seen": 2740060160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017262626262626264, + "loss": 2.3933, + "theoretical_loss": 3.3441941782964637, + "tokens_seen": 2740125696 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001726060606060606, + "loss": 2.659, + "theoretical_loss": 3.344187914901542, + "tokens_seen": 2740191232 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001725858585858586, + "loss": 2.4946, + "theoretical_loss": 3.3441816516983596, + "tokens_seen": 2740256768 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017256565656565656, + "loss": 2.6108, + "theoretical_loss": 3.3441753886869066, + "tokens_seen": 2740322304 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017254545454545455, + "loss": 2.5407, + "theoretical_loss": 3.3441691258671726, + "tokens_seen": 2740387840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017252525252525253, + "loss": 2.3277, + "theoretical_loss": 3.3441628632391476, + "tokens_seen": 2740453376 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001725050505050505, + "loss": 2.4708, + "theoretical_loss": 3.34415660080282, + "tokens_seen": 2740518912 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724848484848485, + "loss": 2.3813, + "theoretical_loss": 3.3441503385581797, + "tokens_seen": 2740584448 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017246464646464646, + "loss": 2.3866, + "theoretical_loss": 3.3441440765052164, + "tokens_seen": 2740649984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017244444444444447, + "loss": 2.6725, + "theoretical_loss": 3.34413781464392, + "tokens_seen": 2740715520 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017242424242424242, + "loss": 2.6437, + "theoretical_loss": 3.3441315529742797, + "tokens_seen": 2740781056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001724040404040404, + "loss": 2.2554, + "theoretical_loss": 3.344125291496285, + "tokens_seen": 2740846592 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723838383838384, + "loss": 2.4826, + "theoretical_loss": 3.3441190302099253, + "tokens_seen": 2740912128 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017236363636363635, + "loss": 2.7015, + "theoretical_loss": 3.3441127691151906, + "tokens_seen": 2740977664 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017234343434343436, + "loss": 2.358, + "theoretical_loss": 3.3441065082120707, + "tokens_seen": 2741043200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017232323232323232, + "loss": 2.5414, + "theoretical_loss": 3.344100247500554, + "tokens_seen": 2741108736 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001723030303030303, + "loss": 2.4944, + "theoretical_loss": 3.344093986980631, + "tokens_seen": 2741174272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017228282828282829, + "loss": 2.5633, + "theoretical_loss": 3.3440877266522913, + "tokens_seen": 2741239808 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1543628, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.442605972290039, + "objective/train/theoretical_loss": 3.344081466515524, + "objective/train/tokens_used": 1100164576, + "theoretical_loss": 3.344081466515524, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017226262626262627, + "loss": 2.3271, + "theoretical_loss": 3.344081466515524, + "tokens_seen": 2741305344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017224242424242425, + "loss": 2.6513, + "theoretical_loss": 3.3440752065703183, + "tokens_seen": 2741370880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017222222222222224, + "loss": 2.4644, + "theoretical_loss": 3.344068946816665, + "tokens_seen": 2741436416 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001722020202020202, + "loss": 2.5913, + "theoretical_loss": 3.344062687254553, + "tokens_seen": 2741501952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017218181818181818, + "loss": 2.6552, + "theoretical_loss": 3.344056427883971, + "tokens_seen": 2741567488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017216161616161616, + "loss": 2.4149, + "theoretical_loss": 3.3440501687049102, + "tokens_seen": 2741633024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017214141414141415, + "loss": 2.6789, + "theoretical_loss": 3.3440439097173593, + "tokens_seen": 2741698560 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017212121212121213, + "loss": 2.618, + "theoretical_loss": 3.3440376509213072, + "tokens_seen": 2741764096 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001721010101010101, + "loss": 2.4522, + "theoretical_loss": 3.344031392316745, + "tokens_seen": 2741829632 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001720808080808081, + "loss": 2.7028, + "theoretical_loss": 3.344025133903661, + "tokens_seen": 2741895168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017206060606060605, + "loss": 2.4483, + "theoretical_loss": 3.3440188756820453, + "tokens_seen": 2741960704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017204040404040407, + "loss": 2.7716, + "theoretical_loss": 3.344012617651887, + "tokens_seen": 2742026240 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017202020202020202, + "loss": 2.7219, + "theoretical_loss": 3.3440063598131764, + "tokens_seen": 2742091776 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017199999999999998, + "loss": 2.5909, + "theoretical_loss": 3.3440001021659027, + "tokens_seen": 2742157312 + }, + { + "epoch": 0.66, + "learning_rate": 0.000171979797979798, + "loss": 2.5711, + "theoretical_loss": 3.3439938447100555, + "tokens_seen": 2742222848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017195959595959595, + "loss": 2.5608, + "theoretical_loss": 3.343987587445624, + "tokens_seen": 2742288384 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017193939393939396, + "loss": 2.6362, + "theoretical_loss": 3.3439813303725985, + "tokens_seen": 2742353920 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017191919191919192, + "loss": 2.5107, + "theoretical_loss": 3.343975073490968, + "tokens_seen": 2742419456 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017189898989898993, + "loss": 2.8134, + "theoretical_loss": 3.343968816800722, + "tokens_seen": 2742484992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017187878787878788, + "loss": 2.5848, + "theoretical_loss": 3.3439625603018506, + "tokens_seen": 2742550528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017185858585858584, + "loss": 2.6618, + "theoretical_loss": 3.343956303994343, + "tokens_seen": 2742616064 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017183838383838385, + "loss": 2.3614, + "theoretical_loss": 3.343950047878189, + "tokens_seen": 2742681600 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001718181818181818, + "loss": 2.6823, + "theoretical_loss": 3.343943791953378, + "tokens_seen": 2742747136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017179797979797982, + "loss": 2.5709, + "theoretical_loss": 3.3439375362198995, + "tokens_seen": 2742812672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017177777777777778, + "loss": 2.6071, + "theoretical_loss": 3.3439312806777433, + "tokens_seen": 2742878208 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1544577, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6363542079925537, + "objective/train/theoretical_loss": 3.343925025326899, + "objective/train/tokens_used": 1101802976, + "theoretical_loss": 3.343925025326899, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017175757575757576, + "loss": 2.5705, + "theoretical_loss": 3.343925025326899, + "tokens_seen": 2742943744 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017173737373737374, + "loss": 2.5271, + "theoretical_loss": 3.3439187701673556, + "tokens_seen": 2743009280 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017171717171717173, + "loss": 2.6206, + "theoretical_loss": 3.3439125151991034, + "tokens_seen": 2743074816 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001716969696969697, + "loss": 2.643, + "theoretical_loss": 3.3439062604221315, + "tokens_seen": 2743140352 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017167676767676767, + "loss": 2.5945, + "theoretical_loss": 3.34390000583643, + "tokens_seen": 2743205888 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017165656565656565, + "loss": 2.4198, + "theoretical_loss": 3.343893751441988, + "tokens_seen": 2743271424 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017163636363636364, + "loss": 2.5555, + "theoretical_loss": 3.3438874972387955, + "tokens_seen": 2743336960 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017161616161616162, + "loss": 2.5001, + "theoretical_loss": 3.3438812432268414, + "tokens_seen": 2743402496 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715959595959596, + "loss": 2.4174, + "theoretical_loss": 3.343874989406116, + "tokens_seen": 2743468032 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715757575757576, + "loss": 2.653, + "theoretical_loss": 3.3438687357766086, + "tokens_seen": 2743533568 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017155555555555555, + "loss": 2.6331, + "theoretical_loss": 3.343862482338309, + "tokens_seen": 2743599104 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017153535353535356, + "loss": 2.5949, + "theoretical_loss": 3.343856229091206, + "tokens_seen": 2743664640 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001715151515151515, + "loss": 2.6122, + "theoretical_loss": 3.3438499760352904, + "tokens_seen": 2743730176 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714949494949495, + "loss": 2.3793, + "theoretical_loss": 3.3438437231705507, + "tokens_seen": 2743795712 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017147474747474748, + "loss": 2.5157, + "theoretical_loss": 3.343837470496977, + "tokens_seen": 2743861248 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017145454545454544, + "loss": 2.5092, + "theoretical_loss": 3.343831218014559, + "tokens_seen": 2743926784 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017143434343434345, + "loss": 2.7181, + "theoretical_loss": 3.343824965723286, + "tokens_seen": 2743992320 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001714141414141414, + "loss": 2.5358, + "theoretical_loss": 3.343818713623148, + "tokens_seen": 2744057856 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017139393939393942, + "loss": 2.5996, + "theoretical_loss": 3.343812461714134, + "tokens_seen": 2744123392 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017137373737373737, + "loss": 2.4109, + "theoretical_loss": 3.343806209996234, + "tokens_seen": 2744188928 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017135353535353536, + "loss": 2.6346, + "theoretical_loss": 3.3437999584694373, + "tokens_seen": 2744254464 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017133333333333334, + "loss": 2.3032, + "theoretical_loss": 3.343793707133734, + "tokens_seen": 2744320000 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001713131313131313, + "loss": 2.6237, + "theoretical_loss": 3.343787455989113, + "tokens_seen": 2744385536 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001712929292929293, + "loss": 2.5861, + "theoretical_loss": 3.343781205035565, + "tokens_seen": 2744451072 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017127272727272727, + "loss": 2.6465, + "theoretical_loss": 3.343774954273078, + "tokens_seen": 2744516608 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1545179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6457576751708984, + "objective/train/theoretical_loss": 3.343768703701643, + "objective/train/tokens_used": 1103441376, + "theoretical_loss": 3.343768703701643, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017125252525252525, + "loss": 2.6907, + "theoretical_loss": 3.343768703701643, + "tokens_seen": 2744582144 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017123232323232323, + "loss": 2.4938, + "theoretical_loss": 3.343762453321249, + "tokens_seen": 2744647680 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017121212121212122, + "loss": 2.5833, + "theoretical_loss": 3.3437562031318855, + "tokens_seen": 2744713216 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001711919191919192, + "loss": 2.5914, + "theoretical_loss": 3.3437499531335426, + "tokens_seen": 2744778752 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017117171717171716, + "loss": 2.66, + "theoretical_loss": 3.3437437033262096, + "tokens_seen": 2744844288 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017115151515151514, + "loss": 2.5081, + "theoretical_loss": 3.343737453709876, + "tokens_seen": 2744909824 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017113131313131313, + "loss": 2.4071, + "theoretical_loss": 3.3437312042845306, + "tokens_seen": 2744975360 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001711111111111111, + "loss": 2.6452, + "theoretical_loss": 3.343724955050165, + "tokens_seen": 2745040896 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001710909090909091, + "loss": 2.527, + "theoretical_loss": 3.3437187060067672, + "tokens_seen": 2745106432 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017107070707070708, + "loss": 2.8068, + "theoretical_loss": 3.3437124571543273, + "tokens_seen": 2745171968 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017105050505050506, + "loss": 2.3496, + "theoretical_loss": 3.343706208492835, + "tokens_seen": 2745237504 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017103030303030305, + "loss": 2.6195, + "theoretical_loss": 3.34369996002228, + "tokens_seen": 2745303040 + }, + { + "epoch": 0.66, + "learning_rate": 0.000171010101010101, + "loss": 2.7304, + "theoretical_loss": 3.3436937117426515, + "tokens_seen": 2745368576 + }, + { + "epoch": 0.66, + "learning_rate": 0.000170989898989899, + "loss": 2.6754, + "theoretical_loss": 3.3436874636539393, + "tokens_seen": 2745434112 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017096969696969697, + "loss": 2.2623, + "theoretical_loss": 3.343681215756133, + "tokens_seen": 2745499648 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017094949494949496, + "loss": 2.4246, + "theoretical_loss": 3.3436749680492226, + "tokens_seen": 2745565184 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017092929292929294, + "loss": 2.6744, + "theoretical_loss": 3.3436687205331967, + "tokens_seen": 2745630720 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001709090909090909, + "loss": 2.4741, + "theoretical_loss": 3.343662473208046, + "tokens_seen": 2745696256 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708888888888889, + "loss": 2.4654, + "theoretical_loss": 3.34365622607376, + "tokens_seen": 2745761792 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017086868686868686, + "loss": 2.763, + "theoretical_loss": 3.3436499791303276, + "tokens_seen": 2745827328 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017084848484848488, + "loss": 2.5629, + "theoretical_loss": 3.343643732377739, + "tokens_seen": 2745892864 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017082828282828283, + "loss": 2.5816, + "theoretical_loss": 3.343637485815983, + "tokens_seen": 2745958400 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001708080808080808, + "loss": 2.2934, + "theoretical_loss": 3.3436312394450507, + "tokens_seen": 2746023936 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707878787878788, + "loss": 2.601, + "theoretical_loss": 3.34362499326493, + "tokens_seen": 2746089472 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017076767676767676, + "loss": 2.6954, + "theoretical_loss": 3.3436187472756123, + "tokens_seen": 2746155008 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1546393, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6517341136932373, + "objective/train/theoretical_loss": 3.343612501477086, + "objective/train/tokens_used": 1105079776, + "theoretical_loss": 3.343612501477086, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017074747474747477, + "loss": 2.688, + "theoretical_loss": 3.343612501477086, + "tokens_seen": 2746220544 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017072727272727273, + "loss": 2.6115, + "theoretical_loss": 3.3436062558693402, + "tokens_seen": 2746286080 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001707070707070707, + "loss": 2.5278, + "theoretical_loss": 3.3436000104523664, + "tokens_seen": 2746351616 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706868686868687, + "loss": 2.4927, + "theoretical_loss": 3.3435937652261525, + "tokens_seen": 2746417152 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017066666666666668, + "loss": 2.4944, + "theoretical_loss": 3.343587520190689, + "tokens_seen": 2746482688 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017064646464646466, + "loss": 2.6687, + "theoretical_loss": 3.343581275345965, + "tokens_seen": 2746548224 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017062626262626262, + "loss": 2.5884, + "theoretical_loss": 3.343575030691971, + "tokens_seen": 2746613760 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001706060606060606, + "loss": 2.5382, + "theoretical_loss": 3.343568786228696, + "tokens_seen": 2746679296 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017058585858585859, + "loss": 2.49, + "theoretical_loss": 3.3435625419561292, + "tokens_seen": 2746744832 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017056565656565657, + "loss": 2.7904, + "theoretical_loss": 3.343556297874261, + "tokens_seen": 2746810368 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017054545454545455, + "loss": 2.6141, + "theoretical_loss": 3.3435500539830807, + "tokens_seen": 2746875904 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017052525252525254, + "loss": 2.5801, + "theoretical_loss": 3.3435438102825774, + "tokens_seen": 2746941440 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001705050505050505, + "loss": 2.3853, + "theoretical_loss": 3.343537566772742, + "tokens_seen": 2747006976 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704848484848485, + "loss": 2.4187, + "theoretical_loss": 3.343531323453563, + "tokens_seen": 2747072512 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017046464646464646, + "loss": 2.5018, + "theoretical_loss": 3.3435250803250307, + "tokens_seen": 2747138048 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017044444444444445, + "loss": 2.6172, + "theoretical_loss": 3.3435188373871343, + "tokens_seen": 2747203584 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017042424242424243, + "loss": 2.5818, + "theoretical_loss": 3.3435125946398636, + "tokens_seen": 2747269120 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001704040404040404, + "loss": 2.5802, + "theoretical_loss": 3.3435063520832085, + "tokens_seen": 2747334656 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001703838383838384, + "loss": 2.6214, + "theoretical_loss": 3.3435001097171577, + "tokens_seen": 2747400192 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017036363636363636, + "loss": 2.6177, + "theoretical_loss": 3.343493867541702, + "tokens_seen": 2747465728 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017034343434343437, + "loss": 2.5791, + "theoretical_loss": 3.3434876255568304, + "tokens_seen": 2747531264 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017032323232323232, + "loss": 2.5969, + "theoretical_loss": 3.3434813837625326, + "tokens_seen": 2747596800 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017030303030303028, + "loss": 2.6742, + "theoretical_loss": 3.3434751421587987, + "tokens_seen": 2747662336 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001702828282828283, + "loss": 2.6939, + "theoretical_loss": 3.3434689007456173, + "tokens_seen": 2747727872 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017026262626262625, + "loss": 2.5994, + "theoretical_loss": 3.343462659522979, + "tokens_seen": 2747793408 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1547118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.563584327697754, + "objective/train/theoretical_loss": 3.3434564184908733, + "objective/train/tokens_used": 1106718176, + "theoretical_loss": 3.3434564184908733, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017024242424242426, + "loss": 2.6194, + "theoretical_loss": 3.3434564184908733, + "tokens_seen": 2747858944 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017022222222222222, + "loss": 2.5407, + "theoretical_loss": 3.3434501776492898, + "tokens_seen": 2747924480 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017020202020202023, + "loss": 2.8475, + "theoretical_loss": 3.3434439369982174, + "tokens_seen": 2747990016 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017018181818181818, + "loss": 2.8274, + "theoretical_loss": 3.343437696537647, + "tokens_seen": 2748055552 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017016161616161617, + "loss": 2.656, + "theoretical_loss": 3.3434314562675667, + "tokens_seen": 2748121088 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017014141414141415, + "loss": 2.5251, + "theoretical_loss": 3.343425216187968, + "tokens_seen": 2748186624 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001701212121212121, + "loss": 2.7055, + "theoretical_loss": 3.343418976298839, + "tokens_seen": 2748252160 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017010101010101012, + "loss": 2.6013, + "theoretical_loss": 3.3434127366001696, + "tokens_seen": 2748317696 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017008080808080808, + "loss": 2.4588, + "theoretical_loss": 3.34340649709195, + "tokens_seen": 2748383232 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017006060606060606, + "loss": 2.5097, + "theoretical_loss": 3.34340025777417, + "tokens_seen": 2748448768 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017004040404040404, + "loss": 2.362, + "theoretical_loss": 3.3433940186468187, + "tokens_seen": 2748514304 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017002020202020203, + "loss": 2.6605, + "theoretical_loss": 3.343387779709886, + "tokens_seen": 2748579840 + }, + { + "epoch": 0.66, + "learning_rate": 0.00017, + "loss": 2.6389, + "theoretical_loss": 3.3433815409633607, + "tokens_seen": 2748645376 + }, + { + "epoch": 0.66, + "learning_rate": 0.000169979797979798, + "loss": 2.646, + "theoretical_loss": 3.3433753024072335, + "tokens_seen": 2748710912 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016995959595959595, + "loss": 2.4816, + "theoretical_loss": 3.3433690640414944, + "tokens_seen": 2748776448 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016993939393939394, + "loss": 2.7665, + "theoretical_loss": 3.343362825866132, + "tokens_seen": 2748841984 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016991919191919192, + "loss": 2.7835, + "theoretical_loss": 3.3433565878811358, + "tokens_seen": 2748907520 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698989898989899, + "loss": 2.4191, + "theoretical_loss": 3.3433503500864967, + "tokens_seen": 2748973056 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001698787878787879, + "loss": 2.4568, + "theoretical_loss": 3.343344112482203, + "tokens_seen": 2749038592 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016985858585858585, + "loss": 2.7004, + "theoretical_loss": 3.343337875068246, + "tokens_seen": 2749104128 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016983838383838386, + "loss": 2.6543, + "theoretical_loss": 3.3433316378446136, + "tokens_seen": 2749169664 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016981818181818181, + "loss": 2.5438, + "theoretical_loss": 3.3433254008112963, + "tokens_seen": 2749235200 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016979797979797982, + "loss": 2.5156, + "theoretical_loss": 3.3433191639682835, + "tokens_seen": 2749300736 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016977777777777778, + "loss": 2.7934, + "theoretical_loss": 3.3433129273155657, + "tokens_seen": 2749366272 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016975757575757574, + "loss": 2.5427, + "theoretical_loss": 3.3433066908531313, + "tokens_seen": 2749431808 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1548324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.194901704788208, + "objective/train/theoretical_loss": 3.343300454580971, + "objective/train/tokens_used": 1108356576, + "theoretical_loss": 3.343300454580971, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016973737373737375, + "loss": 2.7076, + "theoretical_loss": 3.343300454580971, + "tokens_seen": 2749497344 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001697171717171717, + "loss": 2.704, + "theoretical_loss": 3.3432942184990733, + "tokens_seen": 2749562880 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016969696969696972, + "loss": 2.7272, + "theoretical_loss": 3.343287982607429, + "tokens_seen": 2749628416 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016967676767676767, + "loss": 2.5294, + "theoretical_loss": 3.3432817469060274, + "tokens_seen": 2749693952 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016965656565656566, + "loss": 2.6558, + "theoretical_loss": 3.343275511394858, + "tokens_seen": 2749759488 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016963636363636364, + "loss": 2.5233, + "theoretical_loss": 3.3432692760739102, + "tokens_seen": 2749825024 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016961616161616163, + "loss": 2.6553, + "theoretical_loss": 3.3432630409431745, + "tokens_seen": 2749890560 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001695959595959596, + "loss": 2.4105, + "theoretical_loss": 3.34325680600264, + "tokens_seen": 2749956096 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016957575757575757, + "loss": 2.5316, + "theoretical_loss": 3.3432505712522964, + "tokens_seen": 2750021632 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016955555555555555, + "loss": 2.6015, + "theoretical_loss": 3.343244336692133, + "tokens_seen": 2750087168 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016953535353535353, + "loss": 2.6167, + "theoretical_loss": 3.3432381023221405, + "tokens_seen": 2750152704 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016951515151515152, + "loss": 2.7408, + "theoretical_loss": 3.3432318681423077, + "tokens_seen": 2750218240 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694949494949495, + "loss": 2.6551, + "theoretical_loss": 3.3432256341526245, + "tokens_seen": 2750283776 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694747474747475, + "loss": 2.5487, + "theoretical_loss": 3.343219400353081, + "tokens_seen": 2750349312 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016945454545454544, + "loss": 2.5148, + "theoretical_loss": 3.3432131667436655, + "tokens_seen": 2750414848 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016943434343434343, + "loss": 2.6032, + "theoretical_loss": 3.3432069333243692, + "tokens_seen": 2750480384 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001694141414141414, + "loss": 2.6467, + "theoretical_loss": 3.3432007000951813, + "tokens_seen": 2750545920 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693939393939394, + "loss": 2.5873, + "theoretical_loss": 3.343194467056091, + "tokens_seen": 2750611456 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016937373737373738, + "loss": 2.6085, + "theoretical_loss": 3.343188234207089, + "tokens_seen": 2750676992 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016935353535353536, + "loss": 2.8623, + "theoretical_loss": 3.3431820015481635, + "tokens_seen": 2750742528 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016933333333333335, + "loss": 2.5237, + "theoretical_loss": 3.3431757690793056, + "tokens_seen": 2750808064 + }, + { + "epoch": 0.66, + "learning_rate": 0.0001693131313131313, + "loss": 2.5598, + "theoretical_loss": 3.343169536800504, + "tokens_seen": 2750873600 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016929292929292932, + "loss": 2.5737, + "theoretical_loss": 3.343163304711749, + "tokens_seen": 2750939136 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016927272727272727, + "loss": 2.817, + "theoretical_loss": 3.3431570728130295, + "tokens_seen": 2751004672 + }, + { + "epoch": 0.66, + "learning_rate": 0.00016925252525252526, + "loss": 2.5241, + "theoretical_loss": 3.3431508411043365, + "tokens_seen": 2751070208 + }, + { + "epoch": 0.66, + "objective/train/docs_used": 1548689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.55135440826416, + "objective/train/theoretical_loss": 3.3431446095856585, + "objective/train/tokens_used": 1109994976, + "theoretical_loss": 3.3431446095856585, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016923232323232324, + "loss": 2.7022, + "theoretical_loss": 3.3431446095856585, + "tokens_seen": 2751135744 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001692121212121212, + "loss": 2.4466, + "theoretical_loss": 3.3431383782569855, + "tokens_seen": 2751201280 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001691919191919192, + "loss": 2.606, + "theoretical_loss": 3.343132147118307, + "tokens_seen": 2751266816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016917171717171716, + "loss": 2.654, + "theoretical_loss": 3.343125916169613, + "tokens_seen": 2751332352 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016915151515151518, + "loss": 2.5773, + "theoretical_loss": 3.3431196854108935, + "tokens_seen": 2751397888 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016913131313131313, + "loss": 2.6523, + "theoretical_loss": 3.3431134548421375, + "tokens_seen": 2751463424 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016911111111111112, + "loss": 2.4061, + "theoretical_loss": 3.3431072244633353, + "tokens_seen": 2751528960 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001690909090909091, + "loss": 2.579, + "theoretical_loss": 3.343100994274476, + "tokens_seen": 2751594496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016907070707070706, + "loss": 2.576, + "theoretical_loss": 3.3430947642755493, + "tokens_seen": 2751660032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016905050505050507, + "loss": 2.4854, + "theoretical_loss": 3.3430885344665455, + "tokens_seen": 2751725568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016903030303030303, + "loss": 2.5982, + "theoretical_loss": 3.343082304847454, + "tokens_seen": 2751791104 + }, + { + "epoch": 0.67, + "learning_rate": 0.000169010101010101, + "loss": 2.4428, + "theoretical_loss": 3.343076075418264, + "tokens_seen": 2751856640 + }, + { + "epoch": 0.67, + "learning_rate": 0.000168989898989899, + "loss": 2.5684, + "theoretical_loss": 3.343069846178966, + "tokens_seen": 2751922176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016896969696969698, + "loss": 2.6139, + "theoretical_loss": 3.3430636171295487, + "tokens_seen": 2751987712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016894949494949496, + "loss": 2.638, + "theoretical_loss": 3.3430573882700028, + "tokens_seen": 2752053248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016892929292929295, + "loss": 2.7994, + "theoretical_loss": 3.3430511596003174, + "tokens_seen": 2752118784 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001689090909090909, + "loss": 2.4894, + "theoretical_loss": 3.3430449311204824, + "tokens_seen": 2752184320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016888888888888889, + "loss": 2.5981, + "theoretical_loss": 3.3430387028304875, + "tokens_seen": 2752249856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016886868686868687, + "loss": 2.7691, + "theoretical_loss": 3.343032474730322, + "tokens_seen": 2752315392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016884848484848485, + "loss": 2.4223, + "theoretical_loss": 3.3430262468199765, + "tokens_seen": 2752380928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016882828282828284, + "loss": 2.7057, + "theoretical_loss": 3.34302001909944, + "tokens_seen": 2752446464 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001688080808080808, + "loss": 2.7495, + "theoretical_loss": 3.343013791568702, + "tokens_seen": 2752512000 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687878787878788, + "loss": 2.5417, + "theoretical_loss": 3.3430075642277526, + "tokens_seen": 2752577536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016876767676767676, + "loss": 2.6177, + "theoretical_loss": 3.343001337076582, + "tokens_seen": 2752643072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016874747474747477, + "loss": 2.4876, + "theoretical_loss": 3.3429951101151785, + "tokens_seen": 2752708608 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1549978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.778193473815918, + "objective/train/theoretical_loss": 3.342988883343533, + "objective/train/tokens_used": 1111633376, + "theoretical_loss": 3.342988883343533, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016872727272727273, + "loss": 2.8399, + "theoretical_loss": 3.342988883343533, + "tokens_seen": 2752774144 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001687070707070707, + "loss": 2.4926, + "theoretical_loss": 3.342982656761635, + "tokens_seen": 2752839680 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686868686868687, + "loss": 2.6128, + "theoretical_loss": 3.3429764303694736, + "tokens_seen": 2752905216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016866666666666666, + "loss": 2.8041, + "theoretical_loss": 3.342970204167039, + "tokens_seen": 2752970752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016864646464646467, + "loss": 2.7175, + "theoretical_loss": 3.3429639781543212, + "tokens_seen": 2753036288 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016862626262626262, + "loss": 2.6865, + "theoretical_loss": 3.342957752331309, + "tokens_seen": 2753101824 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001686060606060606, + "loss": 2.7547, + "theoretical_loss": 3.342951526697993, + "tokens_seen": 2753167360 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001685858585858586, + "loss": 2.5709, + "theoretical_loss": 3.342945301254362, + "tokens_seen": 2753232896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016856565656565655, + "loss": 2.6639, + "theoretical_loss": 3.342939076000407, + "tokens_seen": 2753298432 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016854545454545456, + "loss": 2.3752, + "theoretical_loss": 3.3429328509361165, + "tokens_seen": 2753363968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016852525252525252, + "loss": 2.7768, + "theoretical_loss": 3.342926626061481, + "tokens_seen": 2753429504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016850505050505053, + "loss": 2.4848, + "theoretical_loss": 3.3429204013764897, + "tokens_seen": 2753495040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016848484848484848, + "loss": 2.6151, + "theoretical_loss": 3.3429141768811323, + "tokens_seen": 2753560576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016846464646464647, + "loss": 2.5375, + "theoretical_loss": 3.342907952575399, + "tokens_seen": 2753626112 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016844444444444445, + "loss": 2.7166, + "theoretical_loss": 3.3429017284592786, + "tokens_seen": 2753691648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016842424242424244, + "loss": 2.6876, + "theoretical_loss": 3.3428955045327617, + "tokens_seen": 2753757184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016840404040404042, + "loss": 2.5975, + "theoretical_loss": 3.3428892807958377, + "tokens_seen": 2753822720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016838383838383838, + "loss": 2.631, + "theoretical_loss": 3.3428830572484967, + "tokens_seen": 2753888256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016836363636363636, + "loss": 2.355, + "theoretical_loss": 3.3428768338907275, + "tokens_seen": 2753953792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016834343434343434, + "loss": 2.4688, + "theoretical_loss": 3.342870610722521, + "tokens_seen": 2754019328 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016832323232323233, + "loss": 2.7418, + "theoretical_loss": 3.3428643877438655, + "tokens_seen": 2754084864 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001683030303030303, + "loss": 2.4188, + "theoretical_loss": 3.3428581649547517, + "tokens_seen": 2754150400 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682828282828283, + "loss": 2.8152, + "theoretical_loss": 3.3428519423551695, + "tokens_seen": 2754215936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016826262626262625, + "loss": 2.657, + "theoretical_loss": 3.342845719945108, + "tokens_seen": 2754281472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016824242424242426, + "loss": 2.6908, + "theoretical_loss": 3.3428394977245572, + "tokens_seen": 2754347008 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1550558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9385275840759277, + "objective/train/theoretical_loss": 3.3428332756935064, + "objective/train/tokens_used": 1113271776, + "theoretical_loss": 3.3428332756935064, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016822222222222222, + "loss": 2.6923, + "theoretical_loss": 3.3428332756935064, + "tokens_seen": 2754412544 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001682020202020202, + "loss": 2.3218, + "theoretical_loss": 3.342827053851946, + "tokens_seen": 2754478080 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001681818181818182, + "loss": 2.7447, + "theoretical_loss": 3.3428208321998656, + "tokens_seen": 2754543616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016816161616161615, + "loss": 2.6976, + "theoretical_loss": 3.3428146107372543, + "tokens_seen": 2754609152 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016814141414141416, + "loss": 2.5072, + "theoretical_loss": 3.3428083894641025, + "tokens_seen": 2754674688 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016812121212121211, + "loss": 2.9028, + "theoretical_loss": 3.3428021683803992, + "tokens_seen": 2754740224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016810101010101013, + "loss": 2.523, + "theoretical_loss": 3.342795947486135, + "tokens_seen": 2754805760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016808080808080808, + "loss": 2.3817, + "theoretical_loss": 3.3427897267812994, + "tokens_seen": 2754871296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016806060606060607, + "loss": 2.6841, + "theoretical_loss": 3.3427835062658815, + "tokens_seen": 2754936832 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016804040404040405, + "loss": 2.4801, + "theoretical_loss": 3.3427772859398717, + "tokens_seen": 2755002368 + }, + { + "epoch": 0.67, + "learning_rate": 0.000168020202020202, + "loss": 2.8243, + "theoretical_loss": 3.3427710658032597, + "tokens_seen": 2755067904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016800000000000002, + "loss": 2.5786, + "theoretical_loss": 3.342764845856035, + "tokens_seen": 2755133440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016797979797979797, + "loss": 2.6414, + "theoretical_loss": 3.342758626098187, + "tokens_seen": 2755198976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016795959595959596, + "loss": 2.6331, + "theoretical_loss": 3.342752406529706, + "tokens_seen": 2755264512 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016793939393939394, + "loss": 2.7982, + "theoretical_loss": 3.3427461871505812, + "tokens_seen": 2755330048 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016791919191919193, + "loss": 2.3956, + "theoretical_loss": 3.342739967960803, + "tokens_seen": 2755395584 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001678989898989899, + "loss": 2.7119, + "theoretical_loss": 3.3427337489603604, + "tokens_seen": 2755461120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016787878787878787, + "loss": 2.6729, + "theoretical_loss": 3.3427275301492436, + "tokens_seen": 2755526656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016785858585858585, + "loss": 2.5353, + "theoretical_loss": 3.342721311527443, + "tokens_seen": 2755592192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016783838383838384, + "loss": 2.3354, + "theoretical_loss": 3.342715093094947, + "tokens_seen": 2755657728 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016781818181818182, + "loss": 2.3855, + "theoretical_loss": 3.3427088748517457, + "tokens_seen": 2755723264 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677979797979798, + "loss": 2.6496, + "theoretical_loss": 3.342702656797829, + "tokens_seen": 2755788800 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677777777777778, + "loss": 2.4684, + "theoretical_loss": 3.3426964389331872, + "tokens_seen": 2755854336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016775757575757574, + "loss": 2.6371, + "theoretical_loss": 3.342690221257809, + "tokens_seen": 2755919872 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016773737373737376, + "loss": 2.5885, + "theoretical_loss": 3.342684003771685, + "tokens_seen": 2755985408 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1551539, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0517077445983887, + "objective/train/theoretical_loss": 3.3426777864748045, + "objective/train/tokens_used": 1114910176, + "theoretical_loss": 3.3426777864748045, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001677171717171717, + "loss": 2.573, + "theoretical_loss": 3.3426777864748045, + "tokens_seen": 2756050944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676969696969697, + "loss": 2.8903, + "theoretical_loss": 3.3426715693671576, + "tokens_seen": 2756116480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016767676767676768, + "loss": 2.5545, + "theoretical_loss": 3.3426653524487335, + "tokens_seen": 2756182016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016765656565656566, + "loss": 2.3754, + "theoretical_loss": 3.342659135719522, + "tokens_seen": 2756247552 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016763636363636365, + "loss": 2.7039, + "theoretical_loss": 3.342652919179513, + "tokens_seen": 2756313088 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001676161616161616, + "loss": 2.4477, + "theoretical_loss": 3.342646702828697, + "tokens_seen": 2756378624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016759595959595962, + "loss": 2.498, + "theoretical_loss": 3.342640486667063, + "tokens_seen": 2756444160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016757575757575757, + "loss": 2.7465, + "theoretical_loss": 3.3426342706946004, + "tokens_seen": 2756509696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016755555555555558, + "loss": 2.5397, + "theoretical_loss": 3.342628054911299, + "tokens_seen": 2756575232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016753535353535354, + "loss": 2.3839, + "theoretical_loss": 3.3426218393171494, + "tokens_seen": 2756640768 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001675151515151515, + "loss": 2.4751, + "theoretical_loss": 3.342615623912141, + "tokens_seen": 2756706304 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001674949494949495, + "loss": 2.4345, + "theoretical_loss": 3.342609408696263, + "tokens_seen": 2756771840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016747474747474747, + "loss": 2.6802, + "theoretical_loss": 3.3426031936695058, + "tokens_seen": 2756837376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016745454545454548, + "loss": 2.6274, + "theoretical_loss": 3.3425969788318586, + "tokens_seen": 2756902912 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016743434343434343, + "loss": 2.3577, + "theoretical_loss": 3.3425907641833117, + "tokens_seen": 2756968448 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016741414141414142, + "loss": 2.5055, + "theoretical_loss": 3.3425845497238544, + "tokens_seen": 2757033984 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001673939393939394, + "loss": 2.7214, + "theoretical_loss": 3.342578335453477, + "tokens_seen": 2757099520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016737373737373739, + "loss": 2.5975, + "theoretical_loss": 3.3425721213721684, + "tokens_seen": 2757165056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016735353535353537, + "loss": 2.5267, + "theoretical_loss": 3.342565907479919, + "tokens_seen": 2757230592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016733333333333333, + "loss": 2.5072, + "theoretical_loss": 3.3425596937767184, + "tokens_seen": 2757296128 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001673131313131313, + "loss": 2.6573, + "theoretical_loss": 3.3425534802625565, + "tokens_seen": 2757361664 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001672929292929293, + "loss": 2.415, + "theoretical_loss": 3.342547266937423, + "tokens_seen": 2757427200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016727272727272728, + "loss": 2.6117, + "theoretical_loss": 3.342541053801307, + "tokens_seen": 2757492736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016725252525252526, + "loss": 2.5033, + "theoretical_loss": 3.3425348408541993, + "tokens_seen": 2757558272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016723232323232325, + "loss": 2.8295, + "theoretical_loss": 3.3425286280960895, + "tokens_seen": 2757623808 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1552106, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.287670850753784, + "objective/train/theoretical_loss": 3.3425224155269664, + "objective/train/tokens_used": 1116548576, + "theoretical_loss": 3.3425224155269664, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001672121212121212, + "loss": 2.526, + "theoretical_loss": 3.3425224155269664, + "tokens_seen": 2757689344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016719191919191921, + "loss": 2.502, + "theoretical_loss": 3.3425162031468205, + "tokens_seen": 2757754880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016717171717171717, + "loss": 2.7052, + "theoretical_loss": 3.3425099909556417, + "tokens_seen": 2757820416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016715151515151515, + "loss": 2.5931, + "theoretical_loss": 3.3425037789534198, + "tokens_seen": 2757885952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016713131313131314, + "loss": 2.3247, + "theoretical_loss": 3.3424975671401436, + "tokens_seen": 2757951488 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001671111111111111, + "loss": 2.3605, + "theoretical_loss": 3.342491355515804, + "tokens_seen": 2758017024 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001670909090909091, + "loss": 2.5172, + "theoretical_loss": 3.34248514408039, + "tokens_seen": 2758082560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016707070707070706, + "loss": 2.7047, + "theoretical_loss": 3.3424789328338917, + "tokens_seen": 2758148096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016705050505050507, + "loss": 2.6241, + "theoretical_loss": 3.342472721776299, + "tokens_seen": 2758213632 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016703030303030303, + "loss": 2.5309, + "theoretical_loss": 3.342466510907602, + "tokens_seen": 2758279168 + }, + { + "epoch": 0.67, + "learning_rate": 0.000167010101010101, + "loss": 2.6563, + "theoretical_loss": 3.3424603002277893, + "tokens_seen": 2758344704 + }, + { + "epoch": 0.67, + "learning_rate": 0.000166989898989899, + "loss": 2.6584, + "theoretical_loss": 3.3424540897368518, + "tokens_seen": 2758410240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016696969696969696, + "loss": 2.6158, + "theoretical_loss": 3.3424478794347783, + "tokens_seen": 2758475776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016694949494949497, + "loss": 2.6866, + "theoretical_loss": 3.3424416693215595, + "tokens_seen": 2758541312 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016692929292929292, + "loss": 2.6237, + "theoretical_loss": 3.3424354593971843, + "tokens_seen": 2758606848 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001669090909090909, + "loss": 2.4689, + "theoretical_loss": 3.3424292496616435, + "tokens_seen": 2758672384 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001668888888888889, + "loss": 2.7814, + "theoretical_loss": 3.342423040114926, + "tokens_seen": 2758737920 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016686868686868688, + "loss": 2.471, + "theoretical_loss": 3.342416830757022, + "tokens_seen": 2758803456 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016684848484848486, + "loss": 2.4489, + "theoretical_loss": 3.342410621587921, + "tokens_seen": 2758868992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016682828282828282, + "loss": 2.468, + "theoretical_loss": 3.342404412607613, + "tokens_seen": 2758934528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016680808080808083, + "loss": 2.7049, + "theoretical_loss": 3.3423982038160878, + "tokens_seen": 2759000064 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016678787878787878, + "loss": 2.5627, + "theoretical_loss": 3.342391995213335, + "tokens_seen": 2759065600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016676767676767677, + "loss": 2.7697, + "theoretical_loss": 3.342385786799344, + "tokens_seen": 2759131136 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016674747474747475, + "loss": 2.4485, + "theoretical_loss": 3.3423795785741057, + "tokens_seen": 2759196672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016672727272727274, + "loss": 2.6684, + "theoretical_loss": 3.342373370537609, + "tokens_seen": 2759262208 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1553410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0841245651245117, + "objective/train/theoretical_loss": 3.3423671626898437, + "objective/train/tokens_used": 1118186976, + "theoretical_loss": 3.3423671626898437, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016670707070707072, + "loss": 2.3973, + "theoretical_loss": 3.3423671626898437, + "tokens_seen": 2759327744 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001666868686868687, + "loss": 2.6412, + "theoretical_loss": 3.3423609550308, + "tokens_seen": 2759393280 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016666666666666666, + "loss": 2.4973, + "theoretical_loss": 3.342354747560467, + "tokens_seen": 2759458816 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016664646464646465, + "loss": 2.7685, + "theoretical_loss": 3.3423485402788353, + "tokens_seen": 2759524352 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016662626262626263, + "loss": 2.7656, + "theoretical_loss": 3.3423423331858944, + "tokens_seen": 2759589888 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001666060606060606, + "loss": 2.6662, + "theoretical_loss": 3.342336126281634, + "tokens_seen": 2759655424 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001665858585858586, + "loss": 2.3578, + "theoretical_loss": 3.342329919566044, + "tokens_seen": 2759720960 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016656565656565655, + "loss": 2.6004, + "theoretical_loss": 3.3423237130391135, + "tokens_seen": 2759786496 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016654545454545456, + "loss": 2.6351, + "theoretical_loss": 3.3423175067008333, + "tokens_seen": 2759852032 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016652525252525252, + "loss": 2.4727, + "theoretical_loss": 3.3423113005511924, + "tokens_seen": 2759917568 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016650505050505053, + "loss": 2.3931, + "theoretical_loss": 3.3423050945901815, + "tokens_seen": 2759983104 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001664848484848485, + "loss": 2.6989, + "theoretical_loss": 3.342298888817789, + "tokens_seen": 2760048640 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016646464646464645, + "loss": 2.5386, + "theoretical_loss": 3.3422926832340063, + "tokens_seen": 2760114176 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016644444444444446, + "loss": 2.549, + "theoretical_loss": 3.342286477838822, + "tokens_seen": 2760179712 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016642424242424241, + "loss": 2.646, + "theoretical_loss": 3.3422802726322263, + "tokens_seen": 2760245248 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016640404040404043, + "loss": 2.5806, + "theoretical_loss": 3.3422740676142086, + "tokens_seen": 2760310784 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016638383838383838, + "loss": 2.6772, + "theoretical_loss": 3.3422678627847597, + "tokens_seen": 2760376320 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016636363636363637, + "loss": 2.7569, + "theoretical_loss": 3.3422616581438684, + "tokens_seen": 2760441856 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016634343434343435, + "loss": 2.5142, + "theoretical_loss": 3.342255453691525, + "tokens_seen": 2760507392 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016632323232323233, + "loss": 2.2636, + "theoretical_loss": 3.342249249427719, + "tokens_seen": 2760572928 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016630303030303032, + "loss": 2.5796, + "theoretical_loss": 3.3422430453524403, + "tokens_seen": 2760638464 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016628282828282828, + "loss": 2.6251, + "theoretical_loss": 3.342236841465679, + "tokens_seen": 2760704000 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016626262626262626, + "loss": 2.6256, + "theoretical_loss": 3.3422306377674245, + "tokens_seen": 2760769536 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016624242424242424, + "loss": 2.6362, + "theoretical_loss": 3.3422244342576666, + "tokens_seen": 2760835072 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016622222222222223, + "loss": 2.5718, + "theoretical_loss": 3.3422182309363953, + "tokens_seen": 2760900608 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1554093, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8625330924987793, + "objective/train/theoretical_loss": 3.3422120278036003, + "objective/train/tokens_used": 1119825376, + "theoretical_loss": 3.3422120278036003, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001662020202020202, + "loss": 2.6792, + "theoretical_loss": 3.3422120278036003, + "tokens_seen": 2760966144 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661818181818182, + "loss": 2.7439, + "theoretical_loss": 3.3422058248592714, + "tokens_seen": 2761031680 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016616161616161615, + "loss": 2.5221, + "theoretical_loss": 3.3421996221033985, + "tokens_seen": 2761097216 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016614141414141414, + "loss": 2.4087, + "theoretical_loss": 3.3421934195359713, + "tokens_seen": 2761162752 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016612121212121212, + "loss": 2.6341, + "theoretical_loss": 3.3421872171569795, + "tokens_seen": 2761228288 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001661010101010101, + "loss": 2.6164, + "theoretical_loss": 3.3421810149664135, + "tokens_seen": 2761293824 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001660808080808081, + "loss": 2.7207, + "theoretical_loss": 3.342174812964262, + "tokens_seen": 2761359360 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016606060606060604, + "loss": 2.7129, + "theoretical_loss": 3.342168611150516, + "tokens_seen": 2761424896 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016604040404040406, + "loss": 2.6534, + "theoretical_loss": 3.3421624095251645, + "tokens_seen": 2761490432 + }, + { + "epoch": 0.67, + "learning_rate": 0.000166020202020202, + "loss": 2.7214, + "theoretical_loss": 3.3421562080881975, + "tokens_seen": 2761555968 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016600000000000002, + "loss": 2.7398, + "theoretical_loss": 3.3421500068396046, + "tokens_seen": 2761621504 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016597979797979798, + "loss": 2.6642, + "theoretical_loss": 3.3421438057793766, + "tokens_seen": 2761687040 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016595959595959594, + "loss": 2.7604, + "theoretical_loss": 3.342137604907502, + "tokens_seen": 2761752576 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016593939393939395, + "loss": 2.7845, + "theoretical_loss": 3.3421314042239714, + "tokens_seen": 2761818112 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001659191919191919, + "loss": 2.6921, + "theoretical_loss": 3.3421252037287745, + "tokens_seen": 2761883648 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016589898989898992, + "loss": 2.4713, + "theoretical_loss": 3.3421190034219004, + "tokens_seen": 2761949184 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016587878787878787, + "loss": 2.6398, + "theoretical_loss": 3.34211280330334, + "tokens_seen": 2762014720 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016585858585858588, + "loss": 2.6183, + "theoretical_loss": 3.342106603373083, + "tokens_seen": 2762080256 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016583838383838384, + "loss": 2.6871, + "theoretical_loss": 3.342100403631118, + "tokens_seen": 2762145792 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016581818181818182, + "loss": 2.4559, + "theoretical_loss": 3.342094204077436, + "tokens_seen": 2762211328 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001657979797979798, + "loss": 2.726, + "theoretical_loss": 3.3420880047120267, + "tokens_seen": 2762276864 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016577777777777777, + "loss": 2.5734, + "theoretical_loss": 3.34208180553488, + "tokens_seen": 2762342400 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016575757575757578, + "loss": 2.5337, + "theoretical_loss": 3.3420756065459845, + "tokens_seen": 2762407936 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016573737373737373, + "loss": 2.5621, + "theoretical_loss": 3.3420694077453317, + "tokens_seen": 2762473472 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016571717171717172, + "loss": 2.3945, + "theoretical_loss": 3.34206320913291, + "tokens_seen": 2762539008 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1555178, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6472907066345215, + "objective/train/theoretical_loss": 3.3420570107087104, + "objective/train/tokens_used": 1121463776, + "theoretical_loss": 3.3420570107087104, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656969696969697, + "loss": 2.6312, + "theoretical_loss": 3.3420570107087104, + "tokens_seen": 2762604544 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016567676767676769, + "loss": 2.6195, + "theoretical_loss": 3.342050812472722, + "tokens_seen": 2762670080 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016565656565656567, + "loss": 2.6163, + "theoretical_loss": 3.3420446144249345, + "tokens_seen": 2762735616 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016563636363636365, + "loss": 2.6117, + "theoretical_loss": 3.3420384165653383, + "tokens_seen": 2762801152 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001656161616161616, + "loss": 2.5675, + "theoretical_loss": 3.342032218893923, + "tokens_seen": 2762866688 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001655959595959596, + "loss": 2.8273, + "theoretical_loss": 3.3420260214106783, + "tokens_seen": 2762932224 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016557575757575758, + "loss": 2.5073, + "theoretical_loss": 3.342019824115594, + "tokens_seen": 2762997760 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016555555555555556, + "loss": 2.6496, + "theoretical_loss": 3.3420136270086602, + "tokens_seen": 2763063296 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016553535353535355, + "loss": 2.6041, + "theoretical_loss": 3.3420074300898666, + "tokens_seen": 2763128832 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001655151515151515, + "loss": 2.6778, + "theoretical_loss": 3.342001233359203, + "tokens_seen": 2763194368 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016549494949494951, + "loss": 2.6473, + "theoretical_loss": 3.341995036816659, + "tokens_seen": 2763259904 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016547474747474747, + "loss": 2.5825, + "theoretical_loss": 3.3419888404622244, + "tokens_seen": 2763325440 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016545454545454548, + "loss": 2.7741, + "theoretical_loss": 3.3419826442958893, + "tokens_seen": 2763390976 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016543434343434344, + "loss": 2.7132, + "theoretical_loss": 3.3419764483176437, + "tokens_seen": 2763456512 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001654141414141414, + "loss": 2.4746, + "theoretical_loss": 3.3419702525274775, + "tokens_seen": 2763522048 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001653939393939394, + "loss": 2.6175, + "theoretical_loss": 3.34196405692538, + "tokens_seen": 2763587584 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016537373737373736, + "loss": 2.6288, + "theoretical_loss": 3.341957861511341, + "tokens_seen": 2763653120 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016535353535353537, + "loss": 2.5238, + "theoretical_loss": 3.341951666285351, + "tokens_seen": 2763718656 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016533333333333333, + "loss": 2.5992, + "theoretical_loss": 3.341945471247399, + "tokens_seen": 2763784192 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016531313131313132, + "loss": 2.512, + "theoretical_loss": 3.3419392763974756, + "tokens_seen": 2763849728 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652929292929293, + "loss": 2.6216, + "theoretical_loss": 3.34193308173557, + "tokens_seen": 2763915264 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016527272727272726, + "loss": 2.6817, + "theoretical_loss": 3.3419268872616725, + "tokens_seen": 2763980800 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016525252525252527, + "loss": 2.8641, + "theoretical_loss": 3.3419206929757728, + "tokens_seen": 2764046336 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016523232323232322, + "loss": 2.4892, + "theoretical_loss": 3.341914498877861, + "tokens_seen": 2764111872 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001652121212121212, + "loss": 2.6773, + "theoretical_loss": 3.341908304967926, + "tokens_seen": 2764177408 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1555836, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.236354351043701, + "objective/train/theoretical_loss": 3.341902111245959, + "objective/train/tokens_used": 1123102176, + "theoretical_loss": 3.341902111245959, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001651919191919192, + "loss": 2.4035, + "theoretical_loss": 3.341902111245959, + "tokens_seen": 2764242944 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016517171717171718, + "loss": 2.6957, + "theoretical_loss": 3.3418959177119487, + "tokens_seen": 2764308480 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016515151515151516, + "loss": 2.5812, + "theoretical_loss": 3.3418897243658856, + "tokens_seen": 2764374016 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016513131313131314, + "loss": 2.3526, + "theoretical_loss": 3.341883531207759, + "tokens_seen": 2764439552 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001651111111111111, + "loss": 2.6721, + "theoretical_loss": 3.341877338237559, + "tokens_seen": 2764505088 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016509090909090908, + "loss": 2.2483, + "theoretical_loss": 3.341871145455276, + "tokens_seen": 2764570624 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016507070707070707, + "loss": 2.5303, + "theoretical_loss": 3.341864952860899, + "tokens_seen": 2764636160 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016505050505050505, + "loss": 2.6798, + "theoretical_loss": 3.3418587604544183, + "tokens_seen": 2764701696 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016503030303030304, + "loss": 2.4754, + "theoretical_loss": 3.3418525682358236, + "tokens_seen": 2764767232 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016501010101010102, + "loss": 2.6561, + "theoretical_loss": 3.3418463762051047, + "tokens_seen": 2764832768 + }, + { + "epoch": 0.67, + "learning_rate": 0.000164989898989899, + "loss": 2.676, + "theoretical_loss": 3.3418401843622516, + "tokens_seen": 2764898304 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016496969696969696, + "loss": 2.7129, + "theoretical_loss": 3.3418339927072545, + "tokens_seen": 2764963840 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016494949494949497, + "loss": 2.5688, + "theoretical_loss": 3.341827801240102, + "tokens_seen": 2765029376 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016492929292929293, + "loss": 2.4984, + "theoretical_loss": 3.3418216099607854, + "tokens_seen": 2765094912 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001649090909090909, + "loss": 2.2653, + "theoretical_loss": 3.3418154188692935, + "tokens_seen": 2765160448 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001648888888888889, + "loss": 2.6176, + "theoretical_loss": 3.3418092279656166, + "tokens_seen": 2765225984 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016486868686868685, + "loss": 2.6795, + "theoretical_loss": 3.3418030372497447, + "tokens_seen": 2765291520 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016484848484848487, + "loss": 2.6185, + "theoretical_loss": 3.3417968467216674, + "tokens_seen": 2765357056 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016482828282828282, + "loss": 2.6592, + "theoretical_loss": 3.3417906563813746, + "tokens_seen": 2765422592 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016480808080808083, + "loss": 2.6505, + "theoretical_loss": 3.341784466228856, + "tokens_seen": 2765488128 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001647878787878788, + "loss": 2.7159, + "theoretical_loss": 3.341778276264102, + "tokens_seen": 2765553664 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016476767676767677, + "loss": 2.6969, + "theoretical_loss": 3.341772086487102, + "tokens_seen": 2765619200 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016474747474747476, + "loss": 2.6427, + "theoretical_loss": 3.341765896897846, + "tokens_seen": 2765684736 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016472727272727271, + "loss": 2.6845, + "theoretical_loss": 3.3417597074963235, + "tokens_seen": 2765750272 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016470707070707073, + "loss": 2.6249, + "theoretical_loss": 3.341753518282525, + "tokens_seen": 2765815808 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1556560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.742440938949585, + "objective/train/theoretical_loss": 3.3417473292564397, + "objective/train/tokens_used": 1124740576, + "theoretical_loss": 3.3417473292564397, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016468686868686868, + "loss": 2.613, + "theoretical_loss": 3.3417473292564397, + "tokens_seen": 2765881344 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016466666666666667, + "loss": 2.5609, + "theoretical_loss": 3.3417411404180584, + "tokens_seen": 2765946880 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016464646464646465, + "loss": 2.7594, + "theoretical_loss": 3.3417349517673696, + "tokens_seen": 2766012416 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016462626262626263, + "loss": 2.5638, + "theoretical_loss": 3.3417287633043644, + "tokens_seen": 2766077952 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016460606060606062, + "loss": 2.6152, + "theoretical_loss": 3.341722575029032, + "tokens_seen": 2766143488 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016458585858585858, + "loss": 2.7704, + "theoretical_loss": 3.341716386941362, + "tokens_seen": 2766209024 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016456565656565656, + "loss": 2.7337, + "theoretical_loss": 3.341710199041345, + "tokens_seen": 2766274560 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016454545454545454, + "loss": 2.5798, + "theoretical_loss": 3.3417040113289707, + "tokens_seen": 2766340096 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016452525252525253, + "loss": 2.6578, + "theoretical_loss": 3.3416978238042283, + "tokens_seen": 2766405632 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001645050505050505, + "loss": 2.7462, + "theoretical_loss": 3.3416916364671088, + "tokens_seen": 2766471168 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644848484848485, + "loss": 2.4032, + "theoretical_loss": 3.341685449317601, + "tokens_seen": 2766536704 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016446464646464645, + "loss": 2.5491, + "theoretical_loss": 3.3416792623556955, + "tokens_seen": 2766602240 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016444444444444446, + "loss": 2.7853, + "theoretical_loss": 3.341673075581382, + "tokens_seen": 2766667776 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016442424242424242, + "loss": 2.6997, + "theoretical_loss": 3.34166688899465, + "tokens_seen": 2766733312 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001644040404040404, + "loss": 2.7913, + "theoretical_loss": 3.3416607025954894, + "tokens_seen": 2766798848 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001643838383838384, + "loss": 2.6881, + "theoretical_loss": 3.341654516383891, + "tokens_seen": 2766864384 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016436363636363634, + "loss": 2.5466, + "theoretical_loss": 3.341648330359843, + "tokens_seen": 2766929920 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016434343434343436, + "loss": 2.5204, + "theoretical_loss": 3.341642144523337, + "tokens_seen": 2766995456 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001643232323232323, + "loss": 2.6031, + "theoretical_loss": 3.341635958874362, + "tokens_seen": 2767060992 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016430303030303032, + "loss": 2.7231, + "theoretical_loss": 3.3416297734129077, + "tokens_seen": 2767126528 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016428282828282828, + "loss": 2.607, + "theoretical_loss": 3.3416235881389644, + "tokens_seen": 2767192064 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016426262626262626, + "loss": 2.5668, + "theoretical_loss": 3.341617403052522, + "tokens_seen": 2767257600 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016424242424242425, + "loss": 2.6513, + "theoretical_loss": 3.3416112181535698, + "tokens_seen": 2767323136 + }, + { + "epoch": 0.67, + "learning_rate": 0.0001642222222222222, + "loss": 2.8428, + "theoretical_loss": 3.3416050334420984, + "tokens_seen": 2767388672 + }, + { + "epoch": 0.67, + "learning_rate": 0.00016420202020202022, + "loss": 2.4963, + "theoretical_loss": 3.341598848918097, + "tokens_seen": 2767454208 + }, + { + "epoch": 0.67, + "objective/train/docs_used": 1557767, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5527548789978027, + "objective/train/theoretical_loss": 3.3415926645815563, + "objective/train/tokens_used": 1126378976, + "theoretical_loss": 3.3415926645815563, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016418181818181817, + "loss": 2.709, + "theoretical_loss": 3.3415926645815563, + "tokens_seen": 2767519744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016416161616161618, + "loss": 2.7094, + "theoretical_loss": 3.3415864804324653, + "tokens_seen": 2767585280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016414141414141414, + "loss": 2.6139, + "theoretical_loss": 3.3415802964708146, + "tokens_seen": 2767650816 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016412121212121213, + "loss": 2.4361, + "theoretical_loss": 3.3415741126965934, + "tokens_seen": 2767716352 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001641010101010101, + "loss": 2.6898, + "theoretical_loss": 3.341567929109792, + "tokens_seen": 2767781888 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001640808080808081, + "loss": 2.3862, + "theoretical_loss": 3.3415617457104005, + "tokens_seen": 2767847424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016406060606060608, + "loss": 2.5943, + "theoretical_loss": 3.341555562498409, + "tokens_seen": 2767912960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016404040404040403, + "loss": 2.5107, + "theoretical_loss": 3.341549379473806, + "tokens_seen": 2767978496 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016402020202020202, + "loss": 2.4942, + "theoretical_loss": 3.3415431966365827, + "tokens_seen": 2768044032 + }, + { + "epoch": 0.68, + "learning_rate": 0.000164, + "loss": 2.4952, + "theoretical_loss": 3.3415370139867284, + "tokens_seen": 2768109568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016397979797979799, + "loss": 2.7408, + "theoretical_loss": 3.341530831524233, + "tokens_seen": 2768175104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016395959595959597, + "loss": 2.4624, + "theoretical_loss": 3.341524649249087, + "tokens_seen": 2768240640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016393939393939395, + "loss": 2.3244, + "theoretical_loss": 3.34151846716128, + "tokens_seen": 2768306176 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001639191919191919, + "loss": 2.4542, + "theoretical_loss": 3.3415122852608015, + "tokens_seen": 2768371712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016389898989898992, + "loss": 2.7439, + "theoretical_loss": 3.3415061035476414, + "tokens_seen": 2768437248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016387878787878788, + "loss": 2.6156, + "theoretical_loss": 3.34149992202179, + "tokens_seen": 2768502784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016385858585858586, + "loss": 2.3152, + "theoretical_loss": 3.3414937406832363, + "tokens_seen": 2768568320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016383838383838385, + "loss": 2.6331, + "theoretical_loss": 3.3414875595319717, + "tokens_seen": 2768633856 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001638181818181818, + "loss": 2.6515, + "theoretical_loss": 3.341481378567985, + "tokens_seen": 2768699392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016379797979797981, + "loss": 2.5802, + "theoretical_loss": 3.3414751977912664, + "tokens_seen": 2768764928 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016377777777777777, + "loss": 2.5866, + "theoretical_loss": 3.341469017201806, + "tokens_seen": 2768830464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016375757575757578, + "loss": 2.3753, + "theoretical_loss": 3.341462836799593, + "tokens_seen": 2768896000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016373737373737374, + "loss": 2.7874, + "theoretical_loss": 3.341456656584618, + "tokens_seen": 2768961536 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001637171717171717, + "loss": 2.495, + "theoretical_loss": 3.3414504765568704, + "tokens_seen": 2769027072 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001636969696969697, + "loss": 2.5404, + "theoretical_loss": 3.341444296716341, + "tokens_seen": 2769092608 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1558514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6093356609344482, + "objective/train/theoretical_loss": 3.3414381170630185, + "objective/train/tokens_used": 1128017376, + "theoretical_loss": 3.3414381170630185, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016367676767676766, + "loss": 2.5117, + "theoretical_loss": 3.3414381170630185, + "tokens_seen": 2769158144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016365656565656568, + "loss": 2.5502, + "theoretical_loss": 3.3414319375968935, + "tokens_seen": 2769223680 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016363636363636363, + "loss": 2.6688, + "theoretical_loss": 3.341425758317955, + "tokens_seen": 2769289216 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016361616161616162, + "loss": 2.6037, + "theoretical_loss": 3.3414195792261947, + "tokens_seen": 2769354752 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001635959595959596, + "loss": 2.5216, + "theoretical_loss": 3.341413400321601, + "tokens_seen": 2769420288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016357575757575758, + "loss": 2.3276, + "theoretical_loss": 3.3414072216041646, + "tokens_seen": 2769485824 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016355555555555557, + "loss": 2.77, + "theoretical_loss": 3.3414010430738745, + "tokens_seen": 2769551360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016353535353535352, + "loss": 2.6692, + "theoretical_loss": 3.3413948647307214, + "tokens_seen": 2769616896 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001635151515151515, + "loss": 2.7702, + "theoretical_loss": 3.341388686574695, + "tokens_seen": 2769682432 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634949494949495, + "loss": 2.5478, + "theoretical_loss": 3.3413825086057853, + "tokens_seen": 2769747968 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016347474747474748, + "loss": 2.6001, + "theoretical_loss": 3.341376330823982, + "tokens_seen": 2769813504 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016345454545454546, + "loss": 2.6599, + "theoretical_loss": 3.341370153229275, + "tokens_seen": 2769879040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016343434343434344, + "loss": 2.4018, + "theoretical_loss": 3.341363975821654, + "tokens_seen": 2769944576 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001634141414141414, + "loss": 2.4786, + "theoretical_loss": 3.341357798601109, + "tokens_seen": 2770010112 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001633939393939394, + "loss": 2.4292, + "theoretical_loss": 3.3413516215676307, + "tokens_seen": 2770075648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016337373737373737, + "loss": 2.6569, + "theoretical_loss": 3.3413454447212083, + "tokens_seen": 2770141184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016335353535353535, + "loss": 2.7641, + "theoretical_loss": 3.341339268061832, + "tokens_seen": 2770206720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016333333333333334, + "loss": 2.4969, + "theoretical_loss": 3.341333091589491, + "tokens_seen": 2770272256 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016331313131313132, + "loss": 2.4472, + "theoretical_loss": 3.341326915304176, + "tokens_seen": 2770337792 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001632929292929293, + "loss": 2.3475, + "theoretical_loss": 3.3413207392058766, + "tokens_seen": 2770403328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016327272727272726, + "loss": 2.6593, + "theoretical_loss": 3.3413145632945827, + "tokens_seen": 2770468864 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016325252525252527, + "loss": 2.6011, + "theoretical_loss": 3.3413083875702845, + "tokens_seen": 2770534400 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016323232323232323, + "loss": 2.6887, + "theoretical_loss": 3.3413022120329714, + "tokens_seen": 2770599936 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016321212121212124, + "loss": 2.5335, + "theoretical_loss": 3.3412960366826336, + "tokens_seen": 2770665472 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001631919191919192, + "loss": 2.5882, + "theoretical_loss": 3.3412898615192614, + "tokens_seen": 2770731008 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1559839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.417963743209839, + "objective/train/theoretical_loss": 3.341283686542844, + "objective/train/tokens_used": 1129655776, + "theoretical_loss": 3.341283686542844, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016317171717171715, + "loss": 2.582, + "theoretical_loss": 3.341283686542844, + "tokens_seen": 2770796544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016315151515151517, + "loss": 2.5261, + "theoretical_loss": 3.341277511753372, + "tokens_seen": 2770862080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016313131313131312, + "loss": 2.3288, + "theoretical_loss": 3.3412713371508347, + "tokens_seen": 2770927616 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016311111111111113, + "loss": 2.5066, + "theoretical_loss": 3.3412651627352226, + "tokens_seen": 2770993152 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001630909090909091, + "loss": 2.6177, + "theoretical_loss": 3.341258988506525, + "tokens_seen": 2771058688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016307070707070707, + "loss": 2.4215, + "theoretical_loss": 3.341252814464732, + "tokens_seen": 2771124224 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016305050505050506, + "loss": 2.6943, + "theoretical_loss": 3.341246640609834, + "tokens_seen": 2771189760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016303030303030304, + "loss": 2.5518, + "theoretical_loss": 3.3412404669418203, + "tokens_seen": 2771255296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016301010101010103, + "loss": 2.6275, + "theoretical_loss": 3.3412342934606816, + "tokens_seen": 2771320832 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016298989898989898, + "loss": 2.6249, + "theoretical_loss": 3.3412281201664067, + "tokens_seen": 2771386368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016296969696969697, + "loss": 2.6768, + "theoretical_loss": 3.341221947058987, + "tokens_seen": 2771451904 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016294949494949495, + "loss": 2.6966, + "theoretical_loss": 3.3412157741384108, + "tokens_seen": 2771517440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016292929292929294, + "loss": 2.5252, + "theoretical_loss": 3.341209601404669, + "tokens_seen": 2771582976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016290909090909092, + "loss": 2.5594, + "theoretical_loss": 3.3412034288577512, + "tokens_seen": 2771648512 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001628888888888889, + "loss": 2.6467, + "theoretical_loss": 3.3411972564976478, + "tokens_seen": 2771714048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016286868686868686, + "loss": 2.4254, + "theoretical_loss": 3.341191084324348, + "tokens_seen": 2771779584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016284848484848484, + "loss": 2.1732, + "theoretical_loss": 3.3411849123378428, + "tokens_seen": 2771845120 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016282828282828283, + "loss": 2.6649, + "theoretical_loss": 3.341178740538121, + "tokens_seen": 2771910656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001628080808080808, + "loss": 2.3579, + "theoretical_loss": 3.341172568925173, + "tokens_seen": 2771976192 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001627878787878788, + "loss": 2.6393, + "theoretical_loss": 3.3411663974989887, + "tokens_seen": 2772041728 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016276767676767675, + "loss": 2.7144, + "theoretical_loss": 3.341160226259558, + "tokens_seen": 2772107264 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016274747474747476, + "loss": 2.6608, + "theoretical_loss": 3.341154055206871, + "tokens_seen": 2772172800 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016272727272727272, + "loss": 2.7125, + "theoretical_loss": 3.3411478843409177, + "tokens_seen": 2772238336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016270707070707073, + "loss": 2.3242, + "theoretical_loss": 3.3411417136616874, + "tokens_seen": 2772303872 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626868686868687, + "loss": 2.7614, + "theoretical_loss": 3.341135543169171, + "tokens_seen": 2772369408 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1560640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.92094087600708, + "objective/train/theoretical_loss": 3.3411293728633575, + "objective/train/tokens_used": 1131294176, + "theoretical_loss": 3.3411293728633575, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016266666666666665, + "loss": 2.563, + "theoretical_loss": 3.3411293728633575, + "tokens_seen": 2772434944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016264646464646466, + "loss": 2.6436, + "theoretical_loss": 3.3411232027442375, + "tokens_seen": 2772500480 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001626262626262626, + "loss": 2.7111, + "theoretical_loss": 3.341117032811801, + "tokens_seen": 2772566016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016260606060606062, + "loss": 2.6834, + "theoretical_loss": 3.341110863066037, + "tokens_seen": 2772631552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016258585858585858, + "loss": 2.4149, + "theoretical_loss": 3.3411046935069364, + "tokens_seen": 2772697088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016256565656565656, + "loss": 2.6828, + "theoretical_loss": 3.341098524134489, + "tokens_seen": 2772762624 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016254545454545455, + "loss": 2.4402, + "theoretical_loss": 3.3410923549486844, + "tokens_seen": 2772828160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016252525252525253, + "loss": 2.5785, + "theoretical_loss": 3.3410861859495125, + "tokens_seen": 2772893696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016250505050505052, + "loss": 2.5087, + "theoretical_loss": 3.3410800171369637, + "tokens_seen": 2772959232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016248484848484847, + "loss": 2.7276, + "theoretical_loss": 3.3410738485110274, + "tokens_seen": 2773024768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016246464646464648, + "loss": 2.4827, + "theoretical_loss": 3.3410676800716943, + "tokens_seen": 2773090304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016244444444444444, + "loss": 2.4233, + "theoretical_loss": 3.3410615118189537, + "tokens_seen": 2773155840 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016242424242424243, + "loss": 2.595, + "theoretical_loss": 3.341055343752796, + "tokens_seen": 2773221376 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001624040404040404, + "loss": 2.5999, + "theoretical_loss": 3.34104917587321, + "tokens_seen": 2773286912 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623838383838384, + "loss": 2.5694, + "theoretical_loss": 3.341043008180187, + "tokens_seen": 2773352448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016236363636363638, + "loss": 2.5109, + "theoretical_loss": 3.3410368406737168, + "tokens_seen": 2773417984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016234343434343436, + "loss": 2.6321, + "theoretical_loss": 3.3410306733537887, + "tokens_seen": 2773483520 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016232323232323232, + "loss": 2.5464, + "theoretical_loss": 3.3410245062203936, + "tokens_seen": 2773549056 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001623030303030303, + "loss": 2.8352, + "theoretical_loss": 3.34101833927352, + "tokens_seen": 2773614592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016228282828282829, + "loss": 2.4403, + "theoretical_loss": 3.341012172513159, + "tokens_seen": 2773680128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016226262626262627, + "loss": 2.6656, + "theoretical_loss": 3.3410060059393003, + "tokens_seen": 2773745664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016224242424242425, + "loss": 2.5953, + "theoretical_loss": 3.340999839551934, + "tokens_seen": 2773811200 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001622222222222222, + "loss": 2.4201, + "theoretical_loss": 3.3409936733510492, + "tokens_seen": 2773876736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016220202020202022, + "loss": 2.5774, + "theoretical_loss": 3.340987507336637, + "tokens_seen": 2773942272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016218181818181818, + "loss": 2.3017, + "theoretical_loss": 3.3409813415086864, + "tokens_seen": 2774007808 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1561903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4264309406280518, + "objective/train/theoretical_loss": 3.340975175867188, + "objective/train/tokens_used": 1132932576, + "theoretical_loss": 3.340975175867188, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001621616161616162, + "loss": 2.7215, + "theoretical_loss": 3.340975175867188, + "tokens_seen": 2774073344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016214141414141415, + "loss": 2.4994, + "theoretical_loss": 3.340969010412132, + "tokens_seen": 2774138880 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001621212121212121, + "loss": 2.6574, + "theoretical_loss": 3.3409628451435074, + "tokens_seen": 2774204416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016210101010101011, + "loss": 2.5983, + "theoretical_loss": 3.340956680061305, + "tokens_seen": 2774269952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016208080808080807, + "loss": 2.4484, + "theoretical_loss": 3.340950515165514, + "tokens_seen": 2774335488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016206060606060608, + "loss": 2.4301, + "theoretical_loss": 3.340944350456125, + "tokens_seen": 2774401024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016204040404040404, + "loss": 2.6174, + "theoretical_loss": 3.340938185933128, + "tokens_seen": 2774466560 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016202020202020202, + "loss": 2.638, + "theoretical_loss": 3.3409320215965126, + "tokens_seen": 2774532096 + }, + { + "epoch": 0.68, + "learning_rate": 0.000162, + "loss": 2.6394, + "theoretical_loss": 3.340925857446269, + "tokens_seen": 2774597632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016197979797979796, + "loss": 2.4507, + "theoretical_loss": 3.3409196934823866, + "tokens_seen": 2774663168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016195959595959598, + "loss": 2.3558, + "theoretical_loss": 3.3409135297048564, + "tokens_seen": 2774728704 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016193939393939393, + "loss": 2.5479, + "theoretical_loss": 3.340907366113667, + "tokens_seen": 2774794240 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016191919191919192, + "loss": 2.7444, + "theoretical_loss": 3.34090120270881, + "tokens_seen": 2774859776 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001618989898989899, + "loss": 2.2333, + "theoretical_loss": 3.340895039490274, + "tokens_seen": 2774925312 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016187878787878788, + "loss": 2.5224, + "theoretical_loss": 3.3408888764580498, + "tokens_seen": 2774990848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016185858585858587, + "loss": 2.3929, + "theoretical_loss": 3.3408827136121264, + "tokens_seen": 2775056384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016183838383838385, + "loss": 2.4212, + "theoretical_loss": 3.340876550952495, + "tokens_seen": 2775121920 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001618181818181818, + "loss": 2.5406, + "theoretical_loss": 3.340870388479145, + "tokens_seen": 2775187456 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001617979797979798, + "loss": 2.7757, + "theoretical_loss": 3.3408642261920662, + "tokens_seen": 2775252992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016177777777777778, + "loss": 2.3138, + "theoretical_loss": 3.3408580640912486, + "tokens_seen": 2775318528 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016175757575757576, + "loss": 2.5826, + "theoretical_loss": 3.3408519021766825, + "tokens_seen": 2775384064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016173737373737374, + "loss": 2.3065, + "theoretical_loss": 3.3408457404483576, + "tokens_seen": 2775449600 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001617171717171717, + "loss": 2.616, + "theoretical_loss": 3.3408395789062637, + "tokens_seen": 2775515136 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001616969696969697, + "loss": 2.6397, + "theoretical_loss": 3.3408334175503915, + "tokens_seen": 2775580672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016167676767676767, + "loss": 2.5598, + "theoretical_loss": 3.34082725638073, + "tokens_seen": 2775646208 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1562750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.69679856300354, + "objective/train/theoretical_loss": 3.3408210953972697, + "objective/train/tokens_used": 1134570976, + "theoretical_loss": 3.3408210953972697, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016165656565656568, + "loss": 2.6293, + "theoretical_loss": 3.3408210953972697, + "tokens_seen": 2775711744 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016163636363636364, + "loss": 2.5772, + "theoretical_loss": 3.340814934600001, + "tokens_seen": 2775777280 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016161616161616162, + "loss": 2.567, + "theoretical_loss": 3.340808773988913, + "tokens_seen": 2775842816 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001615959595959596, + "loss": 2.5594, + "theoretical_loss": 3.3408026135639965, + "tokens_seen": 2775908352 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016157575757575756, + "loss": 2.6146, + "theoretical_loss": 3.3407964533252406, + "tokens_seen": 2775973888 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016155555555555557, + "loss": 2.4528, + "theoretical_loss": 3.340790293272636, + "tokens_seen": 2776039424 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016153535353535353, + "loss": 2.5448, + "theoretical_loss": 3.3407841334061725, + "tokens_seen": 2776104960 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016151515151515154, + "loss": 2.6835, + "theoretical_loss": 3.34077797372584, + "tokens_seen": 2776170496 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001614949494949495, + "loss": 2.6918, + "theoretical_loss": 3.3407718142316285, + "tokens_seen": 2776236032 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016147474747474748, + "loss": 2.5402, + "theoretical_loss": 3.340765654923528, + "tokens_seen": 2776301568 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016145454545454547, + "loss": 2.3257, + "theoretical_loss": 3.340759495801528, + "tokens_seen": 2776367104 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016143434343434342, + "loss": 2.7122, + "theoretical_loss": 3.3407533368656197, + "tokens_seen": 2776432640 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016141414141414143, + "loss": 2.6641, + "theoretical_loss": 3.340747178115792, + "tokens_seen": 2776498176 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001613939393939394, + "loss": 2.5134, + "theoretical_loss": 3.340741019552035, + "tokens_seen": 2776563712 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016137373737373737, + "loss": 2.4251, + "theoretical_loss": 3.3407348611743393, + "tokens_seen": 2776629248 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016135353535353536, + "loss": 2.5672, + "theoretical_loss": 3.340728702982694, + "tokens_seen": 2776694784 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016133333333333334, + "loss": 2.6573, + "theoretical_loss": 3.34072254497709, + "tokens_seen": 2776760320 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016131313131313133, + "loss": 2.5363, + "theoretical_loss": 3.340716387157517, + "tokens_seen": 2776825856 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016129292929292928, + "loss": 2.4534, + "theoretical_loss": 3.3407102295239643, + "tokens_seen": 2776891392 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016127272727272727, + "loss": 2.3891, + "theoretical_loss": 3.340704072076423, + "tokens_seen": 2776956928 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016125252525252525, + "loss": 2.4836, + "theoretical_loss": 3.340697914814882, + "tokens_seen": 2777022464 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016123232323232324, + "loss": 2.6796, + "theoretical_loss": 3.3406917577393322, + "tokens_seen": 2777088000 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016121212121212122, + "loss": 2.6729, + "theoretical_loss": 3.3406856008497634, + "tokens_seen": 2777153536 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001611919191919192, + "loss": 2.3707, + "theoretical_loss": 3.3406794441461654, + "tokens_seen": 2777219072 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016117171717171716, + "loss": 2.5204, + "theoretical_loss": 3.340673287628528, + "tokens_seen": 2777284608 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1563516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8078527450561523, + "objective/train/theoretical_loss": 3.3406671312968417, + "objective/train/tokens_used": 1136209376, + "theoretical_loss": 3.3406671312968417, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016115151515151517, + "loss": 2.5601, + "theoretical_loss": 3.3406671312968417, + "tokens_seen": 2777350144 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016113131313131313, + "loss": 2.3998, + "theoretical_loss": 3.3406609751510956, + "tokens_seen": 2777415680 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001611111111111111, + "loss": 2.6246, + "theoretical_loss": 3.340654819191281, + "tokens_seen": 2777481216 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001610909090909091, + "loss": 2.5636, + "theoretical_loss": 3.3406486634173866, + "tokens_seen": 2777546752 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016107070707070705, + "loss": 2.5558, + "theoretical_loss": 3.3406425078294033, + "tokens_seen": 2777612288 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016105050505050506, + "loss": 2.667, + "theoretical_loss": 3.3406363524273206, + "tokens_seen": 2777677824 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016103030303030302, + "loss": 2.76, + "theoretical_loss": 3.340630197211129, + "tokens_seen": 2777743360 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016101010101010103, + "loss": 2.5436, + "theoretical_loss": 3.340624042180818, + "tokens_seen": 2777808896 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160989898989899, + "loss": 2.5802, + "theoretical_loss": 3.340617887336378, + "tokens_seen": 2777874432 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016096969696969697, + "loss": 2.5729, + "theoretical_loss": 3.3406117326777984, + "tokens_seen": 2777939968 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016094949494949496, + "loss": 2.7422, + "theoretical_loss": 3.34060557820507, + "tokens_seen": 2778005504 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001609292929292929, + "loss": 2.3502, + "theoretical_loss": 3.3405994239181824, + "tokens_seen": 2778071040 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016090909090909092, + "loss": 2.8191, + "theoretical_loss": 3.3405932698171257, + "tokens_seen": 2778136576 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016088888888888888, + "loss": 2.4748, + "theoretical_loss": 3.3405871159018896, + "tokens_seen": 2778202112 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016086868686868687, + "loss": 2.4873, + "theoretical_loss": 3.3405809621724645, + "tokens_seen": 2778267648 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016084848484848485, + "loss": 2.7523, + "theoretical_loss": 3.3405748086288396, + "tokens_seen": 2778333184 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016082828282828283, + "loss": 2.4669, + "theoretical_loss": 3.340568655271006, + "tokens_seen": 2778398720 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016080808080808082, + "loss": 2.6025, + "theoretical_loss": 3.340562502098954, + "tokens_seen": 2778464256 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607878787878788, + "loss": 2.2056, + "theoretical_loss": 3.3405563491126715, + "tokens_seen": 2778529792 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016076767676767679, + "loss": 2.6251, + "theoretical_loss": 3.340550196312151, + "tokens_seen": 2778595328 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016074747474747474, + "loss": 2.7162, + "theoretical_loss": 3.340544043697381, + "tokens_seen": 2778660864 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016072727272727273, + "loss": 2.6119, + "theoretical_loss": 3.3405378912683514, + "tokens_seen": 2778726400 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001607070707070707, + "loss": 2.5879, + "theoretical_loss": 3.340531739025053, + "tokens_seen": 2778791936 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001606868686868687, + "loss": 2.6473, + "theoretical_loss": 3.340525586967476, + "tokens_seen": 2778857472 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016066666666666668, + "loss": 2.5684, + "theoretical_loss": 3.3405194350956093, + "tokens_seen": 2778923008 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1564998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2400360107421875, + "objective/train/theoretical_loss": 3.340513283409444, + "objective/train/tokens_used": 1137847776, + "theoretical_loss": 3.340513283409444, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016064646464646466, + "loss": 2.2792, + "theoretical_loss": 3.340513283409444, + "tokens_seen": 2778988544 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016062626262626262, + "loss": 2.5692, + "theoretical_loss": 3.340507131908969, + "tokens_seen": 2779054080 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016060606060606063, + "loss": 2.6753, + "theoretical_loss": 3.3405009805941757, + "tokens_seen": 2779119616 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001605858585858586, + "loss": 2.4907, + "theoretical_loss": 3.340494829465053, + "tokens_seen": 2779185152 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016056565656565657, + "loss": 2.5227, + "theoretical_loss": 3.3404886785215915, + "tokens_seen": 2779250688 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016054545454545455, + "loss": 2.5346, + "theoretical_loss": 3.340482527763781, + "tokens_seen": 2779316224 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001605252525252525, + "loss": 2.6147, + "theoretical_loss": 3.3404763771916115, + "tokens_seen": 2779381760 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016050505050505052, + "loss": 2.5787, + "theoretical_loss": 3.340470226805073, + "tokens_seen": 2779447296 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016048484848484848, + "loss": 2.5271, + "theoretical_loss": 3.340464076604156, + "tokens_seen": 2779512832 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604646464646465, + "loss": 2.5817, + "theoretical_loss": 3.3404579265888494, + "tokens_seen": 2779578368 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016044444444444445, + "loss": 2.6833, + "theoretical_loss": 3.3404517767591444, + "tokens_seen": 2779643904 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001604242424242424, + "loss": 2.5983, + "theoretical_loss": 3.3404456271150305, + "tokens_seen": 2779709440 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016040404040404042, + "loss": 2.6535, + "theoretical_loss": 3.3404394776564974, + "tokens_seen": 2779774976 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016038383838383837, + "loss": 2.5269, + "theoretical_loss": 3.340433328383536, + "tokens_seen": 2779840512 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016036363636363638, + "loss": 2.3085, + "theoretical_loss": 3.3404271792961358, + "tokens_seen": 2779906048 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016034343434343434, + "loss": 2.4548, + "theoretical_loss": 3.3404210303942863, + "tokens_seen": 2779971584 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016032323232323232, + "loss": 2.5624, + "theoretical_loss": 3.3404148816779786, + "tokens_seen": 2780037120 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001603030303030303, + "loss": 2.3378, + "theoretical_loss": 3.3404087331472025, + "tokens_seen": 2780102656 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602828282828283, + "loss": 2.6041, + "theoretical_loss": 3.340402584801947, + "tokens_seen": 2780168192 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016026262626262628, + "loss": 2.6894, + "theoretical_loss": 3.3403964366422034, + "tokens_seen": 2780233728 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016024242424242423, + "loss": 2.2144, + "theoretical_loss": 3.340390288667961, + "tokens_seen": 2780299264 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016022222222222222, + "loss": 2.6175, + "theoretical_loss": 3.34038414087921, + "tokens_seen": 2780364800 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001602020202020202, + "loss": 2.5868, + "theoretical_loss": 3.3403779932759403, + "tokens_seen": 2780430336 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016018181818181818, + "loss": 2.5112, + "theoretical_loss": 3.3403718458581424, + "tokens_seen": 2780495872 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016016161616161617, + "loss": 2.7383, + "theoretical_loss": 3.340365698625806, + "tokens_seen": 2780561408 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1565682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4069387912750244, + "objective/train/theoretical_loss": 3.340359551578921, + "objective/train/tokens_used": 1139486176, + "theoretical_loss": 3.340359551578921, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016014141414141415, + "loss": 2.4095, + "theoretical_loss": 3.340359551578921, + "tokens_seen": 2780626944 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001601212121212121, + "loss": 2.6067, + "theoretical_loss": 3.3403534047174777, + "tokens_seen": 2780692480 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016010101010101012, + "loss": 2.5551, + "theoretical_loss": 3.340347258041466, + "tokens_seen": 2780758016 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016008080808080808, + "loss": 2.3327, + "theoretical_loss": 3.3403411115508757, + "tokens_seen": 2780823552 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016006060606060606, + "loss": 2.4661, + "theoretical_loss": 3.3403349652456975, + "tokens_seen": 2780889088 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016004040404040405, + "loss": 2.4686, + "theoretical_loss": 3.340328819125921, + "tokens_seen": 2780954624 + }, + { + "epoch": 0.68, + "learning_rate": 0.000160020202020202, + "loss": 2.7027, + "theoretical_loss": 3.340322673191536, + "tokens_seen": 2781020160 + }, + { + "epoch": 0.68, + "learning_rate": 0.00016, + "loss": 2.5157, + "theoretical_loss": 3.340316527442533, + "tokens_seen": 2781085696 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015997979797979797, + "loss": 2.6944, + "theoretical_loss": 3.340310381878902, + "tokens_seen": 2781151232 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015995959595959598, + "loss": 2.6286, + "theoretical_loss": 3.3403042365006326, + "tokens_seen": 2781216768 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015993939393939394, + "loss": 2.7748, + "theoretical_loss": 3.3402980913077154, + "tokens_seen": 2781282304 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015991919191919192, + "loss": 2.5344, + "theoretical_loss": 3.34029194630014, + "tokens_seen": 2781347840 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001598989898989899, + "loss": 2.841, + "theoretical_loss": 3.3402858014778967, + "tokens_seen": 2781413376 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015987878787878786, + "loss": 2.4859, + "theoretical_loss": 3.3402796568409756, + "tokens_seen": 2781478912 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015985858585858587, + "loss": 2.609, + "theoretical_loss": 3.3402735123893663, + "tokens_seen": 2781544448 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015983838383838383, + "loss": 2.4203, + "theoretical_loss": 3.3402673681230595, + "tokens_seen": 2781609984 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015981818181818184, + "loss": 2.6736, + "theoretical_loss": 3.3402612240420444, + "tokens_seen": 2781675520 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001597979797979798, + "loss": 2.8731, + "theoretical_loss": 3.340255080146312, + "tokens_seen": 2781741056 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015977777777777778, + "loss": 2.3251, + "theoretical_loss": 3.340248936435852, + "tokens_seen": 2781806592 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015975757575757577, + "loss": 2.4345, + "theoretical_loss": 3.340242792910654, + "tokens_seen": 2781872128 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015973737373737372, + "loss": 2.5985, + "theoretical_loss": 3.3402366495707083, + "tokens_seen": 2781937664 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015971717171717173, + "loss": 2.5839, + "theoretical_loss": 3.340230506416005, + "tokens_seen": 2782003200 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001596969696969697, + "loss": 2.7091, + "theoretical_loss": 3.3402243634465347, + "tokens_seen": 2782068736 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015967676767676768, + "loss": 2.4968, + "theoretical_loss": 3.340218220662287, + "tokens_seen": 2782134272 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015965656565656566, + "loss": 2.4041, + "theoretical_loss": 3.340212078063251, + "tokens_seen": 2782199808 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1566838, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.612881898880005, + "objective/train/theoretical_loss": 3.3402059356494185, + "objective/train/tokens_used": 1141124576, + "theoretical_loss": 3.3402059356494185, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015963636363636364, + "loss": 2.6558, + "theoretical_loss": 3.3402059356494185, + "tokens_seen": 2782265344 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015961616161616163, + "loss": 2.5871, + "theoretical_loss": 3.3401997934207786, + "tokens_seen": 2782330880 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001595959595959596, + "loss": 2.5086, + "theoretical_loss": 3.340193651377321, + "tokens_seen": 2782396416 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015957575757575757, + "loss": 2.6862, + "theoretical_loss": 3.340187509519037, + "tokens_seen": 2782461952 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015955555555555555, + "loss": 2.5861, + "theoretical_loss": 3.3401813678459154, + "tokens_seen": 2782527488 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015953535353535354, + "loss": 2.3044, + "theoretical_loss": 3.3401752263579465, + "tokens_seen": 2782593024 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015951515151515152, + "loss": 2.3548, + "theoretical_loss": 3.3401690850551207, + "tokens_seen": 2782658560 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001594949494949495, + "loss": 2.6277, + "theoretical_loss": 3.340162943937428, + "tokens_seen": 2782724096 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015947474747474746, + "loss": 2.4665, + "theoretical_loss": 3.3401568030048585, + "tokens_seen": 2782789632 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015945454545454547, + "loss": 2.5377, + "theoretical_loss": 3.340150662257402, + "tokens_seen": 2782855168 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015943434343434343, + "loss": 2.5627, + "theoretical_loss": 3.3401445216950494, + "tokens_seen": 2782920704 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015941414141414144, + "loss": 2.8464, + "theoretical_loss": 3.3401383813177894, + "tokens_seen": 2782986240 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001593939393939394, + "loss": 2.4555, + "theoretical_loss": 3.340132241125613, + "tokens_seen": 2783051776 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015937373737373735, + "loss": 2.6487, + "theoretical_loss": 3.3401261011185097, + "tokens_seen": 2783117312 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015935353535353536, + "loss": 2.6483, + "theoretical_loss": 3.3401199612964705, + "tokens_seen": 2783182848 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015933333333333332, + "loss": 2.4074, + "theoretical_loss": 3.340113821659484, + "tokens_seen": 2783248384 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015931313131313133, + "loss": 2.5841, + "theoretical_loss": 3.3401076822075417, + "tokens_seen": 2783313920 + }, + { + "epoch": 0.68, + "learning_rate": 0.0001592929292929293, + "loss": 2.5901, + "theoretical_loss": 3.3401015429406327, + "tokens_seen": 2783379456 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015927272727272727, + "loss": 2.6237, + "theoretical_loss": 3.340095403858748, + "tokens_seen": 2783444992 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015925252525252526, + "loss": 2.4506, + "theoretical_loss": 3.3400892649618767, + "tokens_seen": 2783510528 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015923232323232324, + "loss": 2.6244, + "theoretical_loss": 3.3400831262500095, + "tokens_seen": 2783576064 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015921212121212122, + "loss": 2.5583, + "theoretical_loss": 3.340076987723136, + "tokens_seen": 2783641600 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015919191919191918, + "loss": 2.7429, + "theoretical_loss": 3.3400708493812465, + "tokens_seen": 2783707136 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015917171717171717, + "loss": 2.5697, + "theoretical_loss": 3.3400647112243314, + "tokens_seen": 2783772672 + }, + { + "epoch": 0.68, + "learning_rate": 0.00015915151515151515, + "loss": 2.5057, + "theoretical_loss": 3.3400585732523798, + "tokens_seen": 2783838208 + }, + { + "epoch": 0.68, + "objective/train/docs_used": 1567583, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5856521129608154, + "objective/train/theoretical_loss": 3.340052435465383, + "objective/train/tokens_used": 1142762976, + "theoretical_loss": 3.340052435465383, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015913131313131313, + "loss": 2.6528, + "theoretical_loss": 3.340052435465383, + "tokens_seen": 2783903744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015911111111111112, + "loss": 2.4748, + "theoretical_loss": 3.3400462978633305, + "tokens_seen": 2783969280 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001590909090909091, + "loss": 2.5542, + "theoretical_loss": 3.340040160446212, + "tokens_seen": 2784034816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015907070707070706, + "loss": 2.5049, + "theoretical_loss": 3.3400340232140184, + "tokens_seen": 2784100352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015905050505050507, + "loss": 2.602, + "theoretical_loss": 3.340027886166739, + "tokens_seen": 2784165888 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015903030303030303, + "loss": 2.4616, + "theoretical_loss": 3.3400217493043645, + "tokens_seen": 2784231424 + }, + { + "epoch": 0.69, + "learning_rate": 0.000159010101010101, + "loss": 2.4775, + "theoretical_loss": 3.3400156126268845, + "tokens_seen": 2784296960 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158989898989899, + "loss": 2.5874, + "theoretical_loss": 3.340009476134289, + "tokens_seen": 2784362496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015896969696969698, + "loss": 2.5542, + "theoretical_loss": 3.340003339826569, + "tokens_seen": 2784428032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015894949494949496, + "loss": 2.6277, + "theoretical_loss": 3.3399972037037133, + "tokens_seen": 2784493568 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015892929292929292, + "loss": 2.5667, + "theoretical_loss": 3.3399910677657125, + "tokens_seen": 2784559104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015890909090909093, + "loss": 2.5839, + "theoretical_loss": 3.339984932012557, + "tokens_seen": 2784624640 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001588888888888889, + "loss": 2.4772, + "theoretical_loss": 3.339978796444237, + "tokens_seen": 2784690176 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001588686868686869, + "loss": 2.4095, + "theoretical_loss": 3.339972661060742, + "tokens_seen": 2784755712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015884848484848485, + "loss": 2.5984, + "theoretical_loss": 3.3399665258620623, + "tokens_seen": 2784821248 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001588282828282828, + "loss": 2.4212, + "theoretical_loss": 3.339960390848188, + "tokens_seen": 2784886784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015880808080808082, + "loss": 2.4553, + "theoretical_loss": 3.339954256019109, + "tokens_seen": 2784952320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015878787878787878, + "loss": 2.4452, + "theoretical_loss": 3.3399481213748157, + "tokens_seen": 2785017856 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001587676767676768, + "loss": 2.5029, + "theoretical_loss": 3.3399419869152984, + "tokens_seen": 2785083392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015874747474747475, + "loss": 2.5308, + "theoretical_loss": 3.3399358526405463, + "tokens_seen": 2785148928 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015872727272727273, + "loss": 2.5499, + "theoretical_loss": 3.3399297185505503, + "tokens_seen": 2785214464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015870707070707072, + "loss": 2.6807, + "theoretical_loss": 3.3399235846453004, + "tokens_seen": 2785280000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015868686868686867, + "loss": 2.6558, + "theoretical_loss": 3.339917450924786, + "tokens_seen": 2785345536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015866666666666668, + "loss": 2.5823, + "theoretical_loss": 3.3399113173889985, + "tokens_seen": 2785411072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015864646464646464, + "loss": 2.6244, + "theoretical_loss": 3.3399051840379266, + "tokens_seen": 2785476608 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1568939, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9206714630126953, + "objective/train/theoretical_loss": 3.339899050871561, + "objective/train/tokens_used": 1144401376, + "theoretical_loss": 3.339899050871561, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015862626262626262, + "loss": 2.304, + "theoretical_loss": 3.339899050871561, + "tokens_seen": 2785542144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001586060606060606, + "loss": 2.3314, + "theoretical_loss": 3.339892917889892, + "tokens_seen": 2785607680 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585858585858586, + "loss": 2.6028, + "theoretical_loss": 3.3398867850929093, + "tokens_seen": 2785673216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015856565656565658, + "loss": 2.3486, + "theoretical_loss": 3.339880652480603, + "tokens_seen": 2785738752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015854545454545456, + "loss": 2.5013, + "theoretical_loss": 3.339874520052964, + "tokens_seen": 2785804288 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015852525252525252, + "loss": 2.7189, + "theoretical_loss": 3.339868387809981, + "tokens_seen": 2785869824 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001585050505050505, + "loss": 2.2963, + "theoretical_loss": 3.3398622557516453, + "tokens_seen": 2785935360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015848484848484848, + "loss": 2.683, + "theoretical_loss": 3.3398561238779463, + "tokens_seen": 2786000896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015846464646464647, + "loss": 2.5614, + "theoretical_loss": 3.3398499921888742, + "tokens_seen": 2786066432 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015844444444444445, + "loss": 2.7653, + "theoretical_loss": 3.33984386068442, + "tokens_seen": 2786131968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001584242424242424, + "loss": 2.4288, + "theoretical_loss": 3.339837729364572, + "tokens_seen": 2786197504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015840404040404042, + "loss": 2.6189, + "theoretical_loss": 3.3398315982293223, + "tokens_seen": 2786263040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015838383838383838, + "loss": 2.4651, + "theoretical_loss": 3.3398254672786596, + "tokens_seen": 2786328576 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583636363636364, + "loss": 2.4649, + "theoretical_loss": 3.339819336512574, + "tokens_seen": 2786394112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015834343434343435, + "loss": 2.519, + "theoretical_loss": 3.3398132059310566, + "tokens_seen": 2786459648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583232323232323, + "loss": 2.5677, + "theoretical_loss": 3.3398070755340967, + "tokens_seen": 2786525184 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001583030303030303, + "loss": 2.6453, + "theoretical_loss": 3.3398009453216853, + "tokens_seen": 2786590720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015828282828282827, + "loss": 2.4487, + "theoretical_loss": 3.339794815293811, + "tokens_seen": 2786656256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015826262626262628, + "loss": 2.5146, + "theoretical_loss": 3.3397886854504653, + "tokens_seen": 2786721792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015824242424242424, + "loss": 2.6421, + "theoretical_loss": 3.339782555791637, + "tokens_seen": 2786787328 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015822222222222222, + "loss": 2.489, + "theoretical_loss": 3.339776426317318, + "tokens_seen": 2786852864 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001582020202020202, + "loss": 2.6537, + "theoretical_loss": 3.339770297027497, + "tokens_seen": 2786918400 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001581818181818182, + "loss": 2.288, + "theoretical_loss": 3.3397641679221644, + "tokens_seen": 2786983936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015816161616161617, + "loss": 2.5389, + "theoretical_loss": 3.3397580390013104, + "tokens_seen": 2787049472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015814141414141413, + "loss": 2.6531, + "theoretical_loss": 3.339751910264925, + "tokens_seen": 2787115008 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1569381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.321000576019287, + "objective/train/theoretical_loss": 3.3397457817129985, + "objective/train/tokens_used": 1146039776, + "theoretical_loss": 3.3397457817129985, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015812121212121214, + "loss": 2.3858, + "theoretical_loss": 3.3397457817129985, + "tokens_seen": 2787180544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001581010101010101, + "loss": 2.3466, + "theoretical_loss": 3.339739653345521, + "tokens_seen": 2787246080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015808080808080808, + "loss": 2.5478, + "theoretical_loss": 3.3397335251624822, + "tokens_seen": 2787311616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015806060606060607, + "loss": 2.5886, + "theoretical_loss": 3.3397273971638732, + "tokens_seen": 2787377152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015804040404040405, + "loss": 2.7656, + "theoretical_loss": 3.339721269349683, + "tokens_seen": 2787442688 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015802020202020203, + "loss": 2.744, + "theoretical_loss": 3.339715141719902, + "tokens_seen": 2787508224 + }, + { + "epoch": 0.69, + "learning_rate": 0.000158, + "loss": 2.5388, + "theoretical_loss": 3.339709014274521, + "tokens_seen": 2787573760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015797979797979798, + "loss": 2.4609, + "theoretical_loss": 3.3397028870135292, + "tokens_seen": 2787639296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015795959595959596, + "loss": 2.5877, + "theoretical_loss": 3.3396967599369174, + "tokens_seen": 2787704832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015793939393939394, + "loss": 2.4514, + "theoretical_loss": 3.3396906330446754, + "tokens_seen": 2787770368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015791919191919193, + "loss": 2.6964, + "theoretical_loss": 3.339684506336793, + "tokens_seen": 2787835904 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001578989898989899, + "loss": 2.5245, + "theoretical_loss": 3.339678379813261, + "tokens_seen": 2787901440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015787878787878787, + "loss": 2.6026, + "theoretical_loss": 3.3396722534740695, + "tokens_seen": 2787966976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015785858585858588, + "loss": 2.5295, + "theoretical_loss": 3.339666127319208, + "tokens_seen": 2788032512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015783838383838384, + "loss": 2.6126, + "theoretical_loss": 3.339660001348667, + "tokens_seen": 2788098048 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015781818181818182, + "loss": 2.5626, + "theoretical_loss": 3.3396538755624365, + "tokens_seen": 2788163584 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001577979797979798, + "loss": 2.497, + "theoretical_loss": 3.339647749960507, + "tokens_seen": 2788229120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015777777777777776, + "loss": 2.3343, + "theoretical_loss": 3.3396416245428675, + "tokens_seen": 2788294656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015775757575757577, + "loss": 2.3312, + "theoretical_loss": 3.3396354993095096, + "tokens_seen": 2788360192 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015773737373737373, + "loss": 2.3612, + "theoretical_loss": 3.3396293742604226, + "tokens_seen": 2788425728 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015771717171717174, + "loss": 2.6414, + "theoretical_loss": 3.339623249395597, + "tokens_seen": 2788491264 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001576969696969697, + "loss": 2.6847, + "theoretical_loss": 3.3396171247150224, + "tokens_seen": 2788556800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015767676767676768, + "loss": 2.5519, + "theoretical_loss": 3.339611000218689, + "tokens_seen": 2788622336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015765656565656566, + "loss": 2.5061, + "theoretical_loss": 3.3396048759065877, + "tokens_seen": 2788687872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015763636363636362, + "loss": 2.6309, + "theoretical_loss": 3.3395987517787082, + "tokens_seen": 2788753408 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1570879, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5905990600585938, + "objective/train/theoretical_loss": 3.33959262783504, + "objective/train/tokens_used": 1147678176, + "theoretical_loss": 3.33959262783504, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015761616161616163, + "loss": 2.6512, + "theoretical_loss": 3.33959262783504, + "tokens_seen": 2788818944 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001575959595959596, + "loss": 2.278, + "theoretical_loss": 3.339586504075574, + "tokens_seen": 2788884480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015757575757575757, + "loss": 2.5097, + "theoretical_loss": 3.3395803805003004, + "tokens_seen": 2788950016 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015755555555555556, + "loss": 2.3667, + "theoretical_loss": 3.3395742571092084, + "tokens_seen": 2789015552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015753535353535354, + "loss": 2.602, + "theoretical_loss": 3.339568133902289, + "tokens_seen": 2789081088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015751515151515153, + "loss": 2.7681, + "theoretical_loss": 3.3395620108795323, + "tokens_seen": 2789146624 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001574949494949495, + "loss": 2.3503, + "theoretical_loss": 3.339555888040928, + "tokens_seen": 2789212160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015747474747474747, + "loss": 2.6054, + "theoretical_loss": 3.3395497653864665, + "tokens_seen": 2789277696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015745454545454545, + "loss": 2.665, + "theoretical_loss": 3.339543642916138, + "tokens_seen": 2789343232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015743434343434343, + "loss": 2.5792, + "theoretical_loss": 3.339537520629932, + "tokens_seen": 2789408768 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015741414141414142, + "loss": 2.4938, + "theoretical_loss": 3.33953139852784, + "tokens_seen": 2789474304 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001573939393939394, + "loss": 2.5073, + "theoretical_loss": 3.3395252766098507, + "tokens_seen": 2789539840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015737373737373736, + "loss": 2.84, + "theoretical_loss": 3.339519154875955, + "tokens_seen": 2789605376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015735353535353537, + "loss": 2.5159, + "theoretical_loss": 3.339513033326143, + "tokens_seen": 2789670912 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015733333333333333, + "loss": 2.5675, + "theoretical_loss": 3.3395069119604046, + "tokens_seen": 2789736448 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015731313131313134, + "loss": 2.6832, + "theoretical_loss": 3.3395007907787297, + "tokens_seen": 2789801984 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001572929292929293, + "loss": 2.5197, + "theoretical_loss": 3.3394946697811094, + "tokens_seen": 2789867520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015727272727272728, + "loss": 2.3008, + "theoretical_loss": 3.339488548967533, + "tokens_seen": 2789933056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015725252525252526, + "loss": 2.6094, + "theoretical_loss": 3.339482428337991, + "tokens_seen": 2789998592 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015723232323232322, + "loss": 2.4021, + "theoretical_loss": 3.3394763078924736, + "tokens_seen": 2790064128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015721212121212123, + "loss": 2.3681, + "theoretical_loss": 3.3394701876309703, + "tokens_seen": 2790129664 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571919191919192, + "loss": 2.6728, + "theoretical_loss": 3.3394640675534717, + "tokens_seen": 2790195200 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571717171717172, + "loss": 2.5592, + "theoretical_loss": 3.3394579476599686, + "tokens_seen": 2790260736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015715151515151516, + "loss": 2.2934, + "theoretical_loss": 3.33945182795045, + "tokens_seen": 2790326272 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001571313131313131, + "loss": 2.5044, + "theoretical_loss": 3.3394457084249067, + "tokens_seen": 2790391808 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1571508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8654489517211914, + "objective/train/theoretical_loss": 3.3394395890833284, + "objective/train/tokens_used": 1149316576, + "theoretical_loss": 3.3394395890833284, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015711111111111112, + "loss": 2.6096, + "theoretical_loss": 3.3394395890833284, + "tokens_seen": 2790457344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015709090909090908, + "loss": 2.6529, + "theoretical_loss": 3.339433469925706, + "tokens_seen": 2790522880 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001570707070707071, + "loss": 2.606, + "theoretical_loss": 3.339427350952029, + "tokens_seen": 2790588416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015705050505050505, + "loss": 2.7291, + "theoretical_loss": 3.339421232162288, + "tokens_seen": 2790653952 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015703030303030303, + "loss": 2.7036, + "theoretical_loss": 3.3394151135564725, + "tokens_seen": 2790719488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015701010101010102, + "loss": 2.5143, + "theoretical_loss": 3.3394089951345736, + "tokens_seen": 2790785024 + }, + { + "epoch": 0.69, + "learning_rate": 0.000156989898989899, + "loss": 2.325, + "theoretical_loss": 3.3394028768965804, + "tokens_seen": 2790850560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015696969696969698, + "loss": 2.5194, + "theoretical_loss": 3.3393967588424838, + "tokens_seen": 2790916096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015694949494949494, + "loss": 2.5618, + "theoretical_loss": 3.3393906409722733, + "tokens_seen": 2790981632 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015692929292929292, + "loss": 2.6242, + "theoretical_loss": 3.3393845232859403, + "tokens_seen": 2791047168 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001569090909090909, + "loss": 2.683, + "theoretical_loss": 3.3393784057834734, + "tokens_seen": 2791112704 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001568888888888889, + "loss": 2.4782, + "theoretical_loss": 3.339372288464864, + "tokens_seen": 2791178240 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015686868686868688, + "loss": 2.5, + "theoretical_loss": 3.3393661713301013, + "tokens_seen": 2791243776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015684848484848486, + "loss": 2.5101, + "theoretical_loss": 3.3393600543791764, + "tokens_seen": 2791309312 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015682828282828282, + "loss": 2.3604, + "theoretical_loss": 3.3393539376120787, + "tokens_seen": 2791374848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015680808080808083, + "loss": 2.6783, + "theoretical_loss": 3.339347821028799, + "tokens_seen": 2791440384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015678787878787879, + "loss": 2.4867, + "theoretical_loss": 3.3393417046293266, + "tokens_seen": 2791505920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015676767676767677, + "loss": 2.3767, + "theoretical_loss": 3.339335588413652, + "tokens_seen": 2791571456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015674747474747475, + "loss": 2.5675, + "theoretical_loss": 3.339329472381766, + "tokens_seen": 2791636992 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001567272727272727, + "loss": 2.7604, + "theoretical_loss": 3.339323356533658, + "tokens_seen": 2791702528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015670707070707072, + "loss": 2.6327, + "theoretical_loss": 3.3393172408693186, + "tokens_seen": 2791768064 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015668686868686868, + "loss": 2.6316, + "theoretical_loss": 3.3393111253887375, + "tokens_seen": 2791833600 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001566666666666667, + "loss": 2.5019, + "theoretical_loss": 3.3393050100919055, + "tokens_seen": 2791899136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015664646464646465, + "loss": 2.3853, + "theoretical_loss": 3.3392988949788123, + "tokens_seen": 2791964672 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015662626262626263, + "loss": 2.3775, + "theoretical_loss": 3.3392927800494485, + "tokens_seen": 2792030208 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1572267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5414109230041504, + "objective/train/theoretical_loss": 3.3392866653038036, + "objective/train/tokens_used": 1150954976, + "theoretical_loss": 3.3392866653038036, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015660606060606061, + "loss": 2.4941, + "theoretical_loss": 3.3392866653038036, + "tokens_seen": 2792095744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015658585858585857, + "loss": 2.5216, + "theoretical_loss": 3.339280550741868, + "tokens_seen": 2792161280 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015656565656565658, + "loss": 2.617, + "theoretical_loss": 3.3392744363636324, + "tokens_seen": 2792226816 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015654545454545454, + "loss": 2.3374, + "theoretical_loss": 3.3392683221690866, + "tokens_seen": 2792292352 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015652525252525252, + "loss": 2.7115, + "theoretical_loss": 3.33926220815822, + "tokens_seen": 2792357888 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001565050505050505, + "loss": 2.4777, + "theoretical_loss": 3.3392560943310245, + "tokens_seen": 2792423424 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564848484848485, + "loss": 2.6682, + "theoretical_loss": 3.339249980687489, + "tokens_seen": 2792488960 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015646464646464647, + "loss": 2.7402, + "theoretical_loss": 3.3392438672276037, + "tokens_seen": 2792554496 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015644444444444443, + "loss": 2.5366, + "theoretical_loss": 3.339237753951359, + "tokens_seen": 2792620032 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015642424242424244, + "loss": 2.6813, + "theoretical_loss": 3.3392316408587455, + "tokens_seen": 2792685568 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001564040404040404, + "loss": 2.4138, + "theoretical_loss": 3.3392255279497527, + "tokens_seen": 2792751104 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015638383838383838, + "loss": 2.5813, + "theoretical_loss": 3.3392194152243713, + "tokens_seen": 2792816640 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015636363636363637, + "loss": 2.604, + "theoretical_loss": 3.339213302682591, + "tokens_seen": 2792882176 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015634343434343435, + "loss": 2.7129, + "theoretical_loss": 3.339207190324402, + "tokens_seen": 2792947712 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015632323232323234, + "loss": 2.5389, + "theoretical_loss": 3.339201078149795, + "tokens_seen": 2793013248 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015630303030303032, + "loss": 2.6475, + "theoretical_loss": 3.33919496615876, + "tokens_seen": 2793078784 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015628282828282828, + "loss": 2.5245, + "theoretical_loss": 3.3391888543512867, + "tokens_seen": 2793144320 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015626262626262626, + "loss": 2.4973, + "theoretical_loss": 3.339182742727366, + "tokens_seen": 2793209856 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015624242424242424, + "loss": 2.6567, + "theoretical_loss": 3.3391766312869873, + "tokens_seen": 2793275392 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015622222222222223, + "loss": 2.7226, + "theoretical_loss": 3.339170520030142, + "tokens_seen": 2793340928 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001562020202020202, + "loss": 2.4653, + "theoretical_loss": 3.339164408956819, + "tokens_seen": 2793406464 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015618181818181817, + "loss": 2.6457, + "theoretical_loss": 3.3391582980670087, + "tokens_seen": 2793472000 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015616161616161618, + "loss": 2.6801, + "theoretical_loss": 3.3391521873607015, + "tokens_seen": 2793537536 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015614141414141414, + "loss": 2.1577, + "theoretical_loss": 3.3391460768378876, + "tokens_seen": 2793603072 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015612121212121215, + "loss": 2.7098, + "theoretical_loss": 3.3391399664985575, + "tokens_seen": 2793668608 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1573597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3108718395233154, + "objective/train/theoretical_loss": 3.339133856342701, + "objective/train/tokens_used": 1152593376, + "theoretical_loss": 3.339133856342701, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001561010101010101, + "loss": 2.2823, + "theoretical_loss": 3.339133856342701, + "tokens_seen": 2793734144 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015608080808080806, + "loss": 2.453, + "theoretical_loss": 3.3391277463703086, + "tokens_seen": 2793799680 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015606060606060607, + "loss": 2.59, + "theoretical_loss": 3.3391216365813703, + "tokens_seen": 2793865216 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015604040404040403, + "loss": 2.4623, + "theoretical_loss": 3.339115526975876, + "tokens_seen": 2793930752 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015602020202020204, + "loss": 2.5955, + "theoretical_loss": 3.339109417553816, + "tokens_seen": 2793996288 + }, + { + "epoch": 0.69, + "learning_rate": 0.000156, + "loss": 2.5831, + "theoretical_loss": 3.339103308315181, + "tokens_seen": 2794061824 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015597979797979798, + "loss": 2.2763, + "theoretical_loss": 3.3390971992599607, + "tokens_seen": 2794127360 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015595959595959597, + "loss": 2.434, + "theoretical_loss": 3.3390910903881457, + "tokens_seen": 2794192896 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015593939393939395, + "loss": 2.6539, + "theoretical_loss": 3.3390849816997257, + "tokens_seen": 2794258432 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015591919191919193, + "loss": 2.5901, + "theoretical_loss": 3.339078873194691, + "tokens_seen": 2794323968 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001558989898989899, + "loss": 2.533, + "theoretical_loss": 3.339072764873032, + "tokens_seen": 2794389504 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015587878787878787, + "loss": 2.5934, + "theoretical_loss": 3.3390666567347385, + "tokens_seen": 2794455040 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015585858585858586, + "loss": 2.6528, + "theoretical_loss": 3.3390605487798015, + "tokens_seen": 2794520576 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015583838383838384, + "loss": 2.4082, + "theoretical_loss": 3.3390544410082104, + "tokens_seen": 2794586112 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015581818181818183, + "loss": 2.527, + "theoretical_loss": 3.3390483334199557, + "tokens_seen": 2794651648 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001557979797979798, + "loss": 2.5573, + "theoretical_loss": 3.3390422260150276, + "tokens_seen": 2794717184 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015577777777777777, + "loss": 2.4923, + "theoretical_loss": 3.3390361187934166, + "tokens_seen": 2794782720 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015575757575757578, + "loss": 2.5447, + "theoretical_loss": 3.339030011755112, + "tokens_seen": 2794848256 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015573737373737373, + "loss": 2.4894, + "theoretical_loss": 3.339023904900105, + "tokens_seen": 2794913792 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015571717171717172, + "loss": 2.627, + "theoretical_loss": 3.3390177982283853, + "tokens_seen": 2794979328 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001556969696969697, + "loss": 2.5233, + "theoretical_loss": 3.339011691739943, + "tokens_seen": 2795044864 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015567676767676766, + "loss": 2.3913, + "theoretical_loss": 3.339005585434769, + "tokens_seen": 2795110400 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015565656565656567, + "loss": 2.7648, + "theoretical_loss": 3.3389994793128523, + "tokens_seen": 2795175936 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015563636363636363, + "loss": 2.4846, + "theoretical_loss": 3.3389933733741843, + "tokens_seen": 2795241472 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015561616161616164, + "loss": 2.3609, + "theoretical_loss": 3.3389872676187546, + "tokens_seen": 2795307008 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1574190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9211652278900146, + "objective/train/theoretical_loss": 3.3389811620465535, + "objective/train/tokens_used": 1154231776, + "theoretical_loss": 3.3389811620465535, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001555959595959596, + "loss": 2.8569, + "theoretical_loss": 3.3389811620465535, + "tokens_seen": 2795372544 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001555757575757576, + "loss": 2.5486, + "theoretical_loss": 3.338975056657571, + "tokens_seen": 2795438080 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015555555555555556, + "loss": 2.5901, + "theoretical_loss": 3.3389689514517977, + "tokens_seen": 2795503616 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015553535353535352, + "loss": 2.5084, + "theoretical_loss": 3.3389628464292236, + "tokens_seen": 2795569152 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015551515151515153, + "loss": 2.4885, + "theoretical_loss": 3.338956741589839, + "tokens_seen": 2795634688 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554949494949495, + "loss": 2.4149, + "theoretical_loss": 3.338950636933634, + "tokens_seen": 2795700224 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001554747474747475, + "loss": 2.5143, + "theoretical_loss": 3.338944532460599, + "tokens_seen": 2795765760 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015545454545454546, + "loss": 2.451, + "theoretical_loss": 3.3389384281707235, + "tokens_seen": 2795831296 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015543434343434344, + "loss": 2.4662, + "theoretical_loss": 3.338932324063999, + "tokens_seen": 2795896832 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015541414141414142, + "loss": 2.2956, + "theoretical_loss": 3.338926220140414, + "tokens_seen": 2795962368 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015539393939393938, + "loss": 2.4868, + "theoretical_loss": 3.3389201163999607, + "tokens_seen": 2796027904 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001553737373737374, + "loss": 2.5857, + "theoretical_loss": 3.3389140128426282, + "tokens_seen": 2796093440 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015535353535353535, + "loss": 2.7156, + "theoretical_loss": 3.338907909468406, + "tokens_seen": 2796158976 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015533333333333333, + "loss": 2.3562, + "theoretical_loss": 3.338901806277286, + "tokens_seen": 2796224512 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015531313131313132, + "loss": 2.5607, + "theoretical_loss": 3.3388957032692574, + "tokens_seen": 2796290048 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552929292929293, + "loss": 2.7154, + "theoretical_loss": 3.3388896004443107, + "tokens_seen": 2796355584 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015527272727272728, + "loss": 2.6242, + "theoretical_loss": 3.3388834978024358, + "tokens_seen": 2796421120 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015525252525252527, + "loss": 2.4526, + "theoretical_loss": 3.338877395343623, + "tokens_seen": 2796486656 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015523232323232323, + "loss": 2.696, + "theoretical_loss": 3.3388712930678626, + "tokens_seen": 2796552192 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001552121212121212, + "loss": 2.8292, + "theoretical_loss": 3.338865190975145, + "tokens_seen": 2796617728 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001551919191919192, + "loss": 2.705, + "theoretical_loss": 3.33885908906546, + "tokens_seen": 2796683264 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015517171717171718, + "loss": 2.3934, + "theoretical_loss": 3.3388529873387984, + "tokens_seen": 2796748800 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015515151515151516, + "loss": 2.5348, + "theoretical_loss": 3.33884688579515, + "tokens_seen": 2796814336 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015513131313131312, + "loss": 2.607, + "theoretical_loss": 3.338840784434505, + "tokens_seen": 2796879872 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015511111111111113, + "loss": 2.5185, + "theoretical_loss": 3.338834683256854, + "tokens_seen": 2796945408 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1575609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.148301601409912, + "objective/train/theoretical_loss": 3.338828582262187, + "objective/train/tokens_used": 1155870176, + "theoretical_loss": 3.338828582262187, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015509090909090909, + "loss": 2.4606, + "theoretical_loss": 3.338828582262187, + "tokens_seen": 2797010944 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001550707070707071, + "loss": 2.4329, + "theoretical_loss": 3.338822481450494, + "tokens_seen": 2797076480 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015505050505050505, + "loss": 2.517, + "theoretical_loss": 3.3388163808217657, + "tokens_seen": 2797142016 + }, + { + "epoch": 0.69, + "learning_rate": 0.000155030303030303, + "loss": 2.768, + "theoretical_loss": 3.3388102803759914, + "tokens_seen": 2797207552 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015501010101010102, + "loss": 2.6599, + "theoretical_loss": 3.3388041801131627, + "tokens_seen": 2797273088 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015498989898989898, + "loss": 2.5741, + "theoretical_loss": 3.3387980800332686, + "tokens_seen": 2797338624 + }, + { + "epoch": 0.69, + "learning_rate": 0.000154969696969697, + "loss": 2.4949, + "theoretical_loss": 3.3387919801363, + "tokens_seen": 2797404160 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015494949494949495, + "loss": 2.7644, + "theoretical_loss": 3.338785880422247, + "tokens_seen": 2797469696 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015492929292929293, + "loss": 2.6725, + "theoretical_loss": 3.3387797808911, + "tokens_seen": 2797535232 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015490909090909091, + "loss": 2.4148, + "theoretical_loss": 3.338773681542849, + "tokens_seen": 2797600768 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548888888888889, + "loss": 2.3608, + "theoretical_loss": 3.3387675823774843, + "tokens_seen": 2797666304 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015486868686868688, + "loss": 2.5062, + "theoretical_loss": 3.338761483394996, + "tokens_seen": 2797731840 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015484848484848484, + "loss": 2.5811, + "theoretical_loss": 3.338755384595374, + "tokens_seen": 2797797376 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015482828282828282, + "loss": 2.6596, + "theoretical_loss": 3.3387492859786096, + "tokens_seen": 2797862912 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001548080808080808, + "loss": 2.5692, + "theoretical_loss": 3.338743187544692, + "tokens_seen": 2797928448 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547878787878788, + "loss": 2.5659, + "theoretical_loss": 3.338737089293612, + "tokens_seen": 2797993984 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015476767676767677, + "loss": 2.4729, + "theoretical_loss": 3.3387309912253595, + "tokens_seen": 2798059520 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015474747474747476, + "loss": 2.5558, + "theoretical_loss": 3.338724893339925, + "tokens_seen": 2798125056 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015472727272727274, + "loss": 2.6248, + "theoretical_loss": 3.338718795637299, + "tokens_seen": 2798190592 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001547070707070707, + "loss": 2.5112, + "theoretical_loss": 3.3387126981174706, + "tokens_seen": 2798256128 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015468686868686868, + "loss": 2.5179, + "theoretical_loss": 3.338706600780432, + "tokens_seen": 2798321664 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015466666666666667, + "loss": 2.6158, + "theoretical_loss": 3.338700503626171, + "tokens_seen": 2798387200 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015464646464646465, + "loss": 2.4932, + "theoretical_loss": 3.3386944066546795, + "tokens_seen": 2798452736 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015462626262626264, + "loss": 2.4813, + "theoretical_loss": 3.3386883098659474, + "tokens_seen": 2798518272 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015460606060606062, + "loss": 2.5223, + "theoretical_loss": 3.338682213259965, + "tokens_seen": 2798583808 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1576275, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6464033126831055, + "objective/train/theoretical_loss": 3.3386761168367225, + "objective/train/tokens_used": 1157508576, + "theoretical_loss": 3.3386761168367225, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015458585858585858, + "loss": 2.6445, + "theoretical_loss": 3.3386761168367225, + "tokens_seen": 2798649344 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545656565656566, + "loss": 2.4089, + "theoretical_loss": 3.33867002059621, + "tokens_seen": 2798714880 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015454545454545454, + "loss": 2.3676, + "theoretical_loss": 3.3386639245384178, + "tokens_seen": 2798780416 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015452525252525253, + "loss": 2.6981, + "theoretical_loss": 3.3386578286633357, + "tokens_seen": 2798845952 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001545050505050505, + "loss": 2.7485, + "theoretical_loss": 3.338651732970955, + "tokens_seen": 2798911488 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015448484848484847, + "loss": 2.4126, + "theoretical_loss": 3.338645637461265, + "tokens_seen": 2798977024 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015446464646464648, + "loss": 2.38, + "theoretical_loss": 3.3386395421342567, + "tokens_seen": 2799042560 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015444444444444444, + "loss": 2.817, + "theoretical_loss": 3.3386334469899195, + "tokens_seen": 2799108096 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015442424242424245, + "loss": 2.425, + "theoretical_loss": 3.3386273520282446, + "tokens_seen": 2799173632 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001544040404040404, + "loss": 2.6213, + "theoretical_loss": 3.3386212572492213, + "tokens_seen": 2799239168 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001543838383838384, + "loss": 2.4848, + "theoretical_loss": 3.3386151626528404, + "tokens_seen": 2799304704 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015436363636363637, + "loss": 2.5088, + "theoretical_loss": 3.338609068239092, + "tokens_seen": 2799370240 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015434343434343433, + "loss": 2.5742, + "theoretical_loss": 3.3386029740079666, + "tokens_seen": 2799435776 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015432323232323234, + "loss": 2.5745, + "theoretical_loss": 3.338596879959454, + "tokens_seen": 2799501312 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001543030303030303, + "loss": 2.4842, + "theoretical_loss": 3.338590786093545, + "tokens_seen": 2799566848 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015428282828282828, + "loss": 2.5446, + "theoretical_loss": 3.338584692410229, + "tokens_seen": 2799632384 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015426262626262627, + "loss": 2.5424, + "theoretical_loss": 3.338578598909497, + "tokens_seen": 2799697920 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015424242424242425, + "loss": 2.4031, + "theoretical_loss": 3.3385725055913396, + "tokens_seen": 2799763456 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015422222222222223, + "loss": 2.6102, + "theoretical_loss": 3.338566412455746, + "tokens_seen": 2799828992 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015420202020202022, + "loss": 2.49, + "theoretical_loss": 3.338560319502707, + "tokens_seen": 2799894528 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015418181818181817, + "loss": 2.4731, + "theoretical_loss": 3.3385542267322132, + "tokens_seen": 2799960064 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015416161616161616, + "loss": 2.7604, + "theoretical_loss": 3.338548134144254, + "tokens_seen": 2800025600 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015414141414141414, + "loss": 2.4192, + "theoretical_loss": 3.3385420417388207, + "tokens_seen": 2800091136 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015412121212121213, + "loss": 2.7011, + "theoretical_loss": 3.3385359495159026, + "tokens_seen": 2800156672 + }, + { + "epoch": 0.69, + "learning_rate": 0.0001541010101010101, + "loss": 2.5241, + "theoretical_loss": 3.3385298574754905, + "tokens_seen": 2800222208 + }, + { + "epoch": 0.69, + "objective/train/docs_used": 1577506, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4385788440704346, + "objective/train/theoretical_loss": 3.3385237656175746, + "objective/train/tokens_used": 1159146976, + "theoretical_loss": 3.3385237656175746, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.69, + "learning_rate": 0.00015408080808080807, + "loss": 2.4917, + "theoretical_loss": 3.3385237656175746, + "tokens_seen": 2800287744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015406060606060608, + "loss": 2.3852, + "theoretical_loss": 3.338517673942145, + "tokens_seen": 2800353280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015404040404040403, + "loss": 2.5107, + "theoretical_loss": 3.3385115824491924, + "tokens_seen": 2800418816 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015402020202020205, + "loss": 2.5381, + "theoretical_loss": 3.338505491138706, + "tokens_seen": 2800484352 + }, + { + "epoch": 0.7, + "learning_rate": 0.000154, + "loss": 2.4874, + "theoretical_loss": 3.3384994000106776, + "tokens_seen": 2800549888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015397979797979796, + "loss": 2.605, + "theoretical_loss": 3.3384933090650963, + "tokens_seen": 2800615424 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015395959595959597, + "loss": 2.4025, + "theoretical_loss": 3.3384872183019527, + "tokens_seen": 2800680960 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015393939393939393, + "loss": 2.6463, + "theoretical_loss": 3.338481127721237, + "tokens_seen": 2800746496 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015391919191919194, + "loss": 2.5565, + "theoretical_loss": 3.3384750373229397, + "tokens_seen": 2800812032 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538989898989899, + "loss": 2.5961, + "theoretical_loss": 3.3384689471070508, + "tokens_seen": 2800877568 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001538787878787879, + "loss": 2.4876, + "theoretical_loss": 3.338462857073561, + "tokens_seen": 2800943104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015385858585858586, + "loss": 2.5031, + "theoretical_loss": 3.3384567672224597, + "tokens_seen": 2801008640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015383838383838382, + "loss": 2.5497, + "theoretical_loss": 3.3384506775537384, + "tokens_seen": 2801074176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015381818181818183, + "loss": 2.5264, + "theoretical_loss": 3.3384445880673863, + "tokens_seen": 2801139712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537979797979798, + "loss": 2.57, + "theoretical_loss": 3.338438498763394, + "tokens_seen": 2801205248 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001537777777777778, + "loss": 2.6569, + "theoretical_loss": 3.3384324096417517, + "tokens_seen": 2801270784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015375757575757576, + "loss": 2.4039, + "theoretical_loss": 3.3384263207024505, + "tokens_seen": 2801336320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015373737373737374, + "loss": 2.6931, + "theoretical_loss": 3.3384202319454794, + "tokens_seen": 2801401856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015371717171717172, + "loss": 2.7557, + "theoretical_loss": 3.3384141433708296, + "tokens_seen": 2801467392 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001536969696969697, + "loss": 2.398, + "theoretical_loss": 3.338408054978491, + "tokens_seen": 2801532928 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001536767676767677, + "loss": 2.7071, + "theoretical_loss": 3.338401966768454, + "tokens_seen": 2801598464 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015365656565656565, + "loss": 2.6638, + "theoretical_loss": 3.3383958787407084, + "tokens_seen": 2801664000 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015363636363636363, + "loss": 2.6073, + "theoretical_loss": 3.338389790895245, + "tokens_seen": 2801729536 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015361616161616162, + "loss": 2.6253, + "theoretical_loss": 3.3383837032320542, + "tokens_seen": 2801795072 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001535959595959596, + "loss": 2.4915, + "theoretical_loss": 3.338377615751126, + "tokens_seen": 2801860608 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1578159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6187901496887207, + "objective/train/theoretical_loss": 3.3383715284524507, + "objective/train/tokens_used": 1160785376, + "theoretical_loss": 3.3383715284524507, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015357575757575758, + "loss": 2.487, + "theoretical_loss": 3.3383715284524507, + "tokens_seen": 2801926144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015355555555555557, + "loss": 2.8057, + "theoretical_loss": 3.3383654413360184, + "tokens_seen": 2801991680 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015353535353535353, + "loss": 2.5602, + "theoretical_loss": 3.3383593544018195, + "tokens_seen": 2802057216 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015351515151515154, + "loss": 2.6666, + "theoretical_loss": 3.3383532676498446, + "tokens_seen": 2802122752 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001534949494949495, + "loss": 2.8318, + "theoretical_loss": 3.338347181080084, + "tokens_seen": 2802188288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015347474747474748, + "loss": 2.3178, + "theoretical_loss": 3.3383410946925274, + "tokens_seen": 2802253824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015345454545454546, + "loss": 2.5818, + "theoretical_loss": 3.3383350084871655, + "tokens_seen": 2802319360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015343434343434342, + "loss": 2.5957, + "theoretical_loss": 3.3383289224639885, + "tokens_seen": 2802384896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015341414141414143, + "loss": 2.3686, + "theoretical_loss": 3.3383228366229867, + "tokens_seen": 2802450432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015339393939393939, + "loss": 2.5407, + "theoretical_loss": 3.3383167509641503, + "tokens_seen": 2802515968 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001533737373737374, + "loss": 2.426, + "theoretical_loss": 3.3383106654874695, + "tokens_seen": 2802581504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015335353535353535, + "loss": 2.602, + "theoretical_loss": 3.338304580192935, + "tokens_seen": 2802647040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015333333333333334, + "loss": 2.8068, + "theoretical_loss": 3.3382984950805366, + "tokens_seen": 2802712576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015331313131313132, + "loss": 2.5689, + "theoretical_loss": 3.338292410150265, + "tokens_seen": 2802778112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015329292929292928, + "loss": 2.6898, + "theoretical_loss": 3.3382863254021107, + "tokens_seen": 2802843648 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001532727272727273, + "loss": 2.6027, + "theoretical_loss": 3.338280240836063, + "tokens_seen": 2802909184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015325252525252525, + "loss": 2.4398, + "theoretical_loss": 3.338274156452113, + "tokens_seen": 2802974720 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015323232323232323, + "loss": 2.555, + "theoretical_loss": 3.3382680722502505, + "tokens_seen": 2803040256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015321212121212121, + "loss": 2.4176, + "theoretical_loss": 3.3382619882304665, + "tokens_seen": 2803105792 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531919191919192, + "loss": 2.7301, + "theoretical_loss": 3.3382559043927507, + "tokens_seen": 2803171328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015317171717171718, + "loss": 2.5085, + "theoretical_loss": 3.3382498207370936, + "tokens_seen": 2803236864 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015315151515151514, + "loss": 2.5058, + "theoretical_loss": 3.3382437372634857, + "tokens_seen": 2803302400 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015313131313131312, + "loss": 2.3709, + "theoretical_loss": 3.338237653971917, + "tokens_seen": 2803367936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001531111111111111, + "loss": 2.6429, + "theoretical_loss": 3.3382315708623778, + "tokens_seen": 2803433472 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001530909090909091, + "loss": 2.604, + "theoretical_loss": 3.3382254879348583, + "tokens_seen": 2803499008 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6288652420043945, + "objective/train/theoretical_loss": 3.3382194051893492, + "objective/train/tokens_used": 1162423776, + "theoretical_loss": 3.3382194051893492, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015307070707070708, + "loss": 2.7413, + "theoretical_loss": 3.3382194051893492, + "tokens_seen": 2803564544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015305050505050506, + "loss": 2.6304, + "theoretical_loss": 3.3382133226258404, + "tokens_seen": 2803630080 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015303030303030302, + "loss": 2.7027, + "theoretical_loss": 3.3382072402443224, + "tokens_seen": 2803695616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015301010101010103, + "loss": 2.5193, + "theoretical_loss": 3.3382011580447855, + "tokens_seen": 2803761152 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015298989898989898, + "loss": 2.5306, + "theoretical_loss": 3.33819507602722, + "tokens_seen": 2803826688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015296969696969697, + "loss": 2.643, + "theoretical_loss": 3.338188994191616, + "tokens_seen": 2803892224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015294949494949495, + "loss": 2.3785, + "theoretical_loss": 3.338182912537964, + "tokens_seen": 2803957760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015292929292929294, + "loss": 2.4651, + "theoretical_loss": 3.3381768310662543, + "tokens_seen": 2804023296 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015290909090909092, + "loss": 2.5847, + "theoretical_loss": 3.338170749776477, + "tokens_seen": 2804088832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015288888888888888, + "loss": 2.6304, + "theoretical_loss": 3.338164668668623, + "tokens_seen": 2804154368 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001528686868686869, + "loss": 2.6196, + "theoretical_loss": 3.338158587742682, + "tokens_seen": 2804219904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015284848484848484, + "loss": 2.6977, + "theoretical_loss": 3.338152506998645, + "tokens_seen": 2804285440 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015282828282828286, + "loss": 2.6298, + "theoretical_loss": 3.338146426436501, + "tokens_seen": 2804350976 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001528080808080808, + "loss": 2.509, + "theoretical_loss": 3.3381403460562415, + "tokens_seen": 2804416512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015278787878787877, + "loss": 2.6288, + "theoretical_loss": 3.3381342658578563, + "tokens_seen": 2804482048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015276767676767678, + "loss": 2.6887, + "theoretical_loss": 3.338128185841336, + "tokens_seen": 2804547584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015274747474747474, + "loss": 2.588, + "theoretical_loss": 3.3381221060066704, + "tokens_seen": 2804613120 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015272727272727275, + "loss": 2.8478, + "theoretical_loss": 3.3381160263538505, + "tokens_seen": 2804678656 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001527070707070707, + "loss": 2.8422, + "theoretical_loss": 3.338109946882866, + "tokens_seen": 2804744192 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001526868686868687, + "loss": 2.5277, + "theoretical_loss": 3.338103867593708, + "tokens_seen": 2804809728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015266666666666667, + "loss": 2.378, + "theoretical_loss": 3.338097788486366, + "tokens_seen": 2804875264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015264646464646466, + "loss": 2.589, + "theoretical_loss": 3.33809170956083, + "tokens_seen": 2804940800 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015262626262626264, + "loss": 2.6463, + "theoretical_loss": 3.3380856308170914, + "tokens_seen": 2805006336 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001526060606060606, + "loss": 2.7025, + "theoretical_loss": 3.3380795522551403, + "tokens_seen": 2805071872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015258585858585858, + "loss": 2.4681, + "theoretical_loss": 3.3380734738749664, + "tokens_seen": 2805137408 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1579013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898916482925415, + "objective/train/theoretical_loss": 3.3380673956765605, + "objective/train/tokens_used": 1164062176, + "theoretical_loss": 3.3380673956765605, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015256565656565657, + "loss": 2.6715, + "theoretical_loss": 3.3380673956765605, + "tokens_seen": 2805202944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015254545454545455, + "loss": 2.5326, + "theoretical_loss": 3.338061317659913, + "tokens_seen": 2805268480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015252525252525253, + "loss": 2.4046, + "theoretical_loss": 3.338055239825014, + "tokens_seen": 2805334016 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015250505050505052, + "loss": 2.683, + "theoretical_loss": 3.3380491621718535, + "tokens_seen": 2805399552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015248484848484847, + "loss": 2.5331, + "theoretical_loss": 3.3380430847004225, + "tokens_seen": 2805465088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015246464646464649, + "loss": 2.5174, + "theoretical_loss": 3.3380370074107106, + "tokens_seen": 2805530624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015244444444444444, + "loss": 2.5676, + "theoretical_loss": 3.3380309303027085, + "tokens_seen": 2805596160 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015242424242424243, + "loss": 2.7573, + "theoretical_loss": 3.3380248533764068, + "tokens_seen": 2805661696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001524040404040404, + "loss": 2.6111, + "theoretical_loss": 3.3380187766317957, + "tokens_seen": 2805727232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015238383838383837, + "loss": 2.5672, + "theoretical_loss": 3.338012700068865, + "tokens_seen": 2805792768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015236363636363638, + "loss": 2.7674, + "theoretical_loss": 3.3380066236876056, + "tokens_seen": 2805858304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015234343434343434, + "loss": 2.733, + "theoretical_loss": 3.3380005474880075, + "tokens_seen": 2805923840 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015232323232323235, + "loss": 2.8022, + "theoretical_loss": 3.337994471470061, + "tokens_seen": 2805989376 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001523030303030303, + "loss": 2.7872, + "theoretical_loss": 3.3379883956337566, + "tokens_seen": 2806054912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015228282828282826, + "loss": 2.7457, + "theoretical_loss": 3.337982319979085, + "tokens_seen": 2806120448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015226262626262627, + "loss": 2.554, + "theoretical_loss": 3.3379762445060353, + "tokens_seen": 2806185984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015224242424242423, + "loss": 2.6639, + "theoretical_loss": 3.3379701692145995, + "tokens_seen": 2806251520 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015222222222222224, + "loss": 2.5465, + "theoretical_loss": 3.3379640941047666, + "tokens_seen": 2806317056 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001522020202020202, + "loss": 2.699, + "theoretical_loss": 3.3379580191765275, + "tokens_seen": 2806382592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015218181818181818, + "loss": 2.7183, + "theoretical_loss": 3.3379519444298724, + "tokens_seen": 2806448128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015216161616161616, + "loss": 2.6417, + "theoretical_loss": 3.337945869864792, + "tokens_seen": 2806513664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015214141414141415, + "loss": 2.6371, + "theoretical_loss": 3.337939795481276, + "tokens_seen": 2806579200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015212121212121213, + "loss": 2.5801, + "theoretical_loss": 3.337933721279315, + "tokens_seen": 2806644736 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001521010101010101, + "loss": 2.4272, + "theoretical_loss": 3.3379276472589, + "tokens_seen": 2806710272 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001520808080808081, + "loss": 2.6707, + "theoretical_loss": 3.33792157342002, + "tokens_seen": 2806775808 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1579588, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5968844890594482, + "objective/train/theoretical_loss": 3.337915499762666, + "objective/train/tokens_used": 1165700576, + "theoretical_loss": 3.337915499762666, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015206060606060606, + "loss": 2.6082, + "theoretical_loss": 3.337915499762666, + "tokens_seen": 2806841344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015204040404040404, + "loss": 2.7587, + "theoretical_loss": 3.3379094262868287, + "tokens_seen": 2806906880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015202020202020202, + "loss": 2.5028, + "theoretical_loss": 3.337903352992498, + "tokens_seen": 2806972416 + }, + { + "epoch": 0.7, + "learning_rate": 0.000152, + "loss": 2.8005, + "theoretical_loss": 3.3378972798796647, + "tokens_seen": 2807037952 + }, + { + "epoch": 0.7, + "learning_rate": 0.000151979797979798, + "loss": 2.6268, + "theoretical_loss": 3.3378912069483184, + "tokens_seen": 2807103488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015195959595959598, + "loss": 2.7138, + "theoretical_loss": 3.33788513419845, + "tokens_seen": 2807169024 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015193939393939393, + "loss": 2.7206, + "theoretical_loss": 3.3378790616300495, + "tokens_seen": 2807234560 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015191919191919192, + "loss": 2.6912, + "theoretical_loss": 3.337872989243108, + "tokens_seen": 2807300096 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001518989898989899, + "loss": 2.6336, + "theoretical_loss": 3.3378669170376147, + "tokens_seen": 2807365632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015187878787878789, + "loss": 2.7745, + "theoretical_loss": 3.3378608450135605, + "tokens_seen": 2807431168 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015185858585858587, + "loss": 2.9233, + "theoretical_loss": 3.3378547731709363, + "tokens_seen": 2807496704 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015183838383838383, + "loss": 2.4799, + "theoretical_loss": 3.3378487015097313, + "tokens_seen": 2807562240 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015181818181818184, + "loss": 2.4839, + "theoretical_loss": 3.3378426300299364, + "tokens_seen": 2807627776 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001517979797979798, + "loss": 2.5951, + "theoretical_loss": 3.337836558731542, + "tokens_seen": 2807693312 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001517777777777778, + "loss": 2.7283, + "theoretical_loss": 3.337830487614539, + "tokens_seen": 2807758848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015175757575757576, + "loss": 2.7034, + "theoretical_loss": 3.3378244166789166, + "tokens_seen": 2807824384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015173737373737372, + "loss": 2.6223, + "theoretical_loss": 3.337818345924666, + "tokens_seen": 2807889920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015171717171717173, + "loss": 2.6213, + "theoretical_loss": 3.3378122753517774, + "tokens_seen": 2807955456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516969696969697, + "loss": 2.5124, + "theoretical_loss": 3.337806204960241, + "tokens_seen": 2808020992 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001516767676767677, + "loss": 2.4272, + "theoretical_loss": 3.3378001347500468, + "tokens_seen": 2808086528 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015165656565656565, + "loss": 2.4474, + "theoretical_loss": 3.3377940647211854, + "tokens_seen": 2808152064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015163636363636364, + "loss": 2.4837, + "theoretical_loss": 3.337787994873648, + "tokens_seen": 2808217600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015161616161616162, + "loss": 2.5901, + "theoretical_loss": 3.3377819252074237, + "tokens_seen": 2808283136 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515959595959596, + "loss": 2.4835, + "theoretical_loss": 3.3377758557225032, + "tokens_seen": 2808348672 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001515757575757576, + "loss": 2.556, + "theoretical_loss": 3.3377697864188773, + "tokens_seen": 2808414208 + }, + { + "debugging/Self-BLEU-5": 0.20303494455588592, + "debugging/distinct-1-grams": 0.7960725957049486, + "debugging/distinct-2-grams": 0.9561016342598115, + "debugging/entropy-1-grams": 5.007351944943723, + "debugging/entropy-2-grams": 5.529170180274562, + "debugging/length": 479.0, + "debugging/num_segments": 4, + "epoch": 0.7, + "objective/train/docs_used": 1580206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.656341314315796, + "objective/train/theoretical_loss": 3.337763717296536, + "objective/train/tokens_used": 1167338976, + "theoretical_loss": 3.337763717296536, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015155555555555555, + "loss": 2.5916, + "theoretical_loss": 3.337763717296536, + "tokens_seen": 2808479744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015153535353535353, + "loss": 2.5593, + "theoretical_loss": 3.33775764835547, + "tokens_seen": 2808545280 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015151515151515152, + "loss": 2.7176, + "theoretical_loss": 3.337751579595669, + "tokens_seen": 2808610816 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514949494949495, + "loss": 2.5631, + "theoretical_loss": 3.3377455110171246, + "tokens_seen": 2808676352 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015147474747474748, + "loss": 2.9219, + "theoretical_loss": 3.3377394426198252, + "tokens_seen": 2808741888 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015145454545454547, + "loss": 2.4751, + "theoretical_loss": 3.337733374403763, + "tokens_seen": 2808807424 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015143434343434342, + "loss": 2.6036, + "theoretical_loss": 3.3377273063689277, + "tokens_seen": 2808872960 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001514141414141414, + "loss": 2.5344, + "theoretical_loss": 3.337721238515309, + "tokens_seen": 2808938496 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001513939393939394, + "loss": 2.5939, + "theoretical_loss": 3.3377151708428983, + "tokens_seen": 2809004032 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015137373737373738, + "loss": 2.4005, + "theoretical_loss": 3.337709103351685, + "tokens_seen": 2809069568 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015135353535353536, + "loss": 2.5672, + "theoretical_loss": 3.33770303604166, + "tokens_seen": 2809135104 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015133333333333332, + "loss": 2.4688, + "theoretical_loss": 3.337696968912814, + "tokens_seen": 2809200640 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015131313131313133, + "loss": 2.4138, + "theoretical_loss": 3.337690901965137, + "tokens_seen": 2809266176 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015129292929292928, + "loss": 2.6211, + "theoretical_loss": 3.3376848351986195, + "tokens_seen": 2809331712 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001512727272727273, + "loss": 2.6742, + "theoretical_loss": 3.3376787686132516, + "tokens_seen": 2809397248 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015125252525252525, + "loss": 2.6111, + "theoretical_loss": 3.337672702209024, + "tokens_seen": 2809462784 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015123232323232324, + "loss": 2.7811, + "theoretical_loss": 3.337666635985926, + "tokens_seen": 2809528320 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015121212121212122, + "loss": 2.6527, + "theoretical_loss": 3.3376605699439494, + "tokens_seen": 2809593856 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015119191919191918, + "loss": 2.3437, + "theoretical_loss": 3.3376545040830843, + "tokens_seen": 2809659392 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511717171717172, + "loss": 2.4692, + "theoretical_loss": 3.3376484384033205, + "tokens_seen": 2809724928 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015115151515151515, + "loss": 2.8781, + "theoretical_loss": 3.3376423729046483, + "tokens_seen": 2809790464 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015113131313131316, + "loss": 2.5272, + "theoretical_loss": 3.3376363075870588, + "tokens_seen": 2809856000 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001511111111111111, + "loss": 2.7842, + "theoretical_loss": 3.3376302424505417, + "tokens_seen": 2809921536 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001510909090909091, + "loss": 2.5298, + "theoretical_loss": 3.337624177495088, + "tokens_seen": 2809987072 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015107070707070708, + "loss": 2.6166, + "theoretical_loss": 3.337618112720687, + "tokens_seen": 2810052608 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1581474, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.520904541015625, + "objective/train/theoretical_loss": 3.3376120481273306, + "objective/train/tokens_used": 1168977376, + "theoretical_loss": 3.3376120481273306, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015105050505050504, + "loss": 2.7038, + "theoretical_loss": 3.3376120481273306, + "tokens_seen": 2810118144 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015103030303030305, + "loss": 2.4379, + "theoretical_loss": 3.337605983715008, + "tokens_seen": 2810183680 + }, + { + "epoch": 0.7, + "learning_rate": 0.000151010101010101, + "loss": 2.5845, + "theoretical_loss": 3.3375999194837096, + "tokens_seen": 2810249216 + }, + { + "epoch": 0.7, + "learning_rate": 0.000150989898989899, + "loss": 2.6672, + "theoretical_loss": 3.337593855433427, + "tokens_seen": 2810314752 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015096969696969697, + "loss": 2.8704, + "theoretical_loss": 3.337587791564149, + "tokens_seen": 2810380288 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015094949494949496, + "loss": 2.5787, + "theoretical_loss": 3.3375817278758664, + "tokens_seen": 2810445824 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015092929292929294, + "loss": 2.6741, + "theoretical_loss": 3.3375756643685706, + "tokens_seen": 2810511360 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015090909090909093, + "loss": 2.4335, + "theoretical_loss": 3.337569601042251, + "tokens_seen": 2810576896 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015088888888888888, + "loss": 2.6371, + "theoretical_loss": 3.3375635378968975, + "tokens_seen": 2810642432 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015086868686868687, + "loss": 2.5561, + "theoretical_loss": 3.3375574749325017, + "tokens_seen": 2810707968 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015084848484848485, + "loss": 2.6135, + "theoretical_loss": 3.3375514121490535, + "tokens_seen": 2810773504 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015082828282828283, + "loss": 2.7123, + "theoretical_loss": 3.3375453495465432, + "tokens_seen": 2810839040 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015080808080808082, + "loss": 2.5146, + "theoretical_loss": 3.3375392871249616, + "tokens_seen": 2810904576 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015078787878787877, + "loss": 2.5434, + "theoretical_loss": 3.337533224884298, + "tokens_seen": 2810970112 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015076767676767679, + "loss": 2.6544, + "theoretical_loss": 3.3375271628245438, + "tokens_seen": 2811035648 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015074747474747474, + "loss": 2.7965, + "theoretical_loss": 3.337521100945689, + "tokens_seen": 2811101184 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015072727272727275, + "loss": 2.7446, + "theoretical_loss": 3.337515039247724, + "tokens_seen": 2811166720 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001507070707070707, + "loss": 2.499, + "theoretical_loss": 3.3375089777306397, + "tokens_seen": 2811232256 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015068686868686867, + "loss": 2.7329, + "theoretical_loss": 3.3375029163944254, + "tokens_seen": 2811297792 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015066666666666668, + "loss": 2.5805, + "theoretical_loss": 3.3374968552390722, + "tokens_seen": 2811363328 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015064646464646464, + "loss": 2.525, + "theoretical_loss": 3.337490794264571, + "tokens_seen": 2811428864 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015062626262626265, + "loss": 2.5315, + "theoretical_loss": 3.337484733470911, + "tokens_seen": 2811494400 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001506060606060606, + "loss": 2.6003, + "theoretical_loss": 3.337478672858083, + "tokens_seen": 2811559936 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001505858585858586, + "loss": 2.6633, + "theoretical_loss": 3.337472612426078, + "tokens_seen": 2811625472 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015056565656565657, + "loss": 2.5449, + "theoretical_loss": 3.337466552174886, + "tokens_seen": 2811691008 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1582095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.16965651512146, + "objective/train/theoretical_loss": 3.337460492104497, + "objective/train/tokens_used": 1170615776, + "theoretical_loss": 3.337460492104497, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015054545454545453, + "loss": 2.6496, + "theoretical_loss": 3.337460492104497, + "tokens_seen": 2811756544 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015052525252525254, + "loss": 2.6312, + "theoretical_loss": 3.337454432214902, + "tokens_seen": 2811822080 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001505050505050505, + "loss": 2.3944, + "theoretical_loss": 3.337448372506091, + "tokens_seen": 2811887616 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015048484848484848, + "loss": 2.5576, + "theoretical_loss": 3.3374423129780544, + "tokens_seen": 2811953152 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015046464646464646, + "loss": 2.5289, + "theoretical_loss": 3.337436253630783, + "tokens_seen": 2812018688 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015044444444444445, + "loss": 2.5787, + "theoretical_loss": 3.337430194464267, + "tokens_seen": 2812084224 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015042424242424243, + "loss": 2.464, + "theoretical_loss": 3.3374241354784964, + "tokens_seen": 2812149760 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015040404040404042, + "loss": 2.6889, + "theoretical_loss": 3.337418076673462, + "tokens_seen": 2812215296 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503838383838384, + "loss": 2.6114, + "theoretical_loss": 3.3374120180491538, + "tokens_seen": 2812280832 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015036363636363636, + "loss": 2.572, + "theoretical_loss": 3.3374059596055625, + "tokens_seen": 2812346368 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015034343434343434, + "loss": 2.6241, + "theoretical_loss": 3.337399901342679, + "tokens_seen": 2812411904 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015032323232323232, + "loss": 2.54, + "theoretical_loss": 3.337393843260493, + "tokens_seen": 2812477440 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001503030303030303, + "loss": 2.7404, + "theoretical_loss": 3.3373877853589953, + "tokens_seen": 2812542976 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001502828282828283, + "loss": 2.4801, + "theoretical_loss": 3.3373817276381756, + "tokens_seen": 2812608512 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015026262626262628, + "loss": 2.2577, + "theoretical_loss": 3.337375670098025, + "tokens_seen": 2812674048 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015024242424242423, + "loss": 2.7335, + "theoretical_loss": 3.337369612738534, + "tokens_seen": 2812739584 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015022222222222224, + "loss": 2.6466, + "theoretical_loss": 3.3373635555596923, + "tokens_seen": 2812805120 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001502020202020202, + "loss": 2.5616, + "theoretical_loss": 3.3373574985614907, + "tokens_seen": 2812870656 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015018181818181819, + "loss": 2.8454, + "theoretical_loss": 3.3373514417439196, + "tokens_seen": 2812936192 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015016161616161617, + "loss": 2.8032, + "theoretical_loss": 3.33734538510697, + "tokens_seen": 2813001728 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015014141414141413, + "loss": 2.5587, + "theoretical_loss": 3.337339328650631, + "tokens_seen": 2813067264 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015012121212121214, + "loss": 2.7262, + "theoretical_loss": 3.3373332723748943, + "tokens_seen": 2813132800 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001501010101010101, + "loss": 2.5172, + "theoretical_loss": 3.3373272162797494, + "tokens_seen": 2813198336 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001500808080808081, + "loss": 2.636, + "theoretical_loss": 3.337321160365187, + "tokens_seen": 2813263872 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015006060606060606, + "loss": 2.5851, + "theoretical_loss": 3.3373151046311977, + "tokens_seen": 2813329408 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1583341, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2032687664031982, + "objective/train/theoretical_loss": 3.3373090490777715, + "objective/train/tokens_used": 1172254176, + "theoretical_loss": 3.3373090490777715, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015004040404040405, + "loss": 2.6067, + "theoretical_loss": 3.3373090490777715, + "tokens_seen": 2813394944 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015002020202020203, + "loss": 2.4516, + "theoretical_loss": 3.3373029937048995, + "tokens_seen": 2813460480 + }, + { + "epoch": 0.7, + "learning_rate": 0.00015, + "loss": 2.4523, + "theoretical_loss": 3.3372969385125715, + "tokens_seen": 2813526016 + }, + { + "epoch": 0.7, + "learning_rate": 0.000149979797979798, + "loss": 2.6673, + "theoretical_loss": 3.3372908835007777, + "tokens_seen": 2813591552 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014995959595959595, + "loss": 2.6967, + "theoretical_loss": 3.3372848286695094, + "tokens_seen": 2813657088 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014993939393939394, + "loss": 2.5569, + "theoretical_loss": 3.337278774018756, + "tokens_seen": 2813722624 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014991919191919192, + "loss": 2.5527, + "theoretical_loss": 3.3372727195485092, + "tokens_seen": 2813788160 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001498989898989899, + "loss": 2.5515, + "theoretical_loss": 3.3372666652587584, + "tokens_seen": 2813853696 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001498787878787879, + "loss": 2.7088, + "theoretical_loss": 3.3372606111494942, + "tokens_seen": 2813919232 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014985858585858585, + "loss": 2.4832, + "theoretical_loss": 3.337254557220707, + "tokens_seen": 2813984768 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014983838383838383, + "loss": 2.8801, + "theoretical_loss": 3.337248503472387, + "tokens_seen": 2814050304 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014981818181818182, + "loss": 2.7171, + "theoretical_loss": 3.3372424499045255, + "tokens_seen": 2814115840 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001497979797979798, + "loss": 2.4737, + "theoretical_loss": 3.337236396517112, + "tokens_seen": 2814181376 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014977777777777778, + "loss": 2.6653, + "theoretical_loss": 3.337230343310137, + "tokens_seen": 2814246912 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014975757575757577, + "loss": 2.6946, + "theoretical_loss": 3.337224290283592, + "tokens_seen": 2814312448 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014973737373737372, + "loss": 2.5281, + "theoretical_loss": 3.3372182374374657, + "tokens_seen": 2814377984 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014971717171717174, + "loss": 2.6225, + "theoretical_loss": 3.33721218477175, + "tokens_seen": 2814443520 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001496969696969697, + "loss": 2.6007, + "theoretical_loss": 3.3372061322864344, + "tokens_seen": 2814509056 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014967676767676768, + "loss": 2.434, + "theoretical_loss": 3.3372000799815096, + "tokens_seen": 2814574592 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014965656565656566, + "loss": 2.7533, + "theoretical_loss": 3.3371940278569663, + "tokens_seen": 2814640128 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014963636363636362, + "loss": 2.5581, + "theoretical_loss": 3.3371879759127947, + "tokens_seen": 2814705664 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014961616161616163, + "loss": 2.5124, + "theoretical_loss": 3.337181924148985, + "tokens_seen": 2814771200 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014959595959595958, + "loss": 2.5624, + "theoretical_loss": 3.3371758725655285, + "tokens_seen": 2814836736 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001495757575757576, + "loss": 2.7147, + "theoretical_loss": 3.3371698211624143, + "tokens_seen": 2814902272 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014955555555555555, + "loss": 2.6937, + "theoretical_loss": 3.3371637699396337, + "tokens_seen": 2814967808 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1583792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.746033191680908, + "objective/train/theoretical_loss": 3.337157718897177, + "objective/train/tokens_used": 1173892576, + "theoretical_loss": 3.337157718897177, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014953535353535356, + "loss": 2.5862, + "theoretical_loss": 3.337157718897177, + "tokens_seen": 2815033344 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014951515151515152, + "loss": 2.5273, + "theoretical_loss": 3.3371516680350344, + "tokens_seen": 2815098880 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014949494949494948, + "loss": 2.6697, + "theoretical_loss": 3.3371456173531966, + "tokens_seen": 2815164416 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494747474747475, + "loss": 2.7765, + "theoretical_loss": 3.337139566851654, + "tokens_seen": 2815229952 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014945454545454545, + "loss": 2.5274, + "theoretical_loss": 3.337133516530397, + "tokens_seen": 2815295488 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014943434343434346, + "loss": 2.4664, + "theoretical_loss": 3.3371274663894157, + "tokens_seen": 2815361024 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001494141414141414, + "loss": 2.7009, + "theoretical_loss": 3.337121416428701, + "tokens_seen": 2815426560 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493939393939394, + "loss": 2.6525, + "theoretical_loss": 3.337115366648243, + "tokens_seen": 2815492096 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014937373737373738, + "loss": 2.7329, + "theoretical_loss": 3.3371093170480326, + "tokens_seen": 2815557632 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014935353535353537, + "loss": 2.7518, + "theoretical_loss": 3.33710326762806, + "tokens_seen": 2815623168 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014933333333333335, + "loss": 2.6719, + "theoretical_loss": 3.337097218388315, + "tokens_seen": 2815688704 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001493131313131313, + "loss": 2.7366, + "theoretical_loss": 3.3370911693287892, + "tokens_seen": 2815754240 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001492929292929293, + "loss": 2.5463, + "theoretical_loss": 3.337085120449472, + "tokens_seen": 2815819776 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014927272727272727, + "loss": 2.5752, + "theoretical_loss": 3.3370790717503542, + "tokens_seen": 2815885312 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014925252525252526, + "loss": 2.6752, + "theoretical_loss": 3.3370730232314267, + "tokens_seen": 2815950848 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014923232323232324, + "loss": 2.7825, + "theoretical_loss": 3.3370669748926796, + "tokens_seen": 2816016384 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014921212121212123, + "loss": 2.5388, + "theoretical_loss": 3.3370609267341025, + "tokens_seen": 2816081920 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014919191919191918, + "loss": 2.745, + "theoretical_loss": 3.3370548787556875, + "tokens_seen": 2816147456 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001491717171717172, + "loss": 2.6503, + "theoretical_loss": 3.3370488309574236, + "tokens_seen": 2816212992 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014915151515151515, + "loss": 2.6916, + "theoretical_loss": 3.337042783339302, + "tokens_seen": 2816278528 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014913131313131313, + "loss": 2.5294, + "theoretical_loss": 3.337036735901313, + "tokens_seen": 2816344064 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014911111111111112, + "loss": 2.3673, + "theoretical_loss": 3.337030688643447, + "tokens_seen": 2816409600 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014909090909090908, + "loss": 2.7347, + "theoretical_loss": 3.3370246415656943, + "tokens_seen": 2816475136 + }, + { + "epoch": 0.7, + "learning_rate": 0.0001490707070707071, + "loss": 2.6101, + "theoretical_loss": 3.3370185946680455, + "tokens_seen": 2816540672 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014905050505050504, + "loss": 2.7225, + "theoretical_loss": 3.3370125479504913, + "tokens_seen": 2816606208 + }, + { + "epoch": 0.7, + "objective/train/docs_used": 1585248, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.847041606903076, + "objective/train/theoretical_loss": 3.3370065014130215, + "objective/train/tokens_used": 1175530976, + "theoretical_loss": 3.3370065014130215, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.7, + "learning_rate": 0.00014903030303030305, + "loss": 2.5671, + "theoretical_loss": 3.3370065014130215, + "tokens_seen": 2816671744 + }, + { + "epoch": 0.71, + "learning_rate": 0.000149010101010101, + "loss": 2.5915, + "theoretical_loss": 3.337000455055627, + "tokens_seen": 2816737280 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014898989898989897, + "loss": 2.619, + "theoretical_loss": 3.336994408878298, + "tokens_seen": 2816802816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014896969696969698, + "loss": 2.5131, + "theoretical_loss": 3.3369883628810255, + "tokens_seen": 2816868352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014894949494949494, + "loss": 2.6921, + "theoretical_loss": 3.3369823170637996, + "tokens_seen": 2816933888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014892929292929295, + "loss": 2.5397, + "theoretical_loss": 3.3369762714266105, + "tokens_seen": 2816999424 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001489090909090909, + "loss": 2.3838, + "theoretical_loss": 3.336970225969449, + "tokens_seen": 2817064960 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001488888888888889, + "loss": 2.3376, + "theoretical_loss": 3.336964180692305, + "tokens_seen": 2817130496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014886868686868687, + "loss": 2.6503, + "theoretical_loss": 3.33695813559517, + "tokens_seen": 2817196032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014884848484848486, + "loss": 2.4839, + "theoretical_loss": 3.3369520906780332, + "tokens_seen": 2817261568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014882828282828284, + "loss": 2.5883, + "theoretical_loss": 3.336946045940886, + "tokens_seen": 2817327104 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001488080808080808, + "loss": 2.6637, + "theoretical_loss": 3.3369400013837183, + "tokens_seen": 2817392640 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014878787878787878, + "loss": 2.7567, + "theoretical_loss": 3.3369339570065213, + "tokens_seen": 2817458176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014876767676767676, + "loss": 2.5588, + "theoretical_loss": 3.3369279128092844, + "tokens_seen": 2817523712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014874747474747475, + "loss": 2.6365, + "theoretical_loss": 3.3369218687919986, + "tokens_seen": 2817589248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014872727272727273, + "loss": 2.5852, + "theoretical_loss": 3.3369158249546547, + "tokens_seen": 2817654784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014870707070707072, + "loss": 2.7094, + "theoretical_loss": 3.3369097812972424, + "tokens_seen": 2817720320 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486868686868687, + "loss": 2.4317, + "theoretical_loss": 3.336903737819753, + "tokens_seen": 2817785856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014866666666666668, + "loss": 2.9277, + "theoretical_loss": 3.336897694522176, + "tokens_seen": 2817851392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014864646464646464, + "loss": 2.4453, + "theoretical_loss": 3.336891651404503, + "tokens_seen": 2817916928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014862626262626263, + "loss": 2.6589, + "theoretical_loss": 3.3368856084667233, + "tokens_seen": 2817982464 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001486060606060606, + "loss": 2.3943, + "theoretical_loss": 3.3368795657088284, + "tokens_seen": 2818048000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485858585858586, + "loss": 2.4707, + "theoretical_loss": 3.3368735231308078, + "tokens_seen": 2818113536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014856565656565658, + "loss": 2.7306, + "theoretical_loss": 3.3368674807326526, + "tokens_seen": 2818179072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014854545454545453, + "loss": 2.5539, + "theoretical_loss": 3.336861438514353, + "tokens_seen": 2818244608 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1585920, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7153098583221436, + "objective/train/theoretical_loss": 3.3368553964759, + "objective/train/tokens_used": 1177169376, + "theoretical_loss": 3.3368553964759, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014852525252525254, + "loss": 2.7284, + "theoretical_loss": 3.3368553964759, + "tokens_seen": 2818310144 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001485050505050505, + "loss": 2.5718, + "theoretical_loss": 3.336849354617283, + "tokens_seen": 2818375680 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484848484848485, + "loss": 2.6524, + "theoretical_loss": 3.336843312938494, + "tokens_seen": 2818441216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014846464646464647, + "loss": 2.5255, + "theoretical_loss": 3.3368372714395216, + "tokens_seen": 2818506752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014844444444444443, + "loss": 2.3284, + "theoretical_loss": 3.3368312301203575, + "tokens_seen": 2818572288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014842424242424244, + "loss": 2.67, + "theoretical_loss": 3.336825188980992, + "tokens_seen": 2818637824 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001484040404040404, + "loss": 2.6215, + "theoretical_loss": 3.3368191480214153, + "tokens_seen": 2818703360 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001483838383838384, + "loss": 2.5371, + "theoretical_loss": 3.3368131072416185, + "tokens_seen": 2818768896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014836363636363636, + "loss": 2.6207, + "theoretical_loss": 3.336807066641591, + "tokens_seen": 2818834432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014834343434343435, + "loss": 2.6154, + "theoretical_loss": 3.336801026221324, + "tokens_seen": 2818899968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014832323232323233, + "loss": 2.481, + "theoretical_loss": 3.3367949859808084, + "tokens_seen": 2818965504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014832323232323233, + "loss": 2.8218, + "theoretical_loss": 3.3367889459200333, + "tokens_seen": 2819031040 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001483030303030303, + "loss": 2.6668, + "theoretical_loss": 3.3367829060389904, + "tokens_seen": 2819096576 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482828282828283, + "loss": 2.5648, + "theoretical_loss": 3.3367768663376696, + "tokens_seen": 2819162112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014826262626262626, + "loss": 2.6489, + "theoretical_loss": 3.336770826816062, + "tokens_seen": 2819227648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014824242424242424, + "loss": 2.5931, + "theoretical_loss": 3.336764787474157, + "tokens_seen": 2819293184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014822222222222222, + "loss": 2.5757, + "theoretical_loss": 3.3367587483119463, + "tokens_seen": 2819358720 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001482020202020202, + "loss": 2.6458, + "theoretical_loss": 3.336752709329419, + "tokens_seen": 2819424256 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001481818181818182, + "loss": 2.6709, + "theoretical_loss": 3.336746670526567, + "tokens_seen": 2819489792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014816161616161617, + "loss": 2.5954, + "theoretical_loss": 3.3367406319033797, + "tokens_seen": 2819555328 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014814141414141413, + "loss": 2.5947, + "theoretical_loss": 3.3367345934598482, + "tokens_seen": 2819620864 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014812121212121212, + "loss": 2.6275, + "theoretical_loss": 3.336728555195963, + "tokens_seen": 2819686400 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001481010101010101, + "loss": 2.6201, + "theoretical_loss": 3.336722517111714, + "tokens_seen": 2819751936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014808080808080808, + "loss": 2.6573, + "theoretical_loss": 3.336716479207092, + "tokens_seen": 2819817472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014806060606060607, + "loss": 2.6047, + "theoretical_loss": 3.3367104414820874, + "tokens_seen": 2819883008 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1586975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1595256328582764, + "objective/train/theoretical_loss": 3.3367044039366913, + "objective/train/tokens_used": 1178807776, + "theoretical_loss": 3.3367044039366913, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014804040404040402, + "loss": 2.5119, + "theoretical_loss": 3.3367044039366913, + "tokens_seen": 2819948544 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014802020202020204, + "loss": 2.3331, + "theoretical_loss": 3.3366983665708934, + "tokens_seen": 2820014080 + }, + { + "epoch": 0.71, + "learning_rate": 0.000148, + "loss": 2.5922, + "theoretical_loss": 3.3366923293846846, + "tokens_seen": 2820079616 + }, + { + "epoch": 0.71, + "learning_rate": 0.000147979797979798, + "loss": 2.577, + "theoretical_loss": 3.336686292378055, + "tokens_seen": 2820145152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014795959595959596, + "loss": 2.707, + "theoretical_loss": 3.336680255550996, + "tokens_seen": 2820210688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014793939393939392, + "loss": 2.657, + "theoretical_loss": 3.3366742189034966, + "tokens_seen": 2820276224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014791919191919193, + "loss": 2.5023, + "theoretical_loss": 3.336668182435548, + "tokens_seen": 2820341760 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014789898989898989, + "loss": 2.5477, + "theoretical_loss": 3.3366621461471415, + "tokens_seen": 2820407296 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001478787878787879, + "loss": 2.5779, + "theoretical_loss": 3.3366561100382666, + "tokens_seen": 2820472832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014785858585858585, + "loss": 2.7753, + "theoretical_loss": 3.3366500741089142, + "tokens_seen": 2820538368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014783838383838386, + "loss": 2.6621, + "theoretical_loss": 3.3366440383590743, + "tokens_seen": 2820603904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014781818181818182, + "loss": 2.6633, + "theoretical_loss": 3.336638002788738, + "tokens_seen": 2820669440 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477979797979798, + "loss": 2.6956, + "theoretical_loss": 3.3366319673978957, + "tokens_seen": 2820734976 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001477777777777778, + "loss": 2.6357, + "theoretical_loss": 3.3366259321865375, + "tokens_seen": 2820800512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014775757575757575, + "loss": 2.7644, + "theoretical_loss": 3.3366198971546543, + "tokens_seen": 2820866048 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014773737373737376, + "loss": 2.6705, + "theoretical_loss": 3.336613862302236, + "tokens_seen": 2820931584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014771717171717171, + "loss": 2.7115, + "theoretical_loss": 3.3366078276292743, + "tokens_seen": 2820997120 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001476969696969697, + "loss": 2.354, + "theoretical_loss": 3.3366017931357583, + "tokens_seen": 2821062656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014767676767676768, + "loss": 2.4502, + "theoretical_loss": 3.3365957588216792, + "tokens_seen": 2821128192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014765656565656567, + "loss": 2.7223, + "theoretical_loss": 3.3365897246870277, + "tokens_seen": 2821193728 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014763636363636365, + "loss": 2.3915, + "theoretical_loss": 3.3365836907317936, + "tokens_seen": 2821259264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014761616161616163, + "loss": 2.6496, + "theoretical_loss": 3.336577656955968, + "tokens_seen": 2821324800 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001475959595959596, + "loss": 2.7124, + "theoretical_loss": 3.336571623359541, + "tokens_seen": 2821390336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014757575757575757, + "loss": 2.4608, + "theoretical_loss": 3.336565589942504, + "tokens_seen": 2821455872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014755555555555556, + "loss": 2.5795, + "theoretical_loss": 3.336559556704846, + "tokens_seen": 2821521408 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1587530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6532740592956543, + "objective/train/theoretical_loss": 3.3365535236465584, + "objective/train/tokens_used": 1180446176, + "theoretical_loss": 3.3365535236465584, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014753535353535354, + "loss": 2.7027, + "theoretical_loss": 3.3365535236465584, + "tokens_seen": 2821586944 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014751515151515153, + "loss": 2.514, + "theoretical_loss": 3.336547490767632, + "tokens_seen": 2821652480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014749494949494948, + "loss": 2.6643, + "theoretical_loss": 3.336541458068057, + "tokens_seen": 2821718016 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001474747474747475, + "loss": 2.7931, + "theoretical_loss": 3.3365354255478232, + "tokens_seen": 2821783552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014745454545454545, + "loss": 2.6885, + "theoretical_loss": 3.3365293932069218, + "tokens_seen": 2821849088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014743434343434346, + "loss": 2.5243, + "theoretical_loss": 3.3365233610453435, + "tokens_seen": 2821914624 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014741414141414142, + "loss": 2.79, + "theoretical_loss": 3.3365173290630783, + "tokens_seen": 2821980160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014739393939393938, + "loss": 2.6921, + "theoretical_loss": 3.336511297260117, + "tokens_seen": 2822045696 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001473737373737374, + "loss": 2.7087, + "theoretical_loss": 3.3365052656364504, + "tokens_seen": 2822111232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014735353535353534, + "loss": 2.4504, + "theoretical_loss": 3.3364992341920683, + "tokens_seen": 2822176768 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014733333333333335, + "loss": 2.5084, + "theoretical_loss": 3.3364932029269614, + "tokens_seen": 2822242304 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001473131313131313, + "loss": 2.8098, + "theoretical_loss": 3.3364871718411204, + "tokens_seen": 2822307840 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001472929292929293, + "loss": 2.7935, + "theoretical_loss": 3.336481140934536, + "tokens_seen": 2822373376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014727272727272728, + "loss": 2.5439, + "theoretical_loss": 3.336475110207198, + "tokens_seen": 2822438912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014725252525252524, + "loss": 2.6937, + "theoretical_loss": 3.336469079659098, + "tokens_seen": 2822504448 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014723232323232325, + "loss": 2.5313, + "theoretical_loss": 3.3364630492902254, + "tokens_seen": 2822569984 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001472121212121212, + "loss": 2.5362, + "theoretical_loss": 3.3364570191005716, + "tokens_seen": 2822635520 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001471919191919192, + "loss": 2.5404, + "theoretical_loss": 3.3364509890901264, + "tokens_seen": 2822701056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014717171717171717, + "loss": 2.5245, + "theoretical_loss": 3.3364449592588805, + "tokens_seen": 2822766592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014715151515151516, + "loss": 2.7188, + "theoretical_loss": 3.336438929606825, + "tokens_seen": 2822832128 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014713131313131314, + "loss": 2.6414, + "theoretical_loss": 3.33643290013395, + "tokens_seen": 2822897664 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014711111111111112, + "loss": 2.6863, + "theoretical_loss": 3.3364268708402456, + "tokens_seen": 2822963200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014709090909090908, + "loss": 2.4588, + "theoretical_loss": 3.336420841725703, + "tokens_seen": 2823028736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014707070707070706, + "loss": 2.5242, + "theoretical_loss": 3.336414812790312, + "tokens_seen": 2823094272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014705050505050505, + "loss": 2.622, + "theoretical_loss": 3.336408784034064, + "tokens_seen": 2823159808 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1588829, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8554697036743164, + "objective/train/theoretical_loss": 3.336402755456948, + "objective/train/tokens_used": 1182084576, + "theoretical_loss": 3.336402755456948, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014703030303030303, + "loss": 2.7095, + "theoretical_loss": 3.336402755456948, + "tokens_seen": 2823225344 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014701010101010102, + "loss": 2.6132, + "theoretical_loss": 3.3363967270589567, + "tokens_seen": 2823290880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014698989898989897, + "loss": 2.8202, + "theoretical_loss": 3.3363906988400793, + "tokens_seen": 2823356416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014696969696969698, + "loss": 2.5487, + "theoretical_loss": 3.336384670800306, + "tokens_seen": 2823421952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014694949494949494, + "loss": 2.6449, + "theoretical_loss": 3.3363786429396285, + "tokens_seen": 2823487488 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014692929292929295, + "loss": 2.5637, + "theoretical_loss": 3.3363726152580364, + "tokens_seen": 2823553024 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001469090909090909, + "loss": 2.6562, + "theoretical_loss": 3.3363665877555206, + "tokens_seen": 2823618560 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001468888888888889, + "loss": 2.6089, + "theoretical_loss": 3.336360560432071, + "tokens_seen": 2823684096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014686868686868688, + "loss": 2.723, + "theoretical_loss": 3.336354533287679, + "tokens_seen": 2823749632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014684848484848483, + "loss": 2.6515, + "theoretical_loss": 3.336348506322335, + "tokens_seen": 2823815168 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014682828282828285, + "loss": 2.5482, + "theoretical_loss": 3.336342479536029, + "tokens_seen": 2823880704 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001468080808080808, + "loss": 2.7919, + "theoretical_loss": 3.3363364529287516, + "tokens_seen": 2823946240 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467878787878788, + "loss": 2.8372, + "theoretical_loss": 3.3363304265004934, + "tokens_seen": 2824011776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014676767676767677, + "loss": 2.4463, + "theoretical_loss": 3.3363244002512453, + "tokens_seen": 2824077312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014674747474747475, + "loss": 2.692, + "theoretical_loss": 3.3363183741809976, + "tokens_seen": 2824142848 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014672727272727274, + "loss": 2.7299, + "theoretical_loss": 3.3363123482897405, + "tokens_seen": 2824208384 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001467070707070707, + "loss": 2.6351, + "theoretical_loss": 3.3363063225774656, + "tokens_seen": 2824273920 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001466868686868687, + "loss": 2.5857, + "theoretical_loss": 3.336300297044162, + "tokens_seen": 2824339456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014666666666666666, + "loss": 2.756, + "theoretical_loss": 3.3362942716898214, + "tokens_seen": 2824404992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014664646464646465, + "loss": 2.4378, + "theoretical_loss": 3.3362882465144335, + "tokens_seen": 2824470528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014662626262626263, + "loss": 2.6241, + "theoretical_loss": 3.336282221517989, + "tokens_seen": 2824536064 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014660606060606061, + "loss": 2.4451, + "theoretical_loss": 3.336276196700479, + "tokens_seen": 2824601600 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001465858585858586, + "loss": 2.4445, + "theoretical_loss": 3.3362701720618935, + "tokens_seen": 2824667136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014656565656565656, + "loss": 2.6271, + "theoretical_loss": 3.3362641476022232, + "tokens_seen": 2824732672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014654545454545454, + "loss": 2.6219, + "theoretical_loss": 3.3362581233214583, + "tokens_seen": 2824798208 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1589488, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.821836471557617, + "objective/train/theoretical_loss": 3.33625209921959, + "objective/train/tokens_used": 1183722976, + "theoretical_loss": 3.33625209921959, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014652525252525252, + "loss": 2.5892, + "theoretical_loss": 3.33625209921959, + "tokens_seen": 2824863744 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001465050505050505, + "loss": 2.693, + "theoretical_loss": 3.3362460752966085, + "tokens_seen": 2824929280 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464848484848485, + "loss": 2.8048, + "theoretical_loss": 3.336240051552504, + "tokens_seen": 2824994816 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014646464646464648, + "loss": 2.5192, + "theoretical_loss": 3.3362340279872678, + "tokens_seen": 2825060352 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014644444444444443, + "loss": 2.677, + "theoretical_loss": 3.3362280046008896, + "tokens_seen": 2825125888 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014642424242424244, + "loss": 2.6665, + "theoretical_loss": 3.336221981393361, + "tokens_seen": 2825191424 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001464040404040404, + "loss": 2.644, + "theoretical_loss": 3.3362159583646713, + "tokens_seen": 2825256960 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014638383838383838, + "loss": 2.6728, + "theoretical_loss": 3.3362099355148116, + "tokens_seen": 2825322496 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014636363636363637, + "loss": 2.833, + "theoretical_loss": 3.3362039128437724, + "tokens_seen": 2825388032 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014634343434343432, + "loss": 2.692, + "theoretical_loss": 3.336197890351545, + "tokens_seen": 2825453568 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014632323232323234, + "loss": 2.4927, + "theoretical_loss": 3.336191868038118, + "tokens_seen": 2825519104 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001463030303030303, + "loss": 2.7299, + "theoretical_loss": 3.3361858459034845, + "tokens_seen": 2825584640 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001462828282828283, + "loss": 2.4059, + "theoretical_loss": 3.336179823947633, + "tokens_seen": 2825650176 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014626262626262626, + "loss": 2.7649, + "theoretical_loss": 3.336173802170555, + "tokens_seen": 2825715712 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014624242424242424, + "loss": 2.5545, + "theoretical_loss": 3.3361677805722407, + "tokens_seen": 2825781248 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014622222222222223, + "loss": 2.5137, + "theoretical_loss": 3.336161759152681, + "tokens_seen": 2825846784 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014620202020202019, + "loss": 2.6511, + "theoretical_loss": 3.336155737911866, + "tokens_seen": 2825912320 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001461818181818182, + "loss": 2.5898, + "theoretical_loss": 3.3361497168497865, + "tokens_seen": 2825977856 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014616161616161615, + "loss": 2.606, + "theoretical_loss": 3.3361436959664332, + "tokens_seen": 2826043392 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014614141414141414, + "loss": 2.6839, + "theoretical_loss": 3.3361376752617966, + "tokens_seen": 2826108928 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014612121212121212, + "loss": 2.8412, + "theoretical_loss": 3.336131654735867, + "tokens_seen": 2826174464 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001461010101010101, + "loss": 2.5772, + "theoretical_loss": 3.336125634388635, + "tokens_seen": 2826240000 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001460808080808081, + "loss": 2.6362, + "theoretical_loss": 3.336119614220091, + "tokens_seen": 2826305536 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014606060606060607, + "loss": 2.592, + "theoretical_loss": 3.336113594230226, + "tokens_seen": 2826371072 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014604040404040406, + "loss": 2.7922, + "theoretical_loss": 3.3361075744190307, + "tokens_seen": 2826436608 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1590914, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.277503490447998, + "objective/train/theoretical_loss": 3.3361015547864947, + "objective/train/tokens_used": 1185361376, + "theoretical_loss": 3.3361015547864947, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014602020202020201, + "loss": 2.5173, + "theoretical_loss": 3.3361015547864947, + "tokens_seen": 2826502144 + }, + { + "epoch": 0.71, + "learning_rate": 0.000146, + "loss": 2.5956, + "theoretical_loss": 3.3360955353326096, + "tokens_seen": 2826567680 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014597979797979798, + "loss": 2.5857, + "theoretical_loss": 3.336089516057365, + "tokens_seen": 2826633216 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014595959595959597, + "loss": 2.7513, + "theoretical_loss": 3.3360834969607525, + "tokens_seen": 2826698752 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014593939393939395, + "loss": 2.6682, + "theoretical_loss": 3.336077478042762, + "tokens_seen": 2826764288 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014591919191919193, + "loss": 2.5683, + "theoretical_loss": 3.3360714593033842, + "tokens_seen": 2826829824 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001458989898989899, + "loss": 2.6406, + "theoretical_loss": 3.336065440742609, + "tokens_seen": 2826895360 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001458787878787879, + "loss": 2.8081, + "theoretical_loss": 3.3360594223604285, + "tokens_seen": 2826960896 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014585858585858586, + "loss": 2.4921, + "theoretical_loss": 3.336053404156832, + "tokens_seen": 2827026432 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014583838383838384, + "loss": 2.5551, + "theoretical_loss": 3.3360473861318103, + "tokens_seen": 2827091968 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014581818181818183, + "loss": 2.6118, + "theoretical_loss": 3.336041368285354, + "tokens_seen": 2827157504 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014579797979797978, + "loss": 2.5165, + "theoretical_loss": 3.336035350617454, + "tokens_seen": 2827223040 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001457777777777778, + "loss": 2.5102, + "theoretical_loss": 3.336029333128101, + "tokens_seen": 2827288576 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014575757575757575, + "loss": 2.4085, + "theoretical_loss": 3.3360233158172843, + "tokens_seen": 2827354112 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014573737373737376, + "loss": 2.6966, + "theoretical_loss": 3.336017298684996, + "tokens_seen": 2827419648 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014571717171717172, + "loss": 2.5821, + "theoretical_loss": 3.3360112817312255, + "tokens_seen": 2827485184 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014569696969696968, + "loss": 2.5248, + "theoretical_loss": 3.3360052649559644, + "tokens_seen": 2827550720 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456767676767677, + "loss": 2.7466, + "theoretical_loss": 3.3359992483592023, + "tokens_seen": 2827616256 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014565656565656564, + "loss": 2.4718, + "theoretical_loss": 3.3359932319409302, + "tokens_seen": 2827681792 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014563636363636366, + "loss": 2.5353, + "theoretical_loss": 3.335987215701139, + "tokens_seen": 2827747328 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001456161616161616, + "loss": 2.452, + "theoretical_loss": 3.3359811996398188, + "tokens_seen": 2827812864 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455959595959596, + "loss": 2.6481, + "theoretical_loss": 3.3359751837569602, + "tokens_seen": 2827878400 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014557575757575758, + "loss": 2.4688, + "theoretical_loss": 3.335969168052554, + "tokens_seen": 2827943936 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014555555555555556, + "loss": 2.4944, + "theoretical_loss": 3.3359631525265905, + "tokens_seen": 2828009472 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014553535353535355, + "loss": 2.8428, + "theoretical_loss": 3.3359571371790606, + "tokens_seen": 2828075008 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1591402, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.64384126663208, + "objective/train/theoretical_loss": 3.3359511220099547, + "objective/train/tokens_used": 1186999776, + "theoretical_loss": 3.3359511220099547, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001455151515151515, + "loss": 2.4773, + "theoretical_loss": 3.3359511220099547, + "tokens_seen": 2828140544 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001454949494949495, + "loss": 2.6351, + "theoretical_loss": 3.3359451070192634, + "tokens_seen": 2828206080 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014547474747474747, + "loss": 2.5961, + "theoretical_loss": 3.335939092206977, + "tokens_seen": 2828271616 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014545454545454546, + "loss": 2.656, + "theoretical_loss": 3.3359330775730864, + "tokens_seen": 2828337152 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014543434343434344, + "loss": 2.4701, + "theoretical_loss": 3.335927063117582, + "tokens_seen": 2828402688 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014541414141414142, + "loss": 2.4509, + "theoretical_loss": 3.3359210488404547, + "tokens_seen": 2828468224 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014539393939393938, + "loss": 2.6043, + "theoretical_loss": 3.335915034741695, + "tokens_seen": 2828533760 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001453737373737374, + "loss": 2.5882, + "theoretical_loss": 3.335909020821293, + "tokens_seen": 2828599296 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014535353535353535, + "loss": 2.6071, + "theoretical_loss": 3.3359030070792395, + "tokens_seen": 2828664832 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014533333333333333, + "loss": 2.6191, + "theoretical_loss": 3.3358969935155254, + "tokens_seen": 2828730368 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014531313131313132, + "loss": 2.4197, + "theoretical_loss": 3.335890980130141, + "tokens_seen": 2828795904 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014529292929292927, + "loss": 2.7345, + "theoretical_loss": 3.335884966923077, + "tokens_seen": 2828861440 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014527272727272729, + "loss": 2.7525, + "theoretical_loss": 3.3358789538943237, + "tokens_seen": 2828926976 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014525252525252524, + "loss": 2.7103, + "theoretical_loss": 3.3358729410438723, + "tokens_seen": 2828992512 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014523232323232325, + "loss": 2.7273, + "theoretical_loss": 3.3358669283717126, + "tokens_seen": 2829058048 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001452121212121212, + "loss": 2.6687, + "theoretical_loss": 3.335860915877836, + "tokens_seen": 2829123584 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014519191919191922, + "loss": 2.7111, + "theoretical_loss": 3.3358549035622325, + "tokens_seen": 2829189120 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014517171717171718, + "loss": 2.5934, + "theoretical_loss": 3.3358488914248925, + "tokens_seen": 2829254656 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014515151515151513, + "loss": 2.5016, + "theoretical_loss": 3.3358428794658073, + "tokens_seen": 2829320192 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014513131313131315, + "loss": 2.6698, + "theoretical_loss": 3.335836867684967, + "tokens_seen": 2829385728 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001451111111111111, + "loss": 2.7045, + "theoretical_loss": 3.335830856082362, + "tokens_seen": 2829451264 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014509090909090911, + "loss": 2.8239, + "theoretical_loss": 3.3358248446579837, + "tokens_seen": 2829516800 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014507070707070707, + "loss": 2.7165, + "theoretical_loss": 3.335818833411822, + "tokens_seen": 2829582336 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014505050505050505, + "loss": 2.4776, + "theoretical_loss": 3.3358128223438674, + "tokens_seen": 2829647872 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014503030303030304, + "loss": 2.6985, + "theoretical_loss": 3.335806811454111, + "tokens_seen": 2829713408 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1592635, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.982818603515625, + "objective/train/theoretical_loss": 3.335800800742543, + "objective/train/tokens_used": 1188638176, + "theoretical_loss": 3.335800800742543, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.71, + "learning_rate": 0.000145010101010101, + "loss": 2.6625, + "theoretical_loss": 3.335800800742543, + "tokens_seen": 2829778944 + }, + { + "epoch": 0.71, + "learning_rate": 0.000144989898989899, + "loss": 2.6556, + "theoretical_loss": 3.335794790209154, + "tokens_seen": 2829844480 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014496969696969696, + "loss": 2.79, + "theoretical_loss": 3.335788779853935, + "tokens_seen": 2829910016 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014494949494949495, + "loss": 2.5566, + "theoretical_loss": 3.335782769676876, + "tokens_seen": 2829975552 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014492929292929293, + "loss": 2.658, + "theoretical_loss": 3.3357767596779686, + "tokens_seen": 2830041088 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014490909090909092, + "loss": 2.7887, + "theoretical_loss": 3.3357707498572022, + "tokens_seen": 2830106624 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448888888888889, + "loss": 2.4731, + "theoretical_loss": 3.335764740214568, + "tokens_seen": 2830172160 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014486868686868688, + "loss": 2.6829, + "theoretical_loss": 3.3357587307500562, + "tokens_seen": 2830237696 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014484848484848484, + "loss": 2.6332, + "theoretical_loss": 3.335752721463658, + "tokens_seen": 2830303232 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014482828282828282, + "loss": 2.6534, + "theoretical_loss": 3.3357467123553635, + "tokens_seen": 2830368768 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001448080808080808, + "loss": 2.6014, + "theoretical_loss": 3.335740703425164, + "tokens_seen": 2830434304 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447878787878788, + "loss": 2.3947, + "theoretical_loss": 3.335734694673049, + "tokens_seen": 2830499840 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014476767676767678, + "loss": 2.5225, + "theoretical_loss": 3.33572868609901, + "tokens_seen": 2830565376 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014474747474747473, + "loss": 2.7669, + "theoretical_loss": 3.335722677703037, + "tokens_seen": 2830630912 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014472727272727274, + "loss": 2.4193, + "theoretical_loss": 3.3357166694851212, + "tokens_seen": 2830696448 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001447070707070707, + "loss": 2.6232, + "theoretical_loss": 3.3357106614452525, + "tokens_seen": 2830761984 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001446868686868687, + "loss": 2.5304, + "theoretical_loss": 3.3357046535834223, + "tokens_seen": 2830827520 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014466666666666667, + "loss": 2.662, + "theoretical_loss": 3.3356986458996207, + "tokens_seen": 2830893056 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014464646464646463, + "loss": 2.6005, + "theoretical_loss": 3.335692638393838, + "tokens_seen": 2830958592 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014462626262626264, + "loss": 2.4754, + "theoretical_loss": 3.3356866310660656, + "tokens_seen": 2831024128 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001446060606060606, + "loss": 2.6856, + "theoretical_loss": 3.3356806239162937, + "tokens_seen": 2831089664 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445858585858586, + "loss": 2.7395, + "theoretical_loss": 3.335674616944513, + "tokens_seen": 2831155200 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014456565656565656, + "loss": 2.6026, + "theoretical_loss": 3.3356686101507136, + "tokens_seen": 2831220736 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014454545454545455, + "loss": 2.838, + "theoretical_loss": 3.3356626035348866, + "tokens_seen": 2831286272 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014452525252525253, + "loss": 2.6216, + "theoretical_loss": 3.3356565970970227, + "tokens_seen": 2831351808 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1593167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.117902994155884, + "objective/train/theoretical_loss": 3.3356505908371124, + "objective/train/tokens_used": 1190276576, + "theoretical_loss": 3.3356505908371124, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001445050505050505, + "loss": 2.754, + "theoretical_loss": 3.3356505908371124, + "tokens_seen": 2831417344 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001444848484848485, + "loss": 2.8057, + "theoretical_loss": 3.335644584755146, + "tokens_seen": 2831482880 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014446464646464645, + "loss": 2.9434, + "theoretical_loss": 3.3356385788511145, + "tokens_seen": 2831548416 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014444444444444444, + "loss": 2.7686, + "theoretical_loss": 3.3356325731250083, + "tokens_seen": 2831613952 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014442424242424242, + "loss": 2.7816, + "theoretical_loss": 3.335626567576818, + "tokens_seen": 2831679488 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001444040404040404, + "loss": 2.6834, + "theoretical_loss": 3.3356205622065347, + "tokens_seen": 2831745024 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001443838383838384, + "loss": 2.6988, + "theoretical_loss": 3.335614557014148, + "tokens_seen": 2831810560 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014436363636363637, + "loss": 2.5488, + "theoretical_loss": 3.3356085519996497, + "tokens_seen": 2831876096 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014434343434343436, + "loss": 2.584, + "theoretical_loss": 3.3356025471630297, + "tokens_seen": 2831941632 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014432323232323234, + "loss": 2.7109, + "theoretical_loss": 3.3355965425042786, + "tokens_seen": 2832007168 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001443030303030303, + "loss": 2.6676, + "theoretical_loss": 3.335590538023387, + "tokens_seen": 2832072704 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014428282828282828, + "loss": 2.7275, + "theoretical_loss": 3.335584533720346, + "tokens_seen": 2832138240 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014426262626262627, + "loss": 2.6963, + "theoretical_loss": 3.3355785295951454, + "tokens_seen": 2832203776 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014424242424242425, + "loss": 2.5628, + "theoretical_loss": 3.335572525647777, + "tokens_seen": 2832269312 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014422222222222223, + "loss": 2.7117, + "theoretical_loss": 3.3355665218782304, + "tokens_seen": 2832334848 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001442020202020202, + "loss": 2.8327, + "theoretical_loss": 3.3355605182864965, + "tokens_seen": 2832400384 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001441818181818182, + "loss": 2.6926, + "theoretical_loss": 3.335554514872566, + "tokens_seen": 2832465920 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014416161616161616, + "loss": 2.6637, + "theoretical_loss": 3.3355485116364294, + "tokens_seen": 2832531456 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014414141414141417, + "loss": 2.5476, + "theoretical_loss": 3.3355425085780777, + "tokens_seen": 2832596992 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014412121212121213, + "loss": 2.4784, + "theoretical_loss": 3.3355365056975006, + "tokens_seen": 2832662528 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014410101010101008, + "loss": 2.6006, + "theoretical_loss": 3.33553050299469, + "tokens_seen": 2832728064 + }, + { + "epoch": 0.71, + "learning_rate": 0.0001440808080808081, + "loss": 2.6675, + "theoretical_loss": 3.3355245004696354, + "tokens_seen": 2832793600 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014406060606060605, + "loss": 2.6154, + "theoretical_loss": 3.3355184981223283, + "tokens_seen": 2832859136 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014404040404040406, + "loss": 2.7207, + "theoretical_loss": 3.3355124959527584, + "tokens_seen": 2832924672 + }, + { + "epoch": 0.71, + "learning_rate": 0.00014402020202020202, + "loss": 2.7479, + "theoretical_loss": 3.335506493960917, + "tokens_seen": 2832990208 + }, + { + "epoch": 0.71, + "objective/train/docs_used": 1594447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.414046049118042, + "objective/train/theoretical_loss": 3.335500492146795, + "objective/train/tokens_used": 1191914976, + "theoretical_loss": 3.335500492146795, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.71, + "learning_rate": 0.000144, + "loss": 2.6293, + "theoretical_loss": 3.335500492146795, + "tokens_seen": 2833055744 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143979797979798, + "loss": 2.7594, + "theoretical_loss": 3.3354944905103823, + "tokens_seen": 2833121280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014395959595959594, + "loss": 2.4458, + "theoretical_loss": 3.33548848905167, + "tokens_seen": 2833186816 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014393939393939396, + "loss": 2.4703, + "theoretical_loss": 3.3354824877706486, + "tokens_seen": 2833252352 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001439191919191919, + "loss": 2.6737, + "theoretical_loss": 3.335476486667308, + "tokens_seen": 2833317888 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001438989898989899, + "loss": 2.5057, + "theoretical_loss": 3.3354704857416406, + "tokens_seen": 2833383424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014387878787878788, + "loss": 2.6497, + "theoretical_loss": 3.335464484993635, + "tokens_seen": 2833448960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014385858585858586, + "loss": 2.663, + "theoretical_loss": 3.3354584844232833, + "tokens_seen": 2833514496 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014383838383838385, + "loss": 2.5464, + "theoretical_loss": 3.3354524840305757, + "tokens_seen": 2833580032 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014381818181818183, + "loss": 2.5127, + "theoretical_loss": 3.3354464838155025, + "tokens_seen": 2833645568 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001437979797979798, + "loss": 2.5948, + "theoretical_loss": 3.3354404837780542, + "tokens_seen": 2833711104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014377777777777777, + "loss": 2.6095, + "theoretical_loss": 3.3354344839182226, + "tokens_seen": 2833776640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014375757575757576, + "loss": 2.5504, + "theoretical_loss": 3.335428484235997, + "tokens_seen": 2833842176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014373737373737374, + "loss": 2.8531, + "theoretical_loss": 3.3354224847313687, + "tokens_seen": 2833907712 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014371717171717172, + "loss": 2.6015, + "theoretical_loss": 3.3354164854043282, + "tokens_seen": 2833973248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014369696969696968, + "loss": 2.7045, + "theoretical_loss": 3.335410486254866, + "tokens_seen": 2834038784 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001436767676767677, + "loss": 2.6058, + "theoretical_loss": 3.3354044872829736, + "tokens_seen": 2834104320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014365656565656565, + "loss": 2.571, + "theoretical_loss": 3.3353984884886403, + "tokens_seen": 2834169856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014363636363636366, + "loss": 2.608, + "theoretical_loss": 3.3353924898718574, + "tokens_seen": 2834235392 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014361616161616162, + "loss": 2.464, + "theoretical_loss": 3.3353864914326152, + "tokens_seen": 2834300928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014359595959595957, + "loss": 2.4967, + "theoretical_loss": 3.3353804931709052, + "tokens_seen": 2834366464 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014357575757575759, + "loss": 2.4683, + "theoretical_loss": 3.3353744950867172, + "tokens_seen": 2834432000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014355555555555554, + "loss": 2.5328, + "theoretical_loss": 3.3353684971800424, + "tokens_seen": 2834497536 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014353535353535355, + "loss": 2.5829, + "theoretical_loss": 3.335362499450871, + "tokens_seen": 2834563072 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001435151515151515, + "loss": 2.6393, + "theoretical_loss": 3.3353565018991937, + "tokens_seen": 2834628608 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1595629, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.501854181289673, + "objective/train/theoretical_loss": 3.3353505045250014, + "objective/train/tokens_used": 1193553376, + "theoretical_loss": 3.3353505045250014, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014349494949494952, + "loss": 2.7191, + "theoretical_loss": 3.3353505045250014, + "tokens_seen": 2834694144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014347474747474748, + "loss": 2.5599, + "theoretical_loss": 3.335344507328285, + "tokens_seen": 2834759680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014345454545454546, + "loss": 2.5498, + "theoretical_loss": 3.3353385103090343, + "tokens_seen": 2834825216 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014343434343434345, + "loss": 2.6508, + "theoretical_loss": 3.33533251346724, + "tokens_seen": 2834890752 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001434141414141414, + "loss": 2.8004, + "theoretical_loss": 3.3353265168028936, + "tokens_seen": 2834956288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014339393939393941, + "loss": 2.6074, + "theoretical_loss": 3.3353205203159852, + "tokens_seen": 2835021824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014337373737373737, + "loss": 2.6118, + "theoretical_loss": 3.3353145240065056, + "tokens_seen": 2835087360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014335353535353535, + "loss": 2.558, + "theoretical_loss": 3.3353085278744454, + "tokens_seen": 2835152896 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014333333333333334, + "loss": 2.9365, + "theoretical_loss": 3.3353025319197953, + "tokens_seen": 2835218432 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014331313131313132, + "loss": 2.4815, + "theoretical_loss": 3.335296536142546, + "tokens_seen": 2835283968 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001432929292929293, + "loss": 2.5768, + "theoretical_loss": 3.3352905405426876, + "tokens_seen": 2835349504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014327272727272726, + "loss": 2.5678, + "theoretical_loss": 3.3352845451202113, + "tokens_seen": 2835415040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014325252525252525, + "loss": 2.4593, + "theoretical_loss": 3.335278549875108, + "tokens_seen": 2835480576 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014323232323232323, + "loss": 2.5465, + "theoretical_loss": 3.335272554807368, + "tokens_seen": 2835546112 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014321212121212122, + "loss": 2.6759, + "theoretical_loss": 3.335266559916981, + "tokens_seen": 2835611648 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431919191919192, + "loss": 2.6089, + "theoretical_loss": 3.3352605652039395, + "tokens_seen": 2835677184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014317171717171718, + "loss": 2.6763, + "theoretical_loss": 3.335254570668233, + "tokens_seen": 2835742720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014315151515151514, + "loss": 2.694, + "theoretical_loss": 3.3352485763098527, + "tokens_seen": 2835808256 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014313131313131315, + "loss": 2.5521, + "theoretical_loss": 3.3352425821287888, + "tokens_seen": 2835873792 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001431111111111111, + "loss": 2.5518, + "theoretical_loss": 3.335236588125032, + "tokens_seen": 2835939328 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001430909090909091, + "loss": 2.6503, + "theoretical_loss": 3.335230594298573, + "tokens_seen": 2836004864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014307070707070708, + "loss": 2.7272, + "theoretical_loss": 3.3352246006494024, + "tokens_seen": 2836070400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014305050505050503, + "loss": 2.6826, + "theoretical_loss": 3.3352186071775116, + "tokens_seen": 2836135936 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014303030303030304, + "loss": 2.7306, + "theoretical_loss": 3.3352126138828906, + "tokens_seen": 2836201472 + }, + { + "epoch": 0.72, + "learning_rate": 0.000143010101010101, + "loss": 2.8053, + "theoretical_loss": 3.3352066207655295, + "tokens_seen": 2836267008 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1596277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.665980339050293, + "objective/train/theoretical_loss": 3.33520062782542, + "objective/train/tokens_used": 1195191776, + "theoretical_loss": 3.33520062782542, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 0.000142989898989899, + "loss": 2.6699, + "theoretical_loss": 3.33520062782542, + "tokens_seen": 2836332544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014296969696969697, + "loss": 2.7256, + "theoretical_loss": 3.3351946350625523, + "tokens_seen": 2836398080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014294949494949495, + "loss": 2.9139, + "theoretical_loss": 3.335188642476917, + "tokens_seen": 2836463616 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014292929292929294, + "loss": 2.3357, + "theoretical_loss": 3.3351826500685053, + "tokens_seen": 2836529152 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001429090909090909, + "loss": 2.6626, + "theoretical_loss": 3.335176657837307, + "tokens_seen": 2836594688 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001428888888888889, + "loss": 2.5441, + "theoretical_loss": 3.335170665783313, + "tokens_seen": 2836660224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014286868686868686, + "loss": 2.5389, + "theoretical_loss": 3.335164673906515, + "tokens_seen": 2836725760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014284848484848485, + "loss": 2.5666, + "theoretical_loss": 3.335158682206902, + "tokens_seen": 2836791296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014282828282828283, + "loss": 2.4674, + "theoretical_loss": 3.335152690684466, + "tokens_seen": 2836856832 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001428080808080808, + "loss": 2.5407, + "theoretical_loss": 3.335146699339197, + "tokens_seen": 2836922368 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001427878787878788, + "loss": 2.6137, + "theoretical_loss": 3.3351407081710858, + "tokens_seen": 2836987904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014276767676767678, + "loss": 2.6846, + "theoretical_loss": 3.3351347171801233, + "tokens_seen": 2837053440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014274747474747474, + "loss": 2.6416, + "theoretical_loss": 3.3351287263662996, + "tokens_seen": 2837118976 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014272727272727272, + "loss": 2.6147, + "theoretical_loss": 3.3351227357296063, + "tokens_seen": 2837184512 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001427070707070707, + "loss": 2.748, + "theoretical_loss": 3.335116745270033, + "tokens_seen": 2837250048 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426868686868687, + "loss": 2.5118, + "theoretical_loss": 3.335110754987571, + "tokens_seen": 2837315584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014266666666666667, + "loss": 2.5132, + "theoretical_loss": 3.335104764882211, + "tokens_seen": 2837381120 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014264646464646466, + "loss": 2.5616, + "theoretical_loss": 3.3350987749539436, + "tokens_seen": 2837446656 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014262626262626264, + "loss": 2.7447, + "theoretical_loss": 3.335092785202759, + "tokens_seen": 2837512192 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001426060606060606, + "loss": 2.6826, + "theoretical_loss": 3.3350867956286487, + "tokens_seen": 2837577728 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425858585858586, + "loss": 2.4689, + "theoretical_loss": 3.335080806231603, + "tokens_seen": 2837643264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014256565656565657, + "loss": 2.6228, + "theoretical_loss": 3.3350748170116127, + "tokens_seen": 2837708800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014254545454545455, + "loss": 2.6106, + "theoretical_loss": 3.3350688279686675, + "tokens_seen": 2837774336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014252525252525253, + "loss": 2.4376, + "theoretical_loss": 3.3350628391027595, + "tokens_seen": 2837839872 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001425050505050505, + "loss": 2.49, + "theoretical_loss": 3.3350568504138787, + "tokens_seen": 2837905408 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1596937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9120116233825684, + "objective/train/theoretical_loss": 3.3350508619020163, + "objective/train/tokens_used": 1196830176, + "theoretical_loss": 3.3350508619020163, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001424848484848485, + "loss": 2.7132, + "theoretical_loss": 3.3350508619020163, + "tokens_seen": 2837970944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014246464646464646, + "loss": 2.6711, + "theoretical_loss": 3.3350448735671616, + "tokens_seen": 2838036480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014244444444444447, + "loss": 2.398, + "theoretical_loss": 3.3350388854093067, + "tokens_seen": 2838102016 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014242424242424243, + "loss": 2.6308, + "theoretical_loss": 3.3350328974284422, + "tokens_seen": 2838167552 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014240404040404038, + "loss": 2.4999, + "theoretical_loss": 3.335026909624558, + "tokens_seen": 2838233088 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001423838383838384, + "loss": 2.5392, + "theoretical_loss": 3.335020921997645, + "tokens_seen": 2838298624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014236363636363635, + "loss": 2.3818, + "theoretical_loss": 3.335014934547694, + "tokens_seen": 2838364160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014234343434343436, + "loss": 2.6187, + "theoretical_loss": 3.335008947274696, + "tokens_seen": 2838429696 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014232323232323232, + "loss": 2.6513, + "theoretical_loss": 3.3350029601786417, + "tokens_seen": 2838495232 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001423030303030303, + "loss": 2.675, + "theoretical_loss": 3.334996973259521, + "tokens_seen": 2838560768 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001422828282828283, + "loss": 2.4134, + "theoretical_loss": 3.334990986517325, + "tokens_seen": 2838626304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014226262626262627, + "loss": 2.5319, + "theoretical_loss": 3.3349849999520447, + "tokens_seen": 2838691840 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014224242424242426, + "loss": 2.5597, + "theoretical_loss": 3.3349790135636708, + "tokens_seen": 2838757376 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001422222222222222, + "loss": 2.5428, + "theoretical_loss": 3.334973027352193, + "tokens_seen": 2838822912 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001422020202020202, + "loss": 2.5194, + "theoretical_loss": 3.3349670413176034, + "tokens_seen": 2838888448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014218181818181818, + "loss": 2.8301, + "theoretical_loss": 3.334961055459892, + "tokens_seen": 2838953984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014216161616161616, + "loss": 2.5843, + "theoretical_loss": 3.3349550697790495, + "tokens_seen": 2839019520 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014214141414141415, + "loss": 2.6054, + "theoretical_loss": 3.3349490842750664, + "tokens_seen": 2839085056 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014212121212121213, + "loss": 2.5585, + "theoretical_loss": 3.334943098947934, + "tokens_seen": 2839150592 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001421010101010101, + "loss": 2.8251, + "theoretical_loss": 3.334937113797642, + "tokens_seen": 2839216128 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001420808080808081, + "loss": 2.5948, + "theoretical_loss": 3.334931128824182, + "tokens_seen": 2839281664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014206060606060606, + "loss": 2.7429, + "theoretical_loss": 3.3349251440275443, + "tokens_seen": 2839347200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014204040404040404, + "loss": 2.5886, + "theoretical_loss": 3.3349191594077197, + "tokens_seen": 2839412736 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014202020202020203, + "loss": 2.7116, + "theoretical_loss": 3.334913174964699, + "tokens_seen": 2839478272 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014199999999999998, + "loss": 2.6108, + "theoretical_loss": 3.3349071906984724, + "tokens_seen": 2839543808 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1597942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7995715141296387, + "objective/train/theoretical_loss": 3.3349012066090316, + "objective/train/tokens_used": 1198468576, + "theoretical_loss": 3.3349012066090316, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141979797979798, + "loss": 2.8402, + "theoretical_loss": 3.3349012066090316, + "tokens_seen": 2839609344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014195959595959595, + "loss": 2.6183, + "theoretical_loss": 3.334895222696366, + "tokens_seen": 2839674880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014193939393939396, + "loss": 2.8731, + "theoretical_loss": 3.3348892389604674, + "tokens_seen": 2839740416 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014191919191919192, + "loss": 2.4875, + "theoretical_loss": 3.334883255401326, + "tokens_seen": 2839805952 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418989898989899, + "loss": 2.6085, + "theoretical_loss": 3.3348772720189324, + "tokens_seen": 2839871488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014187878787878789, + "loss": 2.6173, + "theoretical_loss": 3.3348712888132774, + "tokens_seen": 2839937024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014185858585858584, + "loss": 2.6086, + "theoretical_loss": 3.3348653057843523, + "tokens_seen": 2840002560 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014183838383838385, + "loss": 2.7312, + "theoretical_loss": 3.3348593229321466, + "tokens_seen": 2840068096 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001418181818181818, + "loss": 2.5555, + "theoretical_loss": 3.334853340256652, + "tokens_seen": 2840133632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014179797979797982, + "loss": 2.3677, + "theoretical_loss": 3.334847357757859, + "tokens_seen": 2840199168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014177777777777778, + "loss": 2.5231, + "theoretical_loss": 3.334841375435758, + "tokens_seen": 2840264704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014175757575757576, + "loss": 2.5424, + "theoretical_loss": 3.33483539329034, + "tokens_seen": 2840330240 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014173737373737375, + "loss": 2.8196, + "theoretical_loss": 3.334829411321595, + "tokens_seen": 2840395776 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001417171717171717, + "loss": 2.5449, + "theoretical_loss": 3.3348234295295147, + "tokens_seen": 2840461312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014169696969696971, + "loss": 2.6563, + "theoretical_loss": 3.3348174479140895, + "tokens_seen": 2840526848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014167676767676767, + "loss": 2.678, + "theoretical_loss": 3.33481146647531, + "tokens_seen": 2840592384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014165656565656566, + "loss": 2.492, + "theoretical_loss": 3.3348054852131668, + "tokens_seen": 2840657920 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014163636363636364, + "loss": 2.5773, + "theoretical_loss": 3.3347995041276506, + "tokens_seen": 2840723456 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014161616161616162, + "loss": 2.4489, + "theoretical_loss": 3.334793523218752, + "tokens_seen": 2840788992 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415959595959596, + "loss": 2.6739, + "theoretical_loss": 3.334787542486463, + "tokens_seen": 2840854528 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001415757575757576, + "loss": 2.5775, + "theoretical_loss": 3.3347815619307726, + "tokens_seen": 2840920064 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014155555555555555, + "loss": 2.7618, + "theoretical_loss": 3.3347755815516718, + "tokens_seen": 2840985600 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014153535353535353, + "loss": 2.6253, + "theoretical_loss": 3.334769601349152, + "tokens_seen": 2841051136 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014151515151515152, + "loss": 2.7437, + "theoretical_loss": 3.3347636213232037, + "tokens_seen": 2841116672 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001414949494949495, + "loss": 2.6764, + "theoretical_loss": 3.3347576414738174, + "tokens_seen": 2841182208 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1598724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.946629762649536, + "objective/train/theoretical_loss": 3.334751661800984, + "objective/train/tokens_used": 1200106976, + "theoretical_loss": 3.334751661800984, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014147474747474748, + "loss": 2.7425, + "theoretical_loss": 3.334751661800984, + "tokens_seen": 2841247744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014145454545454544, + "loss": 2.6395, + "theoretical_loss": 3.334745682304694, + "tokens_seen": 2841313280 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014143434343434345, + "loss": 2.3499, + "theoretical_loss": 3.334739702984938, + "tokens_seen": 2841378816 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001414141414141414, + "loss": 2.5831, + "theoretical_loss": 3.3347337238417074, + "tokens_seen": 2841444352 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014139393939393942, + "loss": 2.4544, + "theoretical_loss": 3.3347277448749924, + "tokens_seen": 2841509888 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014137373737373738, + "loss": 2.5431, + "theoretical_loss": 3.3347217660847837, + "tokens_seen": 2841575424 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014135353535353533, + "loss": 2.466, + "theoretical_loss": 3.334715787471072, + "tokens_seen": 2841640960 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014133333333333334, + "loss": 2.4028, + "theoretical_loss": 3.3347098090338485, + "tokens_seen": 2841706496 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001413131313131313, + "loss": 2.6731, + "theoretical_loss": 3.3347038307731034, + "tokens_seen": 2841772032 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001412929292929293, + "loss": 2.7442, + "theoretical_loss": 3.3346978526888273, + "tokens_seen": 2841837568 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014127272727272727, + "loss": 2.6099, + "theoretical_loss": 3.3346918747810115, + "tokens_seen": 2841903104 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014125252525252525, + "loss": 2.7337, + "theoretical_loss": 3.3346858970496465, + "tokens_seen": 2841968640 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014123232323232324, + "loss": 2.5484, + "theoretical_loss": 3.3346799194947225, + "tokens_seen": 2842034176 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014121212121212122, + "loss": 2.6842, + "theoretical_loss": 3.334673942116231, + "tokens_seen": 2842099712 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001411919191919192, + "loss": 2.537, + "theoretical_loss": 3.3346679649141624, + "tokens_seen": 2842165248 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014117171717171716, + "loss": 2.6439, + "theoretical_loss": 3.3346619878885075, + "tokens_seen": 2842230784 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014115151515151515, + "loss": 2.6806, + "theoretical_loss": 3.3346560110392565, + "tokens_seen": 2842296320 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014113131313131313, + "loss": 2.7751, + "theoretical_loss": 3.334650034366401, + "tokens_seen": 2842361856 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014111111111111111, + "loss": 2.6158, + "theoretical_loss": 3.334644057869931, + "tokens_seen": 2842427392 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001410909090909091, + "loss": 2.6172, + "theoretical_loss": 3.334638081549838, + "tokens_seen": 2842492928 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014107070707070708, + "loss": 2.5884, + "theoretical_loss": 3.334632105406112, + "tokens_seen": 2842558464 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014105050505050504, + "loss": 2.5542, + "theoretical_loss": 3.334626129438744, + "tokens_seen": 2842624000 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014103030303030305, + "loss": 2.5685, + "theoretical_loss": 3.3346201536477245, + "tokens_seen": 2842689536 + }, + { + "epoch": 0.72, + "learning_rate": 0.000141010101010101, + "loss": 2.6295, + "theoretical_loss": 3.3346141780330445, + "tokens_seen": 2842755072 + }, + { + "epoch": 0.72, + "learning_rate": 0.000140989898989899, + "loss": 2.6225, + "theoretical_loss": 3.334608202594695, + "tokens_seen": 2842820608 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1599978, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7706902027130127, + "objective/train/theoretical_loss": 3.334602227332666, + "objective/train/tokens_used": 1201745376, + "theoretical_loss": 3.334602227332666, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014096969696969697, + "loss": 2.7912, + "theoretical_loss": 3.334602227332666, + "tokens_seen": 2842886144 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014094949494949493, + "loss": 2.5822, + "theoretical_loss": 3.334596252246949, + "tokens_seen": 2842951680 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014092929292929294, + "loss": 2.6753, + "theoretical_loss": 3.3345902773375338, + "tokens_seen": 2843017216 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001409090909090909, + "loss": 2.5812, + "theoretical_loss": 3.334584302604412, + "tokens_seen": 2843082752 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001408888888888889, + "loss": 2.6309, + "theoretical_loss": 3.3345783280475745, + "tokens_seen": 2843148288 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014086868686868687, + "loss": 2.7243, + "theoretical_loss": 3.334572353667011, + "tokens_seen": 2843213824 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014084848484848488, + "loss": 2.5842, + "theoretical_loss": 3.334566379462713, + "tokens_seen": 2843279360 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014082828282828284, + "loss": 2.3559, + "theoretical_loss": 3.334560405434671, + "tokens_seen": 2843344896 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001408080808080808, + "loss": 2.5599, + "theoretical_loss": 3.334554431582876, + "tokens_seen": 2843410432 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407878787878788, + "loss": 2.7038, + "theoretical_loss": 3.334548457907318, + "tokens_seen": 2843475968 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014076767676767676, + "loss": 2.3388, + "theoretical_loss": 3.3345424844079887, + "tokens_seen": 2843541504 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014074747474747477, + "loss": 2.5158, + "theoretical_loss": 3.3345365110848784, + "tokens_seen": 2843607040 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014072727272727273, + "loss": 2.7299, + "theoretical_loss": 3.3345305379379777, + "tokens_seen": 2843672576 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001407070707070707, + "loss": 2.7229, + "theoretical_loss": 3.334524564967278, + "tokens_seen": 2843738112 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001406868686868687, + "loss": 2.6431, + "theoretical_loss": 3.3345185921727687, + "tokens_seen": 2843803648 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014066666666666665, + "loss": 2.5833, + "theoretical_loss": 3.3345126195544417, + "tokens_seen": 2843869184 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014064646464646466, + "loss": 2.6287, + "theoretical_loss": 3.3345066471122875, + "tokens_seen": 2843934720 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014062626262626262, + "loss": 2.4918, + "theoretical_loss": 3.334500674846297, + "tokens_seen": 2844000256 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001406060606060606, + "loss": 2.7969, + "theoretical_loss": 3.3344947027564604, + "tokens_seen": 2844065792 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001405858585858586, + "loss": 2.4408, + "theoretical_loss": 3.334488730842769, + "tokens_seen": 2844131328 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014056565656565657, + "loss": 2.569, + "theoretical_loss": 3.334482759105213, + "tokens_seen": 2844196864 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014054545454545456, + "loss": 2.7372, + "theoretical_loss": 3.3344767875437835, + "tokens_seen": 2844262400 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014052525252525254, + "loss": 2.5735, + "theoretical_loss": 3.334470816158471, + "tokens_seen": 2844327936 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001405050505050505, + "loss": 2.5913, + "theoretical_loss": 3.3344648449492666, + "tokens_seen": 2844393472 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014048484848484848, + "loss": 2.6998, + "theoretical_loss": 3.334458873916161, + "tokens_seen": 2844459008 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1601259, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.357177495956421, + "objective/train/theoretical_loss": 3.3344529030591445, + "objective/train/tokens_used": 1203383776, + "theoretical_loss": 3.3344529030591445, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014046464646464647, + "loss": 2.6523, + "theoretical_loss": 3.3344529030591445, + "tokens_seen": 2844524544 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014044444444444445, + "loss": 2.304, + "theoretical_loss": 3.3344469323782087, + "tokens_seen": 2844590080 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014042424242424243, + "loss": 2.5654, + "theoretical_loss": 3.3344409618733435, + "tokens_seen": 2844655616 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001404040404040404, + "loss": 2.6115, + "theoretical_loss": 3.33443499154454, + "tokens_seen": 2844721152 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001403838383838384, + "loss": 2.535, + "theoretical_loss": 3.334429021391789, + "tokens_seen": 2844786688 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014036363636363636, + "loss": 2.4159, + "theoretical_loss": 3.334423051415081, + "tokens_seen": 2844852224 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014034343434343437, + "loss": 2.8584, + "theoretical_loss": 3.3344170816144074, + "tokens_seen": 2844917760 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014032323232323233, + "loss": 2.6047, + "theoretical_loss": 3.334411111989758, + "tokens_seen": 2844983296 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014030303030303028, + "loss": 2.4587, + "theoretical_loss": 3.3344051425411245, + "tokens_seen": 2845048832 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402828282828283, + "loss": 2.6198, + "theoretical_loss": 3.3343991732684968, + "tokens_seen": 2845114368 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014026262626262625, + "loss": 2.4296, + "theoretical_loss": 3.3343932041718665, + "tokens_seen": 2845179904 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014024242424242426, + "loss": 2.5257, + "theoretical_loss": 3.3343872352512234, + "tokens_seen": 2845245440 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014022222222222222, + "loss": 2.5164, + "theoretical_loss": 3.3343812665065595, + "tokens_seen": 2845310976 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001402020202020202, + "loss": 2.5194, + "theoretical_loss": 3.3343752979378642, + "tokens_seen": 2845376512 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014018181818181819, + "loss": 2.6595, + "theoretical_loss": 3.334369329545129, + "tokens_seen": 2845442048 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014016161616161614, + "loss": 2.6513, + "theoretical_loss": 3.3343633613283448, + "tokens_seen": 2845507584 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014014141414141415, + "loss": 2.5109, + "theoretical_loss": 3.334357393287502, + "tokens_seen": 2845573120 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001401212121212121, + "loss": 2.5747, + "theoretical_loss": 3.334351425422591, + "tokens_seen": 2845638656 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001401010101010101, + "loss": 2.6986, + "theoretical_loss": 3.334345457733604, + "tokens_seen": 2845704192 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014008080808080808, + "loss": 2.6248, + "theoretical_loss": 3.3343394902205303, + "tokens_seen": 2845769728 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014006060606060606, + "loss": 2.4531, + "theoretical_loss": 3.334333522883361, + "tokens_seen": 2845835264 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014004040404040405, + "loss": 2.5904, + "theoretical_loss": 3.3343275557220875, + "tokens_seen": 2845900800 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014002020202020203, + "loss": 2.4126, + "theoretical_loss": 3.3343215887366995, + "tokens_seen": 2845966336 + }, + { + "epoch": 0.72, + "learning_rate": 0.00014000000000000001, + "loss": 2.442, + "theoretical_loss": 3.334315621927189, + "tokens_seen": 2846031872 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013997979797979797, + "loss": 2.5713, + "theoretical_loss": 3.334309655293546, + "tokens_seen": 2846097408 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1601933, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.92155385017395, + "objective/train/theoretical_loss": 3.334303688835761, + "objective/train/tokens_used": 1205022176, + "theoretical_loss": 3.334303688835761, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013995959595959596, + "loss": 2.5619, + "theoretical_loss": 3.334303688835761, + "tokens_seen": 2846162944 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013993939393939394, + "loss": 2.6343, + "theoretical_loss": 3.334297722553825, + "tokens_seen": 2846228480 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013991919191919192, + "loss": 2.5565, + "theoretical_loss": 3.3342917564477297, + "tokens_seen": 2846294016 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001398989898989899, + "loss": 2.5957, + "theoretical_loss": 3.3342857905174648, + "tokens_seen": 2846359552 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001398787878787879, + "loss": 2.7775, + "theoretical_loss": 3.334279824763021, + "tokens_seen": 2846425088 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013985858585858585, + "loss": 2.4134, + "theoretical_loss": 3.33427385918439, + "tokens_seen": 2846490624 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013983838383838386, + "loss": 2.4566, + "theoretical_loss": 3.3342678937815617, + "tokens_seen": 2846556160 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013981818181818182, + "loss": 2.1469, + "theoretical_loss": 3.334261928554527, + "tokens_seen": 2846621696 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397979797979798, + "loss": 2.4713, + "theoretical_loss": 3.3342559635032774, + "tokens_seen": 2846687232 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013977777777777778, + "loss": 2.8708, + "theoretical_loss": 3.3342499986278025, + "tokens_seen": 2846752768 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013975757575757574, + "loss": 2.6284, + "theoretical_loss": 3.334244033928094, + "tokens_seen": 2846818304 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013973737373737375, + "loss": 2.7463, + "theoretical_loss": 3.334238069404143, + "tokens_seen": 2846883840 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001397171717171717, + "loss": 2.5605, + "theoretical_loss": 3.3342321050559387, + "tokens_seen": 2846949376 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013969696969696972, + "loss": 2.5248, + "theoretical_loss": 3.334226140883473, + "tokens_seen": 2847014912 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013967676767676768, + "loss": 2.6741, + "theoretical_loss": 3.334220176886737, + "tokens_seen": 2847080448 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013965656565656566, + "loss": 2.4954, + "theoretical_loss": 3.334214213065721, + "tokens_seen": 2847145984 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013963636363636364, + "loss": 2.6356, + "theoretical_loss": 3.3342082494204153, + "tokens_seen": 2847211520 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001396161616161616, + "loss": 2.5555, + "theoretical_loss": 3.334202285950811, + "tokens_seen": 2847277056 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001395959595959596, + "loss": 2.6008, + "theoretical_loss": 3.3341963226568994, + "tokens_seen": 2847342592 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013957575757575757, + "loss": 2.6814, + "theoretical_loss": 3.334190359538671, + "tokens_seen": 2847408128 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013955555555555555, + "loss": 2.5544, + "theoretical_loss": 3.3341843965961164, + "tokens_seen": 2847473664 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013953535353535354, + "loss": 2.4391, + "theoretical_loss": 3.3341784338292264, + "tokens_seen": 2847539200 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013951515151515152, + "loss": 2.4464, + "theoretical_loss": 3.3341724712379914, + "tokens_seen": 2847604736 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001394949494949495, + "loss": 2.6102, + "theoretical_loss": 3.3341665088224035, + "tokens_seen": 2847670272 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001394747474747475, + "loss": 2.3627, + "theoretical_loss": 3.334160546582452, + "tokens_seen": 2847735808 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1603273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9490677118301392, + "objective/train/theoretical_loss": 3.3341545845181284, + "objective/train/tokens_used": 1206660576, + "theoretical_loss": 3.3341545845181284, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013945454545454545, + "loss": 2.3759, + "theoretical_loss": 3.3341545845181284, + "tokens_seen": 2847801344 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013943434343434343, + "loss": 2.3443, + "theoretical_loss": 3.3341486226294235, + "tokens_seen": 2847866880 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013941414141414141, + "loss": 2.536, + "theoretical_loss": 3.334142660916328, + "tokens_seen": 2847932416 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001393939393939394, + "loss": 2.5875, + "theoretical_loss": 3.3341366993788326, + "tokens_seen": 2847997952 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013937373737373738, + "loss": 2.3613, + "theoretical_loss": 3.334130738016928, + "tokens_seen": 2848063488 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013935353535353534, + "loss": 2.4704, + "theoretical_loss": 3.3341247768306053, + "tokens_seen": 2848129024 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013933333333333335, + "loss": 2.7793, + "theoretical_loss": 3.334118815819855, + "tokens_seen": 2848194560 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001393131313131313, + "loss": 2.6997, + "theoretical_loss": 3.334112854984668, + "tokens_seen": 2848260096 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013929292929292932, + "loss": 2.4974, + "theoretical_loss": 3.3341068943250356, + "tokens_seen": 2848325632 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013927272727272727, + "loss": 2.5409, + "theoretical_loss": 3.3341009338409475, + "tokens_seen": 2848391168 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013925252525252523, + "loss": 2.5493, + "theoretical_loss": 3.334094973532395, + "tokens_seen": 2848456704 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013923232323232324, + "loss": 2.4977, + "theoretical_loss": 3.3340890133993693, + "tokens_seen": 2848522240 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001392121212121212, + "loss": 2.7462, + "theoretical_loss": 3.334083053441861, + "tokens_seen": 2848587776 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001391919191919192, + "loss": 2.406, + "theoretical_loss": 3.3340770936598605, + "tokens_seen": 2848653312 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013917171717171717, + "loss": 2.7413, + "theoretical_loss": 3.334071134053359, + "tokens_seen": 2848718848 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013915151515151518, + "loss": 2.7259, + "theoretical_loss": 3.3340651746223466, + "tokens_seen": 2848784384 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013913131313131314, + "loss": 2.5383, + "theoretical_loss": 3.3340592153668154, + "tokens_seen": 2848849920 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001391111111111111, + "loss": 2.5559, + "theoretical_loss": 3.3340532562867553, + "tokens_seen": 2848915456 + }, + { + "epoch": 0.72, + "learning_rate": 0.0001390909090909091, + "loss": 2.6551, + "theoretical_loss": 3.334047297382157, + "tokens_seen": 2848980992 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013907070707070706, + "loss": 2.697, + "theoretical_loss": 3.3340413386530114, + "tokens_seen": 2849046528 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013905050505050507, + "loss": 2.6145, + "theoretical_loss": 3.33403538009931, + "tokens_seen": 2849112064 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013903030303030303, + "loss": 2.6038, + "theoretical_loss": 3.334029421721042, + "tokens_seen": 2849177600 + }, + { + "epoch": 0.72, + "learning_rate": 0.000139010101010101, + "loss": 2.8229, + "theoretical_loss": 3.3340234635182004, + "tokens_seen": 2849243136 + }, + { + "epoch": 0.72, + "learning_rate": 0.000138989898989899, + "loss": 2.6689, + "theoretical_loss": 3.3340175054907744, + "tokens_seen": 2849308672 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013896969696969698, + "loss": 2.6576, + "theoretical_loss": 3.334011547638755, + "tokens_seen": 2849374208 + }, + { + "epoch": 0.72, + "objective/train/docs_used": 1603897, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9762792587280273, + "objective/train/theoretical_loss": 3.3340055899621333, + "objective/train/tokens_used": 1208298976, + "theoretical_loss": 3.3340055899621333, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.72, + "learning_rate": 0.00013894949494949496, + "loss": 2.807, + "theoretical_loss": 3.3340055899621333, + "tokens_seen": 2849439744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013892929292929292, + "loss": 2.4257, + "theoretical_loss": 3.3339996324609, + "tokens_seen": 2849505280 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001389090909090909, + "loss": 2.383, + "theoretical_loss": 3.3339936751350465, + "tokens_seen": 2849570816 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388888888888889, + "loss": 2.5909, + "theoretical_loss": 3.3339877179845625, + "tokens_seen": 2849636352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013886868686868687, + "loss": 2.7873, + "theoretical_loss": 3.3339817610094395, + "tokens_seen": 2849701888 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013884848484848486, + "loss": 2.3917, + "theoretical_loss": 3.333975804209668, + "tokens_seen": 2849767424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013882828282828284, + "loss": 2.4197, + "theoretical_loss": 3.333969847585239, + "tokens_seen": 2849832960 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001388080808080808, + "loss": 2.556, + "theoretical_loss": 3.3339638911361433, + "tokens_seen": 2849898496 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387878787878788, + "loss": 2.5726, + "theoretical_loss": 3.333957934862372, + "tokens_seen": 2849964032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013876767676767677, + "loss": 2.6177, + "theoretical_loss": 3.333951978763915, + "tokens_seen": 2850029568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013874747474747475, + "loss": 2.5688, + "theoretical_loss": 3.333946022840764, + "tokens_seen": 2850095104 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013872727272727273, + "loss": 2.6215, + "theoretical_loss": 3.3339400670929096, + "tokens_seen": 2850160640 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001387070707070707, + "loss": 2.7617, + "theoretical_loss": 3.3339341115203425, + "tokens_seen": 2850226176 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001386868686868687, + "loss": 2.7053, + "theoretical_loss": 3.3339281561230534, + "tokens_seen": 2850291712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013866666666666666, + "loss": 2.5615, + "theoretical_loss": 3.333922200901033, + "tokens_seen": 2850357248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013864646464646467, + "loss": 2.5993, + "theoretical_loss": 3.333916245854273, + "tokens_seen": 2850422784 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013862626262626263, + "loss": 2.5269, + "theoretical_loss": 3.3339102909827627, + "tokens_seen": 2850488320 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001386060606060606, + "loss": 2.3667, + "theoretical_loss": 3.333904336286494, + "tokens_seen": 2850553856 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001385858585858586, + "loss": 2.5283, + "theoretical_loss": 3.333898381765458, + "tokens_seen": 2850619392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013856565656565655, + "loss": 2.6219, + "theoretical_loss": 3.3338924274196446, + "tokens_seen": 2850684928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013854545454545456, + "loss": 2.6058, + "theoretical_loss": 3.3338864732490454, + "tokens_seen": 2850750464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013852525252525252, + "loss": 2.6097, + "theoretical_loss": 3.3338805192536505, + "tokens_seen": 2850816000 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001385050505050505, + "loss": 2.6884, + "theoretical_loss": 3.333874565433451, + "tokens_seen": 2850881536 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384848484848485, + "loss": 2.5721, + "theoretical_loss": 3.333868611788438, + "tokens_seen": 2850947072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013846464646464647, + "loss": 2.4577, + "theoretical_loss": 3.3338626583186017, + "tokens_seen": 2851012608 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1605267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7860958576202393, + "objective/train/theoretical_loss": 3.333856705023934, + "objective/train/tokens_used": 1209937376, + "theoretical_loss": 3.333856705023934, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013844444444444445, + "loss": 2.671, + "theoretical_loss": 3.333856705023934, + "tokens_seen": 2851078144 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384242424242424, + "loss": 2.6515, + "theoretical_loss": 3.333850751904424, + "tokens_seen": 2851143680 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001384040404040404, + "loss": 2.6227, + "theoretical_loss": 3.3338447989600644, + "tokens_seen": 2851209216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013838383838383838, + "loss": 2.6709, + "theoretical_loss": 3.333838846190845, + "tokens_seen": 2851274752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013836363636363636, + "loss": 2.7134, + "theoretical_loss": 3.3338328935967567, + "tokens_seen": 2851340288 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013834343434343435, + "loss": 2.4752, + "theoretical_loss": 3.33382694117779, + "tokens_seen": 2851405824 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013832323232323233, + "loss": 2.4512, + "theoretical_loss": 3.3338209889339367, + "tokens_seen": 2851471360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013830303030303032, + "loss": 2.5679, + "theoretical_loss": 3.333815036865187, + "tokens_seen": 2851536896 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382828282828283, + "loss": 2.7404, + "theoretical_loss": 3.3338090849715316, + "tokens_seen": 2851602432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013826262626262626, + "loss": 2.5592, + "theoretical_loss": 3.333803133252961, + "tokens_seen": 2851667968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013824242424242424, + "loss": 2.7199, + "theoretical_loss": 3.3337971817094676, + "tokens_seen": 2851733504 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013822222222222222, + "loss": 2.7567, + "theoretical_loss": 3.3337912303410406, + "tokens_seen": 2851799040 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001382020202020202, + "loss": 2.6477, + "theoretical_loss": 3.3337852791476714, + "tokens_seen": 2851864576 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001381818181818182, + "loss": 2.8358, + "theoretical_loss": 3.3337793281293506, + "tokens_seen": 2851930112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013816161616161615, + "loss": 2.5073, + "theoretical_loss": 3.33377337728607, + "tokens_seen": 2851995648 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013814141414141416, + "loss": 2.5296, + "theoretical_loss": 3.3337674266178188, + "tokens_seen": 2852061184 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013812121212121212, + "loss": 2.2763, + "theoretical_loss": 3.333761476124589, + "tokens_seen": 2852126720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013810101010101013, + "loss": 2.4521, + "theoretical_loss": 3.3337555258063714, + "tokens_seen": 2852192256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013808080808080808, + "loss": 2.5469, + "theoretical_loss": 3.3337495756631563, + "tokens_seen": 2852257792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013806060606060604, + "loss": 2.5105, + "theoretical_loss": 3.3337436256949347, + "tokens_seen": 2852323328 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013804040404040405, + "loss": 2.509, + "theoretical_loss": 3.3337376759016975, + "tokens_seen": 2852388864 + }, + { + "epoch": 0.73, + "learning_rate": 0.000138020202020202, + "loss": 2.806, + "theoretical_loss": 3.333731726283436, + "tokens_seen": 2852454400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013800000000000002, + "loss": 2.4344, + "theoretical_loss": 3.33372577684014, + "tokens_seen": 2852519936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013797979797979798, + "loss": 2.6394, + "theoretical_loss": 3.3337198275718016, + "tokens_seen": 2852585472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013795959595959596, + "loss": 2.4523, + "theoretical_loss": 3.3337138784784104, + "tokens_seen": 2852651008 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1605428, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9671483039855957, + "objective/train/theoretical_loss": 3.3337079295599583, + "objective/train/tokens_used": 1211575776, + "theoretical_loss": 3.3337079295599583, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013793939393939395, + "loss": 2.6498, + "theoretical_loss": 3.3337079295599583, + "tokens_seen": 2852716544 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013791919191919193, + "loss": 2.6509, + "theoretical_loss": 3.333701980816435, + "tokens_seen": 2852782080 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001378989898989899, + "loss": 2.5775, + "theoretical_loss": 3.3336960322478326, + "tokens_seen": 2852847616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013787878787878787, + "loss": 2.6426, + "theoretical_loss": 3.3336900838541412, + "tokens_seen": 2852913152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013785858585858585, + "loss": 2.4418, + "theoretical_loss": 3.3336841356353517, + "tokens_seen": 2852978688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013783838383838384, + "loss": 2.4718, + "theoretical_loss": 3.3336781875914547, + "tokens_seen": 2853044224 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013781818181818182, + "loss": 2.3927, + "theoretical_loss": 3.333672239722442, + "tokens_seen": 2853109760 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377979797979798, + "loss": 2.8124, + "theoretical_loss": 3.333666292028303, + "tokens_seen": 2853175296 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001377777777777778, + "loss": 2.5232, + "theoretical_loss": 3.33366034450903, + "tokens_seen": 2853240832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013775757575757575, + "loss": 2.7163, + "theoretical_loss": 3.3336543971646124, + "tokens_seen": 2853306368 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013773737373737376, + "loss": 2.6581, + "theoretical_loss": 3.3336484499950423, + "tokens_seen": 2853371904 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013771717171717171, + "loss": 2.6225, + "theoretical_loss": 3.33364250300031, + "tokens_seen": 2853437440 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001376969696969697, + "loss": 2.484, + "theoretical_loss": 3.3336365561804064, + "tokens_seen": 2853502976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013767676767676768, + "loss": 2.6192, + "theoretical_loss": 3.333630609535322, + "tokens_seen": 2853568512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013765656565656564, + "loss": 2.6795, + "theoretical_loss": 3.3336246630650486, + "tokens_seen": 2853634048 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013763636363636365, + "loss": 2.5328, + "theoretical_loss": 3.333618716769576, + "tokens_seen": 2853699584 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001376161616161616, + "loss": 2.536, + "theoretical_loss": 3.333612770648896, + "tokens_seen": 2853765120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013759595959595962, + "loss": 2.6757, + "theoretical_loss": 3.3336068247029984, + "tokens_seen": 2853830656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013757575757575758, + "loss": 2.5896, + "theoretical_loss": 3.3336008789318745, + "tokens_seen": 2853896192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013755555555555553, + "loss": 2.8007, + "theoretical_loss": 3.3335949333355157, + "tokens_seen": 2853961728 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013753535353535354, + "loss": 2.5265, + "theoretical_loss": 3.3335889879139122, + "tokens_seen": 2854027264 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001375151515151515, + "loss": 2.6248, + "theoretical_loss": 3.3335830426670547, + "tokens_seen": 2854092800 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001374949494949495, + "loss": 2.8189, + "theoretical_loss": 3.333577097594935, + "tokens_seen": 2854158336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013747474747474747, + "loss": 2.5896, + "theoretical_loss": 3.3335711526975427, + "tokens_seen": 2854223872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013745454545454548, + "loss": 2.6876, + "theoretical_loss": 3.3335652079748694, + "tokens_seen": 2854289408 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1606174, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8155550956726074, + "objective/train/theoretical_loss": 3.3335592634269062, + "objective/train/tokens_used": 1213214176, + "theoretical_loss": 3.3335592634269062, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013743434343434344, + "loss": 2.7572, + "theoretical_loss": 3.3335592634269062, + "tokens_seen": 2854354944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013741414141414142, + "loss": 2.6741, + "theoretical_loss": 3.3335533190536433, + "tokens_seen": 2854420480 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373939393939394, + "loss": 2.4421, + "theoretical_loss": 3.333547374855072, + "tokens_seen": 2854486016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013737373737373736, + "loss": 2.7302, + "theoretical_loss": 3.333541430831183, + "tokens_seen": 2854551552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013735353535353537, + "loss": 2.4873, + "theoretical_loss": 3.333535486981967, + "tokens_seen": 2854617088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013733333333333333, + "loss": 2.4958, + "theoretical_loss": 3.333529543307415, + "tokens_seen": 2854682624 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001373131313131313, + "loss": 2.5067, + "theoretical_loss": 3.333523599807518, + "tokens_seen": 2854748160 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001372929292929293, + "loss": 2.7101, + "theoretical_loss": 3.333517656482267, + "tokens_seen": 2854813696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013727272727272728, + "loss": 2.5744, + "theoretical_loss": 3.333511713331652, + "tokens_seen": 2854879232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013725252525252526, + "loss": 2.6134, + "theoretical_loss": 3.3335057703556648, + "tokens_seen": 2854944768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013723232323232325, + "loss": 2.6545, + "theoretical_loss": 3.333499827554296, + "tokens_seen": 2855010304 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001372121212121212, + "loss": 2.5635, + "theoretical_loss": 3.3334938849275364, + "tokens_seen": 2855075840 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001371919191919192, + "loss": 2.936, + "theoretical_loss": 3.3334879424753767, + "tokens_seen": 2855141376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013717171717171717, + "loss": 2.5673, + "theoretical_loss": 3.3334820001978076, + "tokens_seen": 2855206912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013715151515151516, + "loss": 2.5123, + "theoretical_loss": 3.3334760580948206, + "tokens_seen": 2855272448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013713131313131314, + "loss": 2.7674, + "theoretical_loss": 3.333470116166406, + "tokens_seen": 2855337984 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001371111111111111, + "loss": 2.6777, + "theoretical_loss": 3.3334641744125553, + "tokens_seen": 2855403520 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001370909090909091, + "loss": 2.6718, + "theoretical_loss": 3.3334582328332587, + "tokens_seen": 2855469056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013707070707070707, + "loss": 2.4197, + "theoretical_loss": 3.3334522914285074, + "tokens_seen": 2855534592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013705050505050508, + "loss": 2.6007, + "theoretical_loss": 3.333446350198292, + "tokens_seen": 2855600128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013703030303030303, + "loss": 2.7164, + "theoretical_loss": 3.3334404091426038, + "tokens_seen": 2855665664 + }, + { + "epoch": 0.73, + "learning_rate": 0.000137010101010101, + "loss": 2.7003, + "theoretical_loss": 3.333434468261433, + "tokens_seen": 2855731200 + }, + { + "epoch": 0.73, + "learning_rate": 0.000136989898989899, + "loss": 2.6585, + "theoretical_loss": 3.333428527554771, + "tokens_seen": 2855796736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013696969696969696, + "loss": 2.5956, + "theoretical_loss": 3.333422587022609, + "tokens_seen": 2855862272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013694949494949497, + "loss": 2.7304, + "theoretical_loss": 3.3334166466649373, + "tokens_seen": 2855927808 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1607592, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.049065589904785, + "objective/train/theoretical_loss": 3.3334107064817466, + "objective/train/tokens_used": 1214852576, + "theoretical_loss": 3.3334107064817466, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013692929292929293, + "loss": 2.5486, + "theoretical_loss": 3.3334107064817466, + "tokens_seen": 2855993344 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001369090909090909, + "loss": 2.5751, + "theoretical_loss": 3.3334047664730284, + "tokens_seen": 2856058880 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001368888888888889, + "loss": 2.5907, + "theoretical_loss": 3.333398826638773, + "tokens_seen": 2856124416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013686868686868685, + "loss": 2.8661, + "theoretical_loss": 3.3333928869789715, + "tokens_seen": 2856189952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013684848484848486, + "loss": 2.5442, + "theoretical_loss": 3.333386947493615, + "tokens_seen": 2856255488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013682828282828282, + "loss": 2.5459, + "theoretical_loss": 3.333381008182694, + "tokens_seen": 2856321024 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001368080808080808, + "loss": 2.5471, + "theoretical_loss": 3.3333750690461996, + "tokens_seen": 2856386560 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001367878787878788, + "loss": 2.6364, + "theoretical_loss": 3.3333691300841224, + "tokens_seen": 2856452096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013676767676767677, + "loss": 2.5372, + "theoretical_loss": 3.3333631912964536, + "tokens_seen": 2856517632 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013674747474747475, + "loss": 2.6285, + "theoretical_loss": 3.333357252683184, + "tokens_seen": 2856583168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013672727272727274, + "loss": 2.7709, + "theoretical_loss": 3.3333513142443048, + "tokens_seen": 2856648704 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001367070707070707, + "loss": 2.868, + "theoretical_loss": 3.3333453759798064, + "tokens_seen": 2856714240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013668686868686868, + "loss": 2.7472, + "theoretical_loss": 3.3333394378896797, + "tokens_seen": 2856779776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013666666666666666, + "loss": 2.6422, + "theoretical_loss": 3.3333334999739157, + "tokens_seen": 2856845312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013664646464646465, + "loss": 2.7164, + "theoretical_loss": 3.333327562232505, + "tokens_seen": 2856910848 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013662626262626263, + "loss": 2.648, + "theoretical_loss": 3.333321624665439, + "tokens_seen": 2856976384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013660606060606062, + "loss": 2.519, + "theoretical_loss": 3.3333156872727083, + "tokens_seen": 2857041920 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365858585858586, + "loss": 2.7532, + "theoretical_loss": 3.333309750054304, + "tokens_seen": 2857107456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013656565656565656, + "loss": 2.446, + "theoretical_loss": 3.3333038130102164, + "tokens_seen": 2857172992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013654545454545457, + "loss": 2.4291, + "theoretical_loss": 3.333297876140437, + "tokens_seen": 2857238528 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013652525252525252, + "loss": 2.4446, + "theoretical_loss": 3.3332919394449565, + "tokens_seen": 2857304064 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001365050505050505, + "loss": 2.6663, + "theoretical_loss": 3.3332860029237654, + "tokens_seen": 2857369600 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001364848484848485, + "loss": 2.5498, + "theoretical_loss": 3.3332800665768554, + "tokens_seen": 2857435136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013646464646464645, + "loss": 2.6893, + "theoretical_loss": 3.3332741304042166, + "tokens_seen": 2857500672 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013644444444444446, + "loss": 2.5717, + "theoretical_loss": 3.33326819440584, + "tokens_seen": 2857566208 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1608324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.499014377593994, + "objective/train/theoretical_loss": 3.3332622585817173, + "objective/train/tokens_used": 1216490976, + "theoretical_loss": 3.3332622585817173, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013642424242424242, + "loss": 2.604, + "theoretical_loss": 3.3332622585817173, + "tokens_seen": 2857631744 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013640404040404043, + "loss": 2.6728, + "theoretical_loss": 3.3332563229318386, + "tokens_seen": 2857697280 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013638383838383838, + "loss": 2.4662, + "theoretical_loss": 3.3332503874561947, + "tokens_seen": 2857762816 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013636363636363637, + "loss": 2.4987, + "theoretical_loss": 3.3332444521547773, + "tokens_seen": 2857828352 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013634343434343435, + "loss": 2.6456, + "theoretical_loss": 3.333238517027576, + "tokens_seen": 2857893888 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001363232323232323, + "loss": 2.6369, + "theoretical_loss": 3.333232582074583, + "tokens_seen": 2857959424 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013630303030303032, + "loss": 2.8388, + "theoretical_loss": 3.3332266472957883, + "tokens_seen": 2858024960 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013628282828282828, + "loss": 2.6351, + "theoretical_loss": 3.3332207126911833, + "tokens_seen": 2858090496 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013626262626262626, + "loss": 2.6902, + "theoretical_loss": 3.333214778260759, + "tokens_seen": 2858156032 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013624242424242425, + "loss": 2.7412, + "theoretical_loss": 3.333208844004506, + "tokens_seen": 2858221568 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013622222222222223, + "loss": 2.6521, + "theoretical_loss": 3.3332029099224147, + "tokens_seen": 2858287104 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001362020202020202, + "loss": 2.5277, + "theoretical_loss": 3.3331969760144764, + "tokens_seen": 2858352640 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001361818181818182, + "loss": 2.6865, + "theoretical_loss": 3.3331910422806827, + "tokens_seen": 2858418176 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013616161616161615, + "loss": 2.6674, + "theoretical_loss": 3.3331851087210236, + "tokens_seen": 2858483712 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013614141414141414, + "loss": 2.5613, + "theoretical_loss": 3.3331791753354905, + "tokens_seen": 2858549248 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013612121212121212, + "loss": 2.6401, + "theoretical_loss": 3.333173242124074, + "tokens_seen": 2858614784 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001361010101010101, + "loss": 2.4694, + "theoretical_loss": 3.333167309086765, + "tokens_seen": 2858680320 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001360808080808081, + "loss": 2.7675, + "theoretical_loss": 3.3331613762235546, + "tokens_seen": 2858745856 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013606060606060605, + "loss": 2.6569, + "theoretical_loss": 3.3331554435344333, + "tokens_seen": 2858811392 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013604040404040406, + "loss": 2.72, + "theoretical_loss": 3.3331495110193927, + "tokens_seen": 2858876928 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013602020202020201, + "loss": 2.5715, + "theoretical_loss": 3.333143578678423, + "tokens_seen": 2858942464 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013600000000000003, + "loss": 2.6062, + "theoretical_loss": 3.3331376465115157, + "tokens_seen": 2859008000 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013597979797979798, + "loss": 2.7249, + "theoretical_loss": 3.333131714518661, + "tokens_seen": 2859073536 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013595959595959594, + "loss": 2.5639, + "theoretical_loss": 3.3331257826998506, + "tokens_seen": 2859139072 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013593939393939395, + "loss": 2.7287, + "theoretical_loss": 3.333119851055075, + "tokens_seen": 2859204608 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1609642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.558042526245117, + "objective/train/theoretical_loss": 3.333113919584325, + "objective/train/tokens_used": 1218129376, + "theoretical_loss": 3.333113919584325, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001359191919191919, + "loss": 2.6185, + "theoretical_loss": 3.333113919584325, + "tokens_seen": 2859270144 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013589898989898992, + "loss": 2.6359, + "theoretical_loss": 3.3331079882875914, + "tokens_seen": 2859335680 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013587878787878788, + "loss": 2.6115, + "theoretical_loss": 3.3331020571648655, + "tokens_seen": 2859401216 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013585858585858586, + "loss": 2.6758, + "theoretical_loss": 3.3330961262161383, + "tokens_seen": 2859466752 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013583838383838384, + "loss": 2.6501, + "theoretical_loss": 3.3330901954414, + "tokens_seen": 2859532288 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001358181818181818, + "loss": 2.4718, + "theoretical_loss": 3.3330842648406422, + "tokens_seen": 2859597824 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001357979797979798, + "loss": 2.7964, + "theoretical_loss": 3.3330783344138553, + "tokens_seen": 2859663360 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013577777777777777, + "loss": 2.5916, + "theoretical_loss": 3.333072404161031, + "tokens_seen": 2859728896 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013575757575757578, + "loss": 2.4871, + "theoretical_loss": 3.333066474082159, + "tokens_seen": 2859794432 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013573737373737374, + "loss": 2.4846, + "theoretical_loss": 3.3330605441772314, + "tokens_seen": 2859859968 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013571717171717172, + "loss": 2.6681, + "theoretical_loss": 3.3330546144462385, + "tokens_seen": 2859925504 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356969696969697, + "loss": 2.4467, + "theoretical_loss": 3.3330486848891714, + "tokens_seen": 2859991040 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356767676767677, + "loss": 2.6016, + "theoretical_loss": 3.3330427555060207, + "tokens_seen": 2860056576 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013565656565656567, + "loss": 2.3326, + "theoretical_loss": 3.3330368262967776, + "tokens_seen": 2860122112 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013563636363636363, + "loss": 2.641, + "theoretical_loss": 3.333030897261433, + "tokens_seen": 2860187648 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001356161616161616, + "loss": 2.7596, + "theoretical_loss": 3.333024968399978, + "tokens_seen": 2860253184 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001355959595959596, + "loss": 2.5003, + "theoretical_loss": 3.333019039712403, + "tokens_seen": 2860318720 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013557575757575758, + "loss": 2.7415, + "theoretical_loss": 3.3330131111986994, + "tokens_seen": 2860384256 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013555555555555556, + "loss": 2.5886, + "theoretical_loss": 3.3330071828588577, + "tokens_seen": 2860449792 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013553535353535355, + "loss": 2.6965, + "theoretical_loss": 3.333001254692869, + "tokens_seen": 2860515328 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001355151515151515, + "loss": 2.8513, + "theoretical_loss": 3.3329953267007246, + "tokens_seen": 2860580864 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013549494949494952, + "loss": 2.4875, + "theoretical_loss": 3.3329893988824146, + "tokens_seen": 2860646400 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013547474747474747, + "loss": 2.5837, + "theoretical_loss": 3.332983471237931, + "tokens_seen": 2860711936 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013545454545454546, + "loss": 2.6837, + "theoretical_loss": 3.3329775437672637, + "tokens_seen": 2860777472 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013543434343434344, + "loss": 2.6853, + "theoretical_loss": 3.332971616470404, + "tokens_seen": 2860843008 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1610230, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9325733184814453, + "objective/train/theoretical_loss": 3.3329656893473434, + "objective/train/tokens_used": 1219767776, + "theoretical_loss": 3.3329656893473434, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001354141414141414, + "loss": 2.8396, + "theoretical_loss": 3.3329656893473434, + "tokens_seen": 2860908544 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001353939393939394, + "loss": 2.5736, + "theoretical_loss": 3.3329597623980716, + "tokens_seen": 2860974080 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013537373737373737, + "loss": 2.615, + "theoretical_loss": 3.3329538356225807, + "tokens_seen": 2861039616 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013535353535353538, + "loss": 2.5219, + "theoretical_loss": 3.332947909020861, + "tokens_seen": 2861105152 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013533333333333333, + "loss": 2.5739, + "theoretical_loss": 3.3329419825929034, + "tokens_seen": 2861170688 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013531313131313132, + "loss": 2.5544, + "theoretical_loss": 3.332936056338699, + "tokens_seen": 2861236224 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352929292929293, + "loss": 2.6029, + "theoretical_loss": 3.332930130258239, + "tokens_seen": 2861301760 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013527272727272726, + "loss": 2.7059, + "theoretical_loss": 3.3329242043515137, + "tokens_seen": 2861367296 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013525252525252527, + "loss": 2.6034, + "theoretical_loss": 3.3329182786185143, + "tokens_seen": 2861432832 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013523232323232323, + "loss": 2.7202, + "theoretical_loss": 3.332912353059232, + "tokens_seen": 2861498368 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001352121212121212, + "loss": 2.2567, + "theoretical_loss": 3.3329064276736577, + "tokens_seen": 2861563904 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351919191919192, + "loss": 2.7986, + "theoretical_loss": 3.332900502461782, + "tokens_seen": 2861629440 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013517171717171718, + "loss": 2.8531, + "theoretical_loss": 3.332894577423596, + "tokens_seen": 2861694976 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013515151515151516, + "loss": 2.4502, + "theoretical_loss": 3.332888652559091, + "tokens_seen": 2861760512 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013513131313131312, + "loss": 2.5779, + "theoretical_loss": 3.332882727868257, + "tokens_seen": 2861826048 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001351111111111111, + "loss": 2.6879, + "theoretical_loss": 3.3328768033510854, + "tokens_seen": 2861891584 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001350909090909091, + "loss": 2.85, + "theoretical_loss": 3.3328708790075674, + "tokens_seen": 2861957120 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013507070707070707, + "loss": 2.5342, + "theoretical_loss": 3.3328649548376936, + "tokens_seen": 2862022656 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013505050505050506, + "loss": 2.5121, + "theoretical_loss": 3.332859030841455, + "tokens_seen": 2862088192 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013503030303030304, + "loss": 2.8289, + "theoretical_loss": 3.332853107018843, + "tokens_seen": 2862153728 + }, + { + "epoch": 0.73, + "learning_rate": 0.000135010101010101, + "loss": 2.6052, + "theoretical_loss": 3.3328471833698483, + "tokens_seen": 2862219264 + }, + { + "epoch": 0.73, + "learning_rate": 0.000134989898989899, + "loss": 2.7471, + "theoretical_loss": 3.3328412598944612, + "tokens_seen": 2862284800 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013496969696969696, + "loss": 2.6793, + "theoretical_loss": 3.3328353365926735, + "tokens_seen": 2862350336 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013494949494949495, + "loss": 2.6528, + "theoretical_loss": 3.3328294134644754, + "tokens_seen": 2862415872 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013492929292929293, + "loss": 2.6011, + "theoretical_loss": 3.3328234905098584, + "tokens_seen": 2862481408 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1611371, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8768606185913086, + "objective/train/theoretical_loss": 3.3328175677288137, + "objective/train/tokens_used": 1221406176, + "theoretical_loss": 3.3328175677288137, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013490909090909092, + "loss": 2.6718, + "theoretical_loss": 3.3328175677288137, + "tokens_seen": 2862546944 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001348888888888889, + "loss": 2.8282, + "theoretical_loss": 3.332811645121331, + "tokens_seen": 2862612480 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013486868686868686, + "loss": 2.5053, + "theoretical_loss": 3.3328057226874024, + "tokens_seen": 2862678016 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013484848484848487, + "loss": 2.6206, + "theoretical_loss": 3.3327998004270185, + "tokens_seen": 2862743552 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013482828282828282, + "loss": 2.4429, + "theoretical_loss": 3.33279387834017, + "tokens_seen": 2862809088 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013480808080808084, + "loss": 2.6392, + "theoretical_loss": 3.332787956426848, + "tokens_seen": 2862874624 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001347878787878788, + "loss": 2.6568, + "theoretical_loss": 3.332782034687044, + "tokens_seen": 2862940160 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013476767676767675, + "loss": 2.7364, + "theoretical_loss": 3.332776113120748, + "tokens_seen": 2863005696 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013474747474747476, + "loss": 2.704, + "theoretical_loss": 3.3327701917279513, + "tokens_seen": 2863071232 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013472727272727272, + "loss": 2.5979, + "theoretical_loss": 3.332764270508645, + "tokens_seen": 2863136768 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013470707070707073, + "loss": 2.6718, + "theoretical_loss": 3.3327583494628197, + "tokens_seen": 2863202304 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013468686868686869, + "loss": 2.6601, + "theoretical_loss": 3.3327524285904673, + "tokens_seen": 2863267840 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013466666666666667, + "loss": 2.7201, + "theoretical_loss": 3.3327465078915774, + "tokens_seen": 2863333376 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013464646464646465, + "loss": 2.6015, + "theoretical_loss": 3.332740587366142, + "tokens_seen": 2863398912 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013462626262626264, + "loss": 2.454, + "theoretical_loss": 3.3327346670141518, + "tokens_seen": 2863464448 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013460606060606062, + "loss": 2.5994, + "theoretical_loss": 3.3327287468355973, + "tokens_seen": 2863529984 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013458585858585858, + "loss": 2.6105, + "theoretical_loss": 3.33272282683047, + "tokens_seen": 2863595520 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013456565656565656, + "loss": 2.5217, + "theoretical_loss": 3.33271690699876, + "tokens_seen": 2863661056 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013454545454545455, + "loss": 2.7872, + "theoretical_loss": 3.3327109873404597, + "tokens_seen": 2863726592 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013452525252525253, + "loss": 2.6695, + "theoretical_loss": 3.3327050678555588, + "tokens_seen": 2863792128 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013450505050505051, + "loss": 2.7476, + "theoretical_loss": 3.332699148544049, + "tokens_seen": 2863857664 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344848484848485, + "loss": 2.5808, + "theoretical_loss": 3.3326932294059204, + "tokens_seen": 2863923200 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013446464646464645, + "loss": 2.6546, + "theoretical_loss": 3.3326873104411647, + "tokens_seen": 2863988736 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013444444444444447, + "loss": 2.7166, + "theoretical_loss": 3.3326813916497726, + "tokens_seen": 2864054272 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013442424242424242, + "loss": 2.8483, + "theoretical_loss": 3.332675473031735, + "tokens_seen": 2864119808 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1612003, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.85954213142395, + "objective/train/theoretical_loss": 3.332669554587043, + "objective/train/tokens_used": 1223044576, + "theoretical_loss": 3.332669554587043, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001344040404040404, + "loss": 2.8214, + "theoretical_loss": 3.332669554587043, + "tokens_seen": 2864185344 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001343838383838384, + "loss": 2.5381, + "theoretical_loss": 3.3326636363156874, + "tokens_seen": 2864250880 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013436363636363635, + "loss": 2.6935, + "theoretical_loss": 3.3326577182176598, + "tokens_seen": 2864316416 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013434343434343436, + "loss": 2.5114, + "theoretical_loss": 3.33265180029295, + "tokens_seen": 2864381952 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013432323232323232, + "loss": 2.8168, + "theoretical_loss": 3.33264588254155, + "tokens_seen": 2864447488 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013430303030303033, + "loss": 2.6386, + "theoretical_loss": 3.33263996496345, + "tokens_seen": 2864513024 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013428282828282828, + "loss": 2.8271, + "theoretical_loss": 3.332634047558641, + "tokens_seen": 2864578560 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013426262626262624, + "loss": 2.6661, + "theoretical_loss": 3.332628130327115, + "tokens_seen": 2864644096 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013424242424242425, + "loss": 2.8101, + "theoretical_loss": 3.332622213268862, + "tokens_seen": 2864709632 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001342222222222222, + "loss": 2.7092, + "theoretical_loss": 3.332616296383873, + "tokens_seen": 2864775168 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013420202020202022, + "loss": 2.7309, + "theoretical_loss": 3.332610379672139, + "tokens_seen": 2864840704 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013418181818181818, + "loss": 2.649, + "theoretical_loss": 3.3326044631336513, + "tokens_seen": 2864906240 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013416161616161616, + "loss": 2.6161, + "theoretical_loss": 3.3325985467684007, + "tokens_seen": 2864971776 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013414141414141414, + "loss": 2.8911, + "theoretical_loss": 3.3325926305763782, + "tokens_seen": 2865037312 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013412121212121213, + "loss": 2.6081, + "theoretical_loss": 3.3325867145575745, + "tokens_seen": 2865102848 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001341010101010101, + "loss": 2.515, + "theoretical_loss": 3.332580798711981, + "tokens_seen": 2865168384 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013408080808080807, + "loss": 2.451, + "theoretical_loss": 3.3325748830395883, + "tokens_seen": 2865233920 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013406060606060605, + "loss": 2.8154, + "theoretical_loss": 3.3325689675403876, + "tokens_seen": 2865299456 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013404040404040404, + "loss": 2.7901, + "theoretical_loss": 3.3325630522143697, + "tokens_seen": 2865364992 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013402020202020202, + "loss": 2.7643, + "theoretical_loss": 3.3325571370615252, + "tokens_seen": 2865430528 + }, + { + "epoch": 0.73, + "learning_rate": 0.000134, + "loss": 2.6082, + "theoretical_loss": 3.332551222081846, + "tokens_seen": 2865496064 + }, + { + "epoch": 0.73, + "learning_rate": 0.000133979797979798, + "loss": 2.5898, + "theoretical_loss": 3.3325453072753226, + "tokens_seen": 2865561600 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013395959595959597, + "loss": 2.8628, + "theoretical_loss": 3.3325393926419458, + "tokens_seen": 2865627136 + }, + { + "epoch": 0.73, + "learning_rate": 0.00013393939393939396, + "loss": 2.4958, + "theoretical_loss": 3.3325334781817064, + "tokens_seen": 2865692672 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001339191919191919, + "loss": 2.424, + "theoretical_loss": 3.332527563894596, + "tokens_seen": 2865758208 + }, + { + "epoch": 0.73, + "objective/train/docs_used": 1613208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4768993854522705, + "objective/train/theoretical_loss": 3.3325216497806056, + "objective/train/tokens_used": 1224682976, + "theoretical_loss": 3.3325216497806056, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.73, + "learning_rate": 0.0001338989898989899, + "loss": 2.7186, + "theoretical_loss": 3.3325216497806056, + "tokens_seen": 2865823744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013387878787878788, + "loss": 2.6104, + "theoretical_loss": 3.3325157358397255, + "tokens_seen": 2865889280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013385858585858587, + "loss": 2.6477, + "theoretical_loss": 3.3325098220719473, + "tokens_seen": 2865954816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013383838383838385, + "loss": 2.7237, + "theoretical_loss": 3.3325039084772614, + "tokens_seen": 2866020352 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001338181818181818, + "loss": 2.3857, + "theoretical_loss": 3.332497995055659, + "tokens_seen": 2866085888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013379797979797982, + "loss": 2.4999, + "theoretical_loss": 3.332492081807131, + "tokens_seen": 2866151424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013377777777777777, + "loss": 2.6987, + "theoretical_loss": 3.332486168731669, + "tokens_seen": 2866216960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013375757575757578, + "loss": 2.666, + "theoretical_loss": 3.332480255829263, + "tokens_seen": 2866282496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013373737373737374, + "loss": 2.6293, + "theoretical_loss": 3.332474343099905, + "tokens_seen": 2866348032 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001337171717171717, + "loss": 2.7885, + "theoretical_loss": 3.332468430543585, + "tokens_seen": 2866413568 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001336969696969697, + "loss": 2.5528, + "theoretical_loss": 3.3324625181602947, + "tokens_seen": 2866479104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013367676767676767, + "loss": 2.639, + "theoretical_loss": 3.332456605950025, + "tokens_seen": 2866544640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013365656565656568, + "loss": 2.788, + "theoretical_loss": 3.332450693912766, + "tokens_seen": 2866610176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013363636363636363, + "loss": 2.911, + "theoretical_loss": 3.33244478204851, + "tokens_seen": 2866675712 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013361616161616162, + "loss": 2.615, + "theoretical_loss": 3.3324388703572474, + "tokens_seen": 2866741248 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001335959595959596, + "loss": 2.8372, + "theoretical_loss": 3.3324329588389685, + "tokens_seen": 2866806784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013357575757575756, + "loss": 2.7921, + "theoretical_loss": 3.3324270474936655, + "tokens_seen": 2866872320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013355555555555557, + "loss": 2.8093, + "theoretical_loss": 3.3324211363213285, + "tokens_seen": 2866937856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013353535353535353, + "loss": 2.7246, + "theoretical_loss": 3.332415225321949, + "tokens_seen": 2867003392 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001335151515151515, + "loss": 2.7832, + "theoretical_loss": 3.332409314495518, + "tokens_seen": 2867068928 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001334949494949495, + "loss": 2.7337, + "theoretical_loss": 3.332403403842026, + "tokens_seen": 2867134464 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013347474747474748, + "loss": 2.7689, + "theoretical_loss": 3.332397493361464, + "tokens_seen": 2867200000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013345454545454546, + "loss": 2.6195, + "theoretical_loss": 3.3323915830538233, + "tokens_seen": 2867265536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013343434343434345, + "loss": 2.517, + "theoretical_loss": 3.332385672919095, + "tokens_seen": 2867331072 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001334141414141414, + "loss": 2.7299, + "theoretical_loss": 3.3323797629572702, + "tokens_seen": 2867396608 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1613631, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7684333324432373, + "objective/train/theoretical_loss": 3.332373853168339, + "objective/train/tokens_used": 1226321376, + "theoretical_loss": 3.332373853168339, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333939393939394, + "loss": 2.7294, + "theoretical_loss": 3.332373853168339, + "tokens_seen": 2867462144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013337373737373737, + "loss": 2.7806, + "theoretical_loss": 3.3323679435522937, + "tokens_seen": 2867527680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013335353535353536, + "loss": 2.7769, + "theoretical_loss": 3.3323620341091242, + "tokens_seen": 2867593216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013333333333333334, + "loss": 2.5552, + "theoretical_loss": 3.332356124838822, + "tokens_seen": 2867658752 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001333131313131313, + "loss": 2.8214, + "theoretical_loss": 3.3323502157413776, + "tokens_seen": 2867724288 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001332929292929293, + "loss": 2.6491, + "theoretical_loss": 3.3323443068167826, + "tokens_seen": 2867789824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013327272727272726, + "loss": 2.4725, + "theoretical_loss": 3.332338398065028, + "tokens_seen": 2867855360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013325252525252528, + "loss": 2.7459, + "theoretical_loss": 3.3323324894861046, + "tokens_seen": 2867920896 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013323232323232323, + "loss": 2.73, + "theoretical_loss": 3.332326581080003, + "tokens_seen": 2867986432 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001332121212121212, + "loss": 2.7139, + "theoretical_loss": 3.332320672846715, + "tokens_seen": 2868051968 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001331919191919192, + "loss": 2.9225, + "theoretical_loss": 3.332314764786231, + "tokens_seen": 2868117504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013317171717171716, + "loss": 2.6178, + "theoretical_loss": 3.3323088568985417, + "tokens_seen": 2868183040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013315151515151517, + "loss": 2.6145, + "theoretical_loss": 3.332302949183639, + "tokens_seen": 2868248576 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013313131313131313, + "loss": 2.8016, + "theoretical_loss": 3.3322970416415134, + "tokens_seen": 2868314112 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013311111111111114, + "loss": 2.5876, + "theoretical_loss": 3.3322911342721557, + "tokens_seen": 2868379648 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001330909090909091, + "loss": 2.666, + "theoretical_loss": 3.332285227075557, + "tokens_seen": 2868445184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013307070707070708, + "loss": 2.6701, + "theoretical_loss": 3.3322793200517093, + "tokens_seen": 2868510720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013305050505050506, + "loss": 2.8431, + "theoretical_loss": 3.332273413200602, + "tokens_seen": 2868576256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013303030303030302, + "loss": 2.525, + "theoretical_loss": 3.332267506522227, + "tokens_seen": 2868641792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013301010101010103, + "loss": 2.5804, + "theoretical_loss": 3.332261600016575, + "tokens_seen": 2868707328 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013298989898989899, + "loss": 2.4476, + "theoretical_loss": 3.3322556936836376, + "tokens_seen": 2868772864 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013296969696969697, + "loss": 2.5927, + "theoretical_loss": 3.332249787523405, + "tokens_seen": 2868838400 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013294949494949495, + "loss": 2.6463, + "theoretical_loss": 3.332243881535869, + "tokens_seen": 2868903936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013292929292929294, + "loss": 2.7214, + "theoretical_loss": 3.3322379757210197, + "tokens_seen": 2868969472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013290909090909092, + "loss": 2.615, + "theoretical_loss": 3.332232070078849, + "tokens_seen": 2869035008 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1614970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5441737174987793, + "objective/train/theoretical_loss": 3.332226164609347, + "objective/train/tokens_used": 1227959776, + "theoretical_loss": 3.332226164609347, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001328888888888889, + "loss": 2.7098, + "theoretical_loss": 3.332226164609347, + "tokens_seen": 2869100544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013286868686868686, + "loss": 2.8492, + "theoretical_loss": 3.332220259312505, + "tokens_seen": 2869166080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013284848484848485, + "loss": 2.5554, + "theoretical_loss": 3.3322143541883147, + "tokens_seen": 2869231616 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013282828282828283, + "loss": 2.7219, + "theoretical_loss": 3.3322084492367665, + "tokens_seen": 2869297152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013280808080808081, + "loss": 2.7081, + "theoretical_loss": 3.332202544457852, + "tokens_seen": 2869362688 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001327878787878788, + "loss": 2.5568, + "theoretical_loss": 3.332196639851561, + "tokens_seen": 2869428224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013276767676767676, + "loss": 2.51, + "theoretical_loss": 3.3321907354178855, + "tokens_seen": 2869493760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013274747474747477, + "loss": 2.8794, + "theoretical_loss": 3.332184831156816, + "tokens_seen": 2869559296 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013272727272727272, + "loss": 2.7367, + "theoretical_loss": 3.332178927068344, + "tokens_seen": 2869624832 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013270707070707073, + "loss": 2.6819, + "theoretical_loss": 3.3321730231524604, + "tokens_seen": 2869690368 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001326868686868687, + "loss": 2.674, + "theoretical_loss": 3.332167119409156, + "tokens_seen": 2869755904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013266666666666665, + "loss": 2.8068, + "theoretical_loss": 3.332161215838422, + "tokens_seen": 2869821440 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013264646464646466, + "loss": 2.8091, + "theoretical_loss": 3.332155312440249, + "tokens_seen": 2869886976 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013262626262626262, + "loss": 2.5275, + "theoretical_loss": 3.3321494092146287, + "tokens_seen": 2869952512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013260606060606063, + "loss": 2.7215, + "theoretical_loss": 3.332143506161551, + "tokens_seen": 2870018048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013258585858585858, + "loss": 2.684, + "theoretical_loss": 3.3321376032810086, + "tokens_seen": 2870083584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013256565656565657, + "loss": 2.583, + "theoretical_loss": 3.332131700572991, + "tokens_seen": 2870149120 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013254545454545455, + "loss": 2.7287, + "theoretical_loss": 3.3321257980374903, + "tokens_seen": 2870214656 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001325252525252525, + "loss": 2.7258, + "theoretical_loss": 3.332119895674497, + "tokens_seen": 2870280192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013250505050505052, + "loss": 2.6444, + "theoretical_loss": 3.3321139934840014, + "tokens_seen": 2870345728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013248484848484848, + "loss": 2.7185, + "theoretical_loss": 3.332108091465996, + "tokens_seen": 2870411264 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013246464646464646, + "loss": 2.5504, + "theoretical_loss": 3.3321021896204708, + "tokens_seen": 2870476800 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013244444444444444, + "loss": 2.7456, + "theoretical_loss": 3.332096287947417, + "tokens_seen": 2870542336 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013242424242424243, + "loss": 2.7221, + "theoretical_loss": 3.3320903864468256, + "tokens_seen": 2870607872 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001324040404040404, + "loss": 2.5781, + "theoretical_loss": 3.3320844851186884, + "tokens_seen": 2870673408 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1615721, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.756335735321045, + "objective/train/theoretical_loss": 3.3320785839629954, + "objective/train/tokens_used": 1229598176, + "theoretical_loss": 3.3320785839629954, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001323838383838384, + "loss": 2.6951, + "theoretical_loss": 3.3320785839629954, + "tokens_seen": 2870738944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013236363636363635, + "loss": 2.5932, + "theoretical_loss": 3.332072682979738, + "tokens_seen": 2870804480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013234343434343434, + "loss": 2.6601, + "theoretical_loss": 3.3320667821689067, + "tokens_seen": 2870870016 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013232323232323232, + "loss": 2.623, + "theoretical_loss": 3.3320608815304937, + "tokens_seen": 2870935552 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001323030303030303, + "loss": 2.5402, + "theoretical_loss": 3.3320549810644895, + "tokens_seen": 2871001088 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001322828282828283, + "loss": 2.6431, + "theoretical_loss": 3.3320490807708847, + "tokens_seen": 2871066624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013226262626262627, + "loss": 2.8052, + "theoretical_loss": 3.3320431806496704, + "tokens_seen": 2871132160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013224242424242426, + "loss": 2.8211, + "theoretical_loss": 3.3320372807008383, + "tokens_seen": 2871197696 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013222222222222221, + "loss": 2.4593, + "theoretical_loss": 3.332031380924379, + "tokens_seen": 2871263232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013220202020202022, + "loss": 2.4851, + "theoretical_loss": 3.332025481320283, + "tokens_seen": 2871328768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013218181818181818, + "loss": 2.7244, + "theoretical_loss": 3.332019581888542, + "tokens_seen": 2871394304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013216161616161617, + "loss": 2.6262, + "theoretical_loss": 3.3320136826291473, + "tokens_seen": 2871459840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013214141414141415, + "loss": 2.6539, + "theoretical_loss": 3.332007783542089, + "tokens_seen": 2871525376 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001321212121212121, + "loss": 2.7308, + "theoretical_loss": 3.332001884627359, + "tokens_seen": 2871590912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013210101010101012, + "loss": 2.6661, + "theoretical_loss": 3.331995985884948, + "tokens_seen": 2871656448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013208080808080807, + "loss": 2.7087, + "theoretical_loss": 3.331990087314847, + "tokens_seen": 2871721984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013206060606060609, + "loss": 2.7079, + "theoretical_loss": 3.3319841889170467, + "tokens_seen": 2871787520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013204040404040404, + "loss": 2.5616, + "theoretical_loss": 3.3319782906915387, + "tokens_seen": 2871853056 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013202020202020203, + "loss": 2.5593, + "theoretical_loss": 3.331972392638314, + "tokens_seen": 2871918592 + }, + { + "epoch": 0.74, + "learning_rate": 0.000132, + "loss": 2.6097, + "theoretical_loss": 3.331966494757363, + "tokens_seen": 2871984128 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013197979797979797, + "loss": 2.5253, + "theoretical_loss": 3.3319605970486776, + "tokens_seen": 2872049664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013195959595959598, + "loss": 2.8091, + "theoretical_loss": 3.331954699512248, + "tokens_seen": 2872115200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013193939393939393, + "loss": 2.4722, + "theoretical_loss": 3.3319488021480663, + "tokens_seen": 2872180736 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013191919191919192, + "loss": 2.7428, + "theoretical_loss": 3.331942904956122, + "tokens_seen": 2872246272 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318989898989899, + "loss": 2.787, + "theoretical_loss": 3.331937007936408, + "tokens_seen": 2872311808 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1616749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.709758758544922, + "objective/train/theoretical_loss": 3.331931111088914, + "objective/train/tokens_used": 1231236576, + "theoretical_loss": 3.331931111088914, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318787878787879, + "loss": 2.8686, + "theoretical_loss": 3.331931111088914, + "tokens_seen": 2872377344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013185858585858587, + "loss": 2.5517, + "theoretical_loss": 3.3319252144136318, + "tokens_seen": 2872442880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013183838383838383, + "loss": 2.5006, + "theoretical_loss": 3.3319193179105513, + "tokens_seen": 2872508416 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001318181818181818, + "loss": 2.6869, + "theoretical_loss": 3.3319134215796646, + "tokens_seen": 2872573952 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317979797979798, + "loss": 2.8061, + "theoretical_loss": 3.331907525420963, + "tokens_seen": 2872639488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013177777777777778, + "loss": 2.7361, + "theoretical_loss": 3.3319016294344364, + "tokens_seen": 2872705024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013175757575757576, + "loss": 2.6171, + "theoretical_loss": 3.3318957336200765, + "tokens_seen": 2872770560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013173737373737375, + "loss": 2.515, + "theoretical_loss": 3.331889837977874, + "tokens_seen": 2872836096 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001317171717171717, + "loss": 2.8687, + "theoretical_loss": 3.3318839425078206, + "tokens_seen": 2872901632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013169696969696972, + "loss": 2.6706, + "theoretical_loss": 3.331878047209907, + "tokens_seen": 2872967168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013167676767676767, + "loss": 2.7479, + "theoretical_loss": 3.3318721520841246, + "tokens_seen": 2873032704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013165656565656566, + "loss": 2.5831, + "theoretical_loss": 3.3318662571304634, + "tokens_seen": 2873098240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013163636363636364, + "loss": 2.521, + "theoretical_loss": 3.3318603623489156, + "tokens_seen": 2873163776 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001316161616161616, + "loss": 2.745, + "theoretical_loss": 3.3318544677394715, + "tokens_seen": 2873229312 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001315959595959596, + "loss": 2.5615, + "theoretical_loss": 3.3318485733021226, + "tokens_seen": 2873294848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013157575757575756, + "loss": 2.6789, + "theoretical_loss": 3.3318426790368596, + "tokens_seen": 2873360384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013155555555555558, + "loss": 2.5927, + "theoretical_loss": 3.3318367849436736, + "tokens_seen": 2873425920 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013153535353535353, + "loss": 2.5597, + "theoretical_loss": 3.3318308910225563, + "tokens_seen": 2873491456 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013151515151515152, + "loss": 2.4991, + "theoretical_loss": 3.3318249972734977, + "tokens_seen": 2873556992 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001314949494949495, + "loss": 2.7545, + "theoretical_loss": 3.3318191036964895, + "tokens_seen": 2873622528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013147474747474746, + "loss": 2.6419, + "theoretical_loss": 3.3318132102915228, + "tokens_seen": 2873688064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013145454545454547, + "loss": 2.7488, + "theoretical_loss": 3.3318073170585882, + "tokens_seen": 2873753600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013143434343434343, + "loss": 2.5346, + "theoretical_loss": 3.3318014239976774, + "tokens_seen": 2873819136 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013141414141414144, + "loss": 2.4931, + "theoretical_loss": 3.331795531108781, + "tokens_seen": 2873884672 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001313939393939394, + "loss": 2.4976, + "theoretical_loss": 3.33178963839189, + "tokens_seen": 2873950208 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1617455, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0157363414764404, + "objective/train/theoretical_loss": 3.331783745846996, + "objective/train/tokens_used": 1232874976, + "theoretical_loss": 3.331783745846996, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013137373737373738, + "loss": 2.471, + "theoretical_loss": 3.331783745846996, + "tokens_seen": 2874015744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013135353535353536, + "loss": 2.7407, + "theoretical_loss": 3.3317778534740894, + "tokens_seen": 2874081280 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013133333333333335, + "loss": 2.6806, + "theoretical_loss": 3.3317719612731613, + "tokens_seen": 2874146816 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013131313131313133, + "loss": 2.7318, + "theoretical_loss": 3.3317660692442033, + "tokens_seen": 2874212352 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013129292929292929, + "loss": 2.7903, + "theoretical_loss": 3.331760177387206, + "tokens_seen": 2874277888 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013127272727272727, + "loss": 2.5763, + "theoretical_loss": 3.3317542857021607, + "tokens_seen": 2874343424 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013125252525252525, + "loss": 2.7925, + "theoretical_loss": 3.3317483941890584, + "tokens_seen": 2874408960 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013123232323232324, + "loss": 2.6661, + "theoretical_loss": 3.3317425028478898, + "tokens_seen": 2874474496 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013121212121212122, + "loss": 2.647, + "theoretical_loss": 3.3317366116786467, + "tokens_seen": 2874540032 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001311919191919192, + "loss": 2.6932, + "theoretical_loss": 3.3317307206813194, + "tokens_seen": 2874605568 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013117171717171716, + "loss": 2.626, + "theoretical_loss": 3.3317248298559, + "tokens_seen": 2874671104 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013115151515151517, + "loss": 2.709, + "theoretical_loss": 3.3317189392023785, + "tokens_seen": 2874736640 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013113131313131313, + "loss": 2.5677, + "theoretical_loss": 3.3317130487207463, + "tokens_seen": 2874802176 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013111111111111111, + "loss": 2.5757, + "theoretical_loss": 3.3317071584109943, + "tokens_seen": 2874867712 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001310909090909091, + "loss": 2.5746, + "theoretical_loss": 3.331701268273114, + "tokens_seen": 2874933248 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013107070707070706, + "loss": 2.7227, + "theoretical_loss": 3.331695378307096, + "tokens_seen": 2874998784 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013105050505050507, + "loss": 2.6629, + "theoretical_loss": 3.3316894885129322, + "tokens_seen": 2875064320 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013103030303030302, + "loss": 2.5961, + "theoretical_loss": 3.3316835988906126, + "tokens_seen": 2875129856 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013101010101010103, + "loss": 2.6392, + "theoretical_loss": 3.331677709440129, + "tokens_seen": 2875195392 + }, + { + "epoch": 0.74, + "learning_rate": 0.000130989898989899, + "loss": 2.7146, + "theoretical_loss": 3.331671820161472, + "tokens_seen": 2875260928 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013096969696969695, + "loss": 2.6811, + "theoretical_loss": 3.3316659310546335, + "tokens_seen": 2875326464 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013094949494949496, + "loss": 2.6537, + "theoretical_loss": 3.331660042119603, + "tokens_seen": 2875392000 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013092929292929292, + "loss": 2.5076, + "theoretical_loss": 3.3316541533563733, + "tokens_seen": 2875457536 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013090909090909093, + "loss": 2.8802, + "theoretical_loss": 3.331648264764935, + "tokens_seen": 2875523072 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013088888888888888, + "loss": 2.741, + "theoretical_loss": 3.331642376345278, + "tokens_seen": 2875588608 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1618853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.385054588317871, + "objective/train/theoretical_loss": 3.3316364880973945, + "objective/train/tokens_used": 1234513376, + "theoretical_loss": 3.3316364880973945, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013086868686868687, + "loss": 2.4003, + "theoretical_loss": 3.3316364880973945, + "tokens_seen": 2875654144 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013084848484848485, + "loss": 2.6425, + "theoretical_loss": 3.331630600021276, + "tokens_seen": 2875719680 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013082828282828284, + "loss": 2.6571, + "theoretical_loss": 3.331624712116912, + "tokens_seen": 2875785216 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013080808080808082, + "loss": 2.4653, + "theoretical_loss": 3.331618824384295, + "tokens_seen": 2875850752 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013078787878787878, + "loss": 2.3626, + "theoretical_loss": 3.331612936823415, + "tokens_seen": 2875916288 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013076767676767676, + "loss": 2.6069, + "theoretical_loss": 3.3316070494342647, + "tokens_seen": 2875981824 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013074747474747474, + "loss": 2.6076, + "theoretical_loss": 3.3316011622168333, + "tokens_seen": 2876047360 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013072727272727273, + "loss": 2.4758, + "theoretical_loss": 3.3315952751711126, + "tokens_seen": 2876112896 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001307070707070707, + "loss": 2.5926, + "theoretical_loss": 3.331589388297094, + "tokens_seen": 2876178432 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306868686868687, + "loss": 2.4931, + "theoretical_loss": 3.3315835015947686, + "tokens_seen": 2876243968 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013066666666666665, + "loss": 2.521, + "theoretical_loss": 3.331577615064127, + "tokens_seen": 2876309504 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013064646464646466, + "loss": 2.6388, + "theoretical_loss": 3.331571728705161, + "tokens_seen": 2876375040 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013062626262626262, + "loss": 2.7792, + "theoretical_loss": 3.3315658425178607, + "tokens_seen": 2876440576 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001306060606060606, + "loss": 2.6247, + "theoretical_loss": 3.3315599565022174, + "tokens_seen": 2876506112 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001305858585858586, + "loss": 2.4937, + "theoretical_loss": 3.331554070658223, + "tokens_seen": 2876571648 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013056565656565657, + "loss": 2.6889, + "theoretical_loss": 3.331548184985868, + "tokens_seen": 2876637184 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013054545454545456, + "loss": 2.6486, + "theoretical_loss": 3.3315422994851436, + "tokens_seen": 2876702720 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013052525252525251, + "loss": 2.4459, + "theoretical_loss": 3.331536414156041, + "tokens_seen": 2876768256 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013050505050505053, + "loss": 2.6849, + "theoretical_loss": 3.3315305289985506, + "tokens_seen": 2876833792 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013048484848484848, + "loss": 2.6162, + "theoretical_loss": 3.331524644012664, + "tokens_seen": 2876899328 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001304646464646465, + "loss": 2.609, + "theoretical_loss": 3.3315187591983726, + "tokens_seen": 2876964864 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013044444444444445, + "loss": 2.5842, + "theoretical_loss": 3.331512874555667, + "tokens_seen": 2877030400 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001304242424242424, + "loss": 2.4832, + "theoretical_loss": 3.3315069900845384, + "tokens_seen": 2877095936 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013040404040404042, + "loss": 2.7323, + "theoretical_loss": 3.331501105784978, + "tokens_seen": 2877161472 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013038383838383837, + "loss": 2.6877, + "theoretical_loss": 3.3314952216569766, + "tokens_seen": 2877227008 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1619565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3675901889801025, + "objective/train/theoretical_loss": 3.3314893377005257, + "objective/train/tokens_used": 1236151776, + "theoretical_loss": 3.3314893377005257, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013036363636363639, + "loss": 2.5775, + "theoretical_loss": 3.3314893377005257, + "tokens_seen": 2877292544 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013034343434343434, + "loss": 2.5599, + "theoretical_loss": 3.3314834539156166, + "tokens_seen": 2877358080 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013032323232323233, + "loss": 2.738, + "theoretical_loss": 3.3314775703022397, + "tokens_seen": 2877423616 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001303030303030303, + "loss": 2.409, + "theoretical_loss": 3.3314716868603864, + "tokens_seen": 2877489152 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013028282828282827, + "loss": 2.5555, + "theoretical_loss": 3.331465803590048, + "tokens_seen": 2877554688 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013026262626262628, + "loss": 2.816, + "theoretical_loss": 3.331459920491215, + "tokens_seen": 2877620224 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013024242424242424, + "loss": 2.5211, + "theoretical_loss": 3.331454037563879, + "tokens_seen": 2877685760 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013022222222222222, + "loss": 2.53, + "theoretical_loss": 3.3314481548080312, + "tokens_seen": 2877751296 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001302020202020202, + "loss": 2.728, + "theoretical_loss": 3.3314422722236623, + "tokens_seen": 2877816832 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301818181818182, + "loss": 2.5885, + "theoretical_loss": 3.3314363898107637, + "tokens_seen": 2877882368 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013016161616161617, + "loss": 2.7598, + "theoretical_loss": 3.3314305075693262, + "tokens_seen": 2877947904 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013014141414141416, + "loss": 2.5071, + "theoretical_loss": 3.3314246254993414, + "tokens_seen": 2878013440 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301212121212121, + "loss": 2.6016, + "theoretical_loss": 3.3314187436008, + "tokens_seen": 2878078976 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001301010101010101, + "loss": 2.3047, + "theoretical_loss": 3.3314128618736927, + "tokens_seen": 2878144512 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013008080808080808, + "loss": 2.6225, + "theoretical_loss": 3.3314069803180115, + "tokens_seen": 2878210048 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013006060606060606, + "loss": 2.5757, + "theoretical_loss": 3.331401098933747, + "tokens_seen": 2878275584 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013004040404040405, + "loss": 2.6711, + "theoretical_loss": 3.33139521772089, + "tokens_seen": 2878341120 + }, + { + "epoch": 0.74, + "learning_rate": 0.000130020202020202, + "loss": 2.8406, + "theoretical_loss": 3.3313893366794325, + "tokens_seen": 2878406656 + }, + { + "epoch": 0.74, + "learning_rate": 0.00013000000000000002, + "loss": 2.3939, + "theoretical_loss": 3.3313834558093647, + "tokens_seen": 2878472192 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012997979797979797, + "loss": 2.5627, + "theoretical_loss": 3.3313775751106784, + "tokens_seen": 2878537728 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012995959595959598, + "loss": 2.5535, + "theoretical_loss": 3.331371694583364, + "tokens_seen": 2878603264 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012993939393939394, + "loss": 2.3711, + "theoretical_loss": 3.3313658142274134, + "tokens_seen": 2878668800 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001299191919191919, + "loss": 2.2984, + "theoretical_loss": 3.331359934042817, + "tokens_seen": 2878734336 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001298989898989899, + "loss": 2.5927, + "theoretical_loss": 3.3313540540295663, + "tokens_seen": 2878799872 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012987878787878787, + "loss": 2.6245, + "theoretical_loss": 3.331348174187652, + "tokens_seen": 2878865408 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1621029, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.509866237640381, + "objective/train/theoretical_loss": 3.331342294517066, + "objective/train/tokens_used": 1237790176, + "theoretical_loss": 3.331342294517066, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012985858585858588, + "loss": 2.6354, + "theoretical_loss": 3.331342294517066, + "tokens_seen": 2878930944 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012983838383838383, + "loss": 2.5311, + "theoretical_loss": 3.3313364150177986, + "tokens_seen": 2878996480 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012981818181818182, + "loss": 2.6407, + "theoretical_loss": 3.3313305356898413, + "tokens_seen": 2879062016 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001297979797979798, + "loss": 2.6622, + "theoretical_loss": 3.3313246565331855, + "tokens_seen": 2879127552 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012977777777777779, + "loss": 2.6628, + "theoretical_loss": 3.3313187775478212, + "tokens_seen": 2879193088 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012975757575757577, + "loss": 2.4871, + "theoretical_loss": 3.331312898733741, + "tokens_seen": 2879258624 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012973737373737373, + "loss": 2.6114, + "theoretical_loss": 3.331307020090935, + "tokens_seen": 2879324160 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012971717171717174, + "loss": 2.6374, + "theoretical_loss": 3.3313011416193943, + "tokens_seen": 2879389696 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001296969696969697, + "loss": 2.4778, + "theoretical_loss": 3.3312952633191104, + "tokens_seen": 2879455232 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012967676767676768, + "loss": 2.5491, + "theoretical_loss": 3.3312893851900744, + "tokens_seen": 2879520768 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012965656565656566, + "loss": 2.4814, + "theoretical_loss": 3.3312835072322775, + "tokens_seen": 2879586304 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012963636363636365, + "loss": 2.5842, + "theoretical_loss": 3.3312776294457103, + "tokens_seen": 2879651840 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012961616161616163, + "loss": 2.7459, + "theoretical_loss": 3.3312717518303643, + "tokens_seen": 2879717376 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012959595959595961, + "loss": 2.7187, + "theoretical_loss": 3.3312658743862307, + "tokens_seen": 2879782912 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012957575757575757, + "loss": 2.408, + "theoretical_loss": 3.3312599971133006, + "tokens_seen": 2879848448 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012955555555555555, + "loss": 2.4996, + "theoretical_loss": 3.3312541200115646, + "tokens_seen": 2879913984 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012953535353535354, + "loss": 2.6858, + "theoretical_loss": 3.3312482430810144, + "tokens_seen": 2879979520 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012951515151515152, + "loss": 2.5175, + "theoretical_loss": 3.331242366321641, + "tokens_seen": 2880045056 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001294949494949495, + "loss": 2.4443, + "theoretical_loss": 3.3312364897334357, + "tokens_seen": 2880110592 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012947474747474746, + "loss": 2.3894, + "theoretical_loss": 3.3312306133163894, + "tokens_seen": 2880176128 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012945454545454547, + "loss": 2.5392, + "theoretical_loss": 3.3312247370704924, + "tokens_seen": 2880241664 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012943434343434343, + "loss": 2.5543, + "theoretical_loss": 3.3312188609957376, + "tokens_seen": 2880307200 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012941414141414142, + "loss": 2.6629, + "theoretical_loss": 3.3312129850921144, + "tokens_seen": 2880372736 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001293939393939394, + "loss": 2.417, + "theoretical_loss": 3.3312071093596147, + "tokens_seen": 2880438272 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012937373737373736, + "loss": 2.4645, + "theoretical_loss": 3.3312012337982297, + "tokens_seen": 2880503808 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1621719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6413655281066895, + "objective/train/theoretical_loss": 3.3311953584079506, + "objective/train/tokens_used": 1239428576, + "theoretical_loss": 3.3311953584079506, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012935353535353537, + "loss": 2.6773, + "theoretical_loss": 3.3311953584079506, + "tokens_seen": 2880569344 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012933333333333332, + "loss": 2.3999, + "theoretical_loss": 3.3311894831887683, + "tokens_seen": 2880634880 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012931313131313133, + "loss": 2.6283, + "theoretical_loss": 3.331183608140674, + "tokens_seen": 2880700416 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001292929292929293, + "loss": 2.399, + "theoretical_loss": 3.331177733263659, + "tokens_seen": 2880765952 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012927272727272728, + "loss": 2.628, + "theoretical_loss": 3.3311718585577137, + "tokens_seen": 2880831488 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012925252525252526, + "loss": 2.4436, + "theoretical_loss": 3.33116598402283, + "tokens_seen": 2880897024 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012923232323232322, + "loss": 2.7337, + "theoretical_loss": 3.3311601096589984, + "tokens_seen": 2880962560 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012921212121212123, + "loss": 2.7011, + "theoretical_loss": 3.3311542354662107, + "tokens_seen": 2881028096 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012919191919191918, + "loss": 2.6508, + "theoretical_loss": 3.3311483614444577, + "tokens_seen": 2881093632 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012917171717171717, + "loss": 2.5568, + "theoretical_loss": 3.3311424875937306, + "tokens_seen": 2881159168 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012915151515151515, + "loss": 2.4353, + "theoretical_loss": 3.3311366139140204, + "tokens_seen": 2881224704 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012913131313131314, + "loss": 2.5129, + "theoretical_loss": 3.3311307404053183, + "tokens_seen": 2881290240 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012911111111111112, + "loss": 2.7299, + "theoretical_loss": 3.3311248670676155, + "tokens_seen": 2881355776 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001290909090909091, + "loss": 2.614, + "theoretical_loss": 3.331118993900903, + "tokens_seen": 2881421312 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012907070707070706, + "loss": 2.6518, + "theoretical_loss": 3.331113120905172, + "tokens_seen": 2881486848 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012905050505050505, + "loss": 2.7182, + "theoretical_loss": 3.331107248080414, + "tokens_seen": 2881552384 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012903030303030303, + "loss": 2.7678, + "theoretical_loss": 3.331101375426619, + "tokens_seen": 2881617920 + }, + { + "epoch": 0.74, + "learning_rate": 0.000129010101010101, + "loss": 2.408, + "theoretical_loss": 3.3310955029437794, + "tokens_seen": 2881683456 + }, + { + "epoch": 0.74, + "learning_rate": 0.000128989898989899, + "loss": 2.6352, + "theoretical_loss": 3.331089630631886, + "tokens_seen": 2881748992 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012896969696969695, + "loss": 2.55, + "theoretical_loss": 3.3310837584909296, + "tokens_seen": 2881814528 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012894949494949496, + "loss": 2.6083, + "theoretical_loss": 3.3310778865209016, + "tokens_seen": 2881880064 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012892929292929292, + "loss": 2.6295, + "theoretical_loss": 3.331072014721793, + "tokens_seen": 2881945600 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012890909090909093, + "loss": 2.6934, + "theoretical_loss": 3.3310661430935946, + "tokens_seen": 2882011136 + }, + { + "epoch": 0.74, + "learning_rate": 0.0001288888888888889, + "loss": 2.6507, + "theoretical_loss": 3.3310602716362983, + "tokens_seen": 2882076672 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012886868686868687, + "loss": 2.7015, + "theoretical_loss": 3.331054400349895, + "tokens_seen": 2882142208 + }, + { + "epoch": 0.74, + "objective/train/docs_used": 1622728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1332671642303467, + "objective/train/theoretical_loss": 3.331048529234376, + "objective/train/tokens_used": 1241066976, + "theoretical_loss": 3.331048529234376, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.74, + "learning_rate": 0.00012884848484848486, + "loss": 2.4693, + "theoretical_loss": 3.331048529234376, + "tokens_seen": 2882207744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012882828282828281, + "loss": 2.4284, + "theoretical_loss": 3.3310426582897312, + "tokens_seen": 2882273280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012880808080808083, + "loss": 2.4727, + "theoretical_loss": 3.3310367875159534, + "tokens_seen": 2882338816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012878787878787878, + "loss": 2.7873, + "theoretical_loss": 3.3310309169130328, + "tokens_seen": 2882404352 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287676767676768, + "loss": 2.5856, + "theoretical_loss": 3.331025046480961, + "tokens_seen": 2882469888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012874747474747475, + "loss": 2.7144, + "theoretical_loss": 3.331019176219729, + "tokens_seen": 2882535424 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001287272727272727, + "loss": 2.7425, + "theoretical_loss": 3.3310133061293277, + "tokens_seen": 2882600960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012870707070707072, + "loss": 2.6591, + "theoretical_loss": 3.3310074362097484, + "tokens_seen": 2882666496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012868686868686868, + "loss": 2.2944, + "theoretical_loss": 3.331001566460982, + "tokens_seen": 2882732032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012866666666666669, + "loss": 2.5686, + "theoretical_loss": 3.3309956968830203, + "tokens_seen": 2882797568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012864646464646464, + "loss": 2.8058, + "theoretical_loss": 3.330989827475854, + "tokens_seen": 2882863104 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012862626262626263, + "loss": 2.5303, + "theoretical_loss": 3.3309839582394742, + "tokens_seen": 2882928640 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001286060606060606, + "loss": 2.5409, + "theoretical_loss": 3.330978089173872, + "tokens_seen": 2882994176 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285858585858586, + "loss": 2.6281, + "theoretical_loss": 3.330972220279039, + "tokens_seen": 2883059712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012856565656565658, + "loss": 2.5007, + "theoretical_loss": 3.330966351554966, + "tokens_seen": 2883125248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012854545454545454, + "loss": 2.5555, + "theoretical_loss": 3.330960483001644, + "tokens_seen": 2883190784 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012852525252525252, + "loss": 2.7489, + "theoretical_loss": 3.3309546146190647, + "tokens_seen": 2883256320 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001285050505050505, + "loss": 2.4208, + "theoretical_loss": 3.3309487464072185, + "tokens_seen": 2883321856 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001284848484848485, + "loss": 2.5686, + "theoretical_loss": 3.330942878366097, + "tokens_seen": 2883387392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012846464646464647, + "loss": 2.5836, + "theoretical_loss": 3.330937010495692, + "tokens_seen": 2883452928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012844444444444446, + "loss": 2.1725, + "theoretical_loss": 3.330931142795993, + "tokens_seen": 2883518464 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001284242424242424, + "loss": 2.4875, + "theoretical_loss": 3.3309252752669924, + "tokens_seen": 2883584000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012840404040404042, + "loss": 2.757, + "theoretical_loss": 3.3309194079086812, + "tokens_seen": 2883649536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012838383838383838, + "loss": 2.6457, + "theoretical_loss": 3.3309135407210504, + "tokens_seen": 2883715072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012836363636363636, + "loss": 2.6592, + "theoretical_loss": 3.3309076737040915, + "tokens_seen": 2883780608 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1623211, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9575917720794678, + "objective/train/theoretical_loss": 3.3309018068577947, + "objective/train/tokens_used": 1242705376, + "theoretical_loss": 3.3309018068577947, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012834343434343435, + "loss": 2.6186, + "theoretical_loss": 3.3309018068577947, + "tokens_seen": 2883846144 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001283232323232323, + "loss": 2.7673, + "theoretical_loss": 3.330895940182152, + "tokens_seen": 2883911680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012830303030303032, + "loss": 2.3844, + "theoretical_loss": 3.3308900736771547, + "tokens_seen": 2883977216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012828282828282827, + "loss": 2.6296, + "theoretical_loss": 3.330884207342794, + "tokens_seen": 2884042752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012826262626262628, + "loss": 2.6093, + "theoretical_loss": 3.3308783411790595, + "tokens_seen": 2884108288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012824242424242424, + "loss": 2.3015, + "theoretical_loss": 3.330872475185944, + "tokens_seen": 2884173824 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012822222222222222, + "loss": 2.6393, + "theoretical_loss": 3.3308666093634383, + "tokens_seen": 2884239360 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001282020202020202, + "loss": 2.4861, + "theoretical_loss": 3.3308607437115336, + "tokens_seen": 2884304896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012818181818181817, + "loss": 2.3972, + "theoretical_loss": 3.3308548782302205, + "tokens_seen": 2884370432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012816161616161618, + "loss": 2.4496, + "theoretical_loss": 3.330849012919491, + "tokens_seen": 2884435968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012814141414141413, + "loss": 2.5344, + "theoretical_loss": 3.3308431477793357, + "tokens_seen": 2884501504 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012812121212121212, + "loss": 2.558, + "theoretical_loss": 3.330837282809746, + "tokens_seen": 2884567040 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001281010101010101, + "loss": 2.6515, + "theoretical_loss": 3.330831418010713, + "tokens_seen": 2884632576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012808080808080809, + "loss": 2.3782, + "theoretical_loss": 3.3308255533822275, + "tokens_seen": 2884698112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012806060606060607, + "loss": 2.5287, + "theoretical_loss": 3.330819688924281, + "tokens_seen": 2884763648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012804040404040405, + "loss": 2.6058, + "theoretical_loss": 3.330813824636865, + "tokens_seen": 2884829184 + }, + { + "epoch": 0.75, + "learning_rate": 0.000128020202020202, + "loss": 2.3718, + "theoretical_loss": 3.3308079605199703, + "tokens_seen": 2884894720 + }, + { + "epoch": 0.75, + "learning_rate": 0.000128, + "loss": 2.502, + "theoretical_loss": 3.330802096573588, + "tokens_seen": 2884960256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012797979797979798, + "loss": 2.4382, + "theoretical_loss": 3.3307962327977094, + "tokens_seen": 2885025792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012795959595959596, + "loss": 2.6464, + "theoretical_loss": 3.330790369192326, + "tokens_seen": 2885091328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012793939393939395, + "loss": 2.5946, + "theoretical_loss": 3.330784505757428, + "tokens_seen": 2885156864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012791919191919193, + "loss": 2.4628, + "theoretical_loss": 3.3307786424930073, + "tokens_seen": 2885222400 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012789898989898991, + "loss": 2.45, + "theoretical_loss": 3.330772779399055, + "tokens_seen": 2885287936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012787878787878787, + "loss": 2.5514, + "theoretical_loss": 3.3307669164755627, + "tokens_seen": 2885353472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012785858585858588, + "loss": 2.6035, + "theoretical_loss": 3.3307610537225205, + "tokens_seen": 2885419008 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1624561, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5527210235595703, + "objective/train/theoretical_loss": 3.3307551911399207, + "objective/train/tokens_used": 1244343776, + "theoretical_loss": 3.3307551911399207, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012783838383838384, + "loss": 2.3889, + "theoretical_loss": 3.3307551911399207, + "tokens_seen": 2885484544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012781818181818182, + "loss": 2.7109, + "theoretical_loss": 3.3307493287277534, + "tokens_seen": 2885550080 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001277979797979798, + "loss": 2.5938, + "theoretical_loss": 3.3307434664860107, + "tokens_seen": 2885615616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012777777777777776, + "loss": 2.4783, + "theoretical_loss": 3.3307376044146833, + "tokens_seen": 2885681152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012775757575757577, + "loss": 2.6178, + "theoretical_loss": 3.330731742513762, + "tokens_seen": 2885746688 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012773737373737373, + "loss": 2.6901, + "theoretical_loss": 3.3307258807832394, + "tokens_seen": 2885812224 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012771717171717174, + "loss": 2.491, + "theoretical_loss": 3.3307200192231052, + "tokens_seen": 2885877760 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001276969696969697, + "loss": 2.3455, + "theoretical_loss": 3.330714157833351, + "tokens_seen": 2885943296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012767676767676766, + "loss": 2.6935, + "theoretical_loss": 3.330708296613968, + "tokens_seen": 2886008832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012765656565656567, + "loss": 2.4235, + "theoretical_loss": 3.3307024355649477, + "tokens_seen": 2886074368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012763636363636362, + "loss": 2.4761, + "theoretical_loss": 3.330696574686281, + "tokens_seen": 2886139904 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012761616161616164, + "loss": 2.7166, + "theoretical_loss": 3.330690713977959, + "tokens_seen": 2886205440 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001275959595959596, + "loss": 2.6014, + "theoretical_loss": 3.3306848534399736, + "tokens_seen": 2886270976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012757575757575758, + "loss": 2.463, + "theoretical_loss": 3.330678993072315, + "tokens_seen": 2886336512 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012755555555555556, + "loss": 2.473, + "theoretical_loss": 3.330673132874974, + "tokens_seen": 2886402048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012753535353535354, + "loss": 2.6609, + "theoretical_loss": 3.3306672728479434, + "tokens_seen": 2886467584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012751515151515153, + "loss": 2.6032, + "theoretical_loss": 3.3306614129912138, + "tokens_seen": 2886533120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012749494949494948, + "loss": 2.6023, + "theoretical_loss": 3.3306555533047755, + "tokens_seen": 2886598656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012747474747474747, + "loss": 2.6298, + "theoretical_loss": 3.3306496937886205, + "tokens_seen": 2886664192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012745454545454545, + "loss": 2.4523, + "theoretical_loss": 3.3306438344427396, + "tokens_seen": 2886729728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012743434343434344, + "loss": 2.5098, + "theoretical_loss": 3.3306379752671247, + "tokens_seen": 2886795264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012741414141414142, + "loss": 2.7576, + "theoretical_loss": 3.330632116261766, + "tokens_seen": 2886860800 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273939393939394, + "loss": 2.6399, + "theoretical_loss": 3.330626257426655, + "tokens_seen": 2886926336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012737373737373736, + "loss": 2.508, + "theoretical_loss": 3.3306203987617833, + "tokens_seen": 2886991872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012735353535353537, + "loss": 2.4085, + "theoretical_loss": 3.330614540267142, + "tokens_seen": 2887057408 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1624832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4979453086853027, + "objective/train/theoretical_loss": 3.330608681942722, + "objective/train/tokens_used": 1245982176, + "theoretical_loss": 3.330608681942722, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012733333333333333, + "loss": 2.6659, + "theoretical_loss": 3.330608681942722, + "tokens_seen": 2887122944 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001273131313131313, + "loss": 2.5279, + "theoretical_loss": 3.3306028237885146, + "tokens_seen": 2887188480 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001272929292929293, + "loss": 2.3952, + "theoretical_loss": 3.330596965804511, + "tokens_seen": 2887254016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012727272727272725, + "loss": 2.5216, + "theoretical_loss": 3.3305911079907027, + "tokens_seen": 2887319552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012725252525252527, + "loss": 2.3524, + "theoretical_loss": 3.33058525034708, + "tokens_seen": 2887385088 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012723232323232322, + "loss": 2.4415, + "theoretical_loss": 3.330579392873635, + "tokens_seen": 2887450624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012721212121212123, + "loss": 2.6323, + "theoretical_loss": 3.3305735355703585, + "tokens_seen": 2887516160 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001271919191919192, + "loss": 2.5683, + "theoretical_loss": 3.3305676784372418, + "tokens_seen": 2887581696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012717171717171717, + "loss": 2.4932, + "theoretical_loss": 3.330561821474276, + "tokens_seen": 2887647232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012715151515151516, + "loss": 2.594, + "theoretical_loss": 3.3305559646814524, + "tokens_seen": 2887712768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012713131313131311, + "loss": 2.5857, + "theoretical_loss": 3.3305501080587625, + "tokens_seen": 2887778304 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012711111111111113, + "loss": 2.5564, + "theoretical_loss": 3.3305442516061965, + "tokens_seen": 2887843840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012709090909090908, + "loss": 2.7536, + "theoretical_loss": 3.330538395323747, + "tokens_seen": 2887909376 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001270707070707071, + "loss": 2.465, + "theoretical_loss": 3.3305325392114034, + "tokens_seen": 2887974912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012705050505050505, + "loss": 2.7017, + "theoretical_loss": 3.3305266832691585, + "tokens_seen": 2888040448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012703030303030303, + "loss": 2.6236, + "theoretical_loss": 3.3305208274970033, + "tokens_seen": 2888105984 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012701010101010102, + "loss": 2.5842, + "theoretical_loss": 3.330514971894928, + "tokens_seen": 2888171520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012698989898989898, + "loss": 2.337, + "theoretical_loss": 3.3305091164629252, + "tokens_seen": 2888237056 + }, + { + "epoch": 0.75, + "learning_rate": 0.000126969696969697, + "loss": 2.4975, + "theoretical_loss": 3.3305032612009846, + "tokens_seen": 2888302592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012694949494949494, + "loss": 2.5985, + "theoretical_loss": 3.3304974061090986, + "tokens_seen": 2888368128 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012692929292929293, + "loss": 2.5536, + "theoretical_loss": 3.330491551187258, + "tokens_seen": 2888433664 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001269090909090909, + "loss": 2.5413, + "theoretical_loss": 3.3304856964354537, + "tokens_seen": 2888499200 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001268888888888889, + "loss": 2.5836, + "theoretical_loss": 3.330479841853677, + "tokens_seen": 2888564736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012686868686868688, + "loss": 2.5321, + "theoretical_loss": 3.33047398744192, + "tokens_seen": 2888630272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012684848484848486, + "loss": 2.6683, + "theoretical_loss": 3.3304681332001724, + "tokens_seen": 2888695808 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1626153, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.086974620819092, + "objective/train/theoretical_loss": 3.3304622791284264, + "objective/train/tokens_used": 1247620576, + "theoretical_loss": 3.3304622791284264, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012682828282828282, + "loss": 2.7019, + "theoretical_loss": 3.3304622791284264, + "tokens_seen": 2888761344 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001268080808080808, + "loss": 2.6821, + "theoretical_loss": 3.3304564252266733, + "tokens_seen": 2888826880 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001267878787878788, + "loss": 2.5883, + "theoretical_loss": 3.3304505714949033, + "tokens_seen": 2888892416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012676767676767677, + "loss": 2.4863, + "theoretical_loss": 3.330444717933109, + "tokens_seen": 2888957952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012674747474747476, + "loss": 2.6559, + "theoretical_loss": 3.3304388645412804, + "tokens_seen": 2889023488 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001267272727272727, + "loss": 2.486, + "theoretical_loss": 3.3304330113194096, + "tokens_seen": 2889089024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012670707070707072, + "loss": 2.5473, + "theoretical_loss": 3.3304271582674874, + "tokens_seen": 2889154560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012668686868686868, + "loss": 2.7428, + "theoretical_loss": 3.3304213053855047, + "tokens_seen": 2889220096 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266666666666667, + "loss": 2.7376, + "theoretical_loss": 3.3304154526734533, + "tokens_seen": 2889285632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012664646464646465, + "loss": 2.2397, + "theoretical_loss": 3.330409600131324, + "tokens_seen": 2889351168 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001266262626262626, + "loss": 2.6822, + "theoretical_loss": 3.3304037477591084, + "tokens_seen": 2889416704 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012660606060606062, + "loss": 2.6561, + "theoretical_loss": 3.330397895556797, + "tokens_seen": 2889482240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012658585858585857, + "loss": 2.3348, + "theoretical_loss": 3.330392043524382, + "tokens_seen": 2889547776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012656565656565658, + "loss": 2.6556, + "theoretical_loss": 3.3303861916618542, + "tokens_seen": 2889613312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012654545454545454, + "loss": 2.5778, + "theoretical_loss": 3.3303803399692047, + "tokens_seen": 2889678848 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012652525252525253, + "loss": 2.3083, + "theoretical_loss": 3.3303744884464246, + "tokens_seen": 2889744384 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001265050505050505, + "loss": 2.5072, + "theoretical_loss": 3.330368637093505, + "tokens_seen": 2889809920 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001264848484848485, + "loss": 2.4373, + "theoretical_loss": 3.3303627859104377, + "tokens_seen": 2889875456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012646464646464648, + "loss": 2.3741, + "theoretical_loss": 3.3303569348972135, + "tokens_seen": 2889940992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012644444444444443, + "loss": 2.6862, + "theoretical_loss": 3.3303510840538237, + "tokens_seen": 2890006528 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012642424242424242, + "loss": 2.4187, + "theoretical_loss": 3.3303452333802595, + "tokens_seen": 2890072064 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001264040404040404, + "loss": 2.7438, + "theoretical_loss": 3.330339382876512, + "tokens_seen": 2890137600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012638383838383839, + "loss": 2.7608, + "theoretical_loss": 3.330333532542573, + "tokens_seen": 2890203136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012636363636363637, + "loss": 2.6634, + "theoretical_loss": 3.330327682378433, + "tokens_seen": 2890268672 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012634343434343435, + "loss": 2.4646, + "theoretical_loss": 3.330321832384084, + "tokens_seen": 2890334208 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1626814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7551934719085693, + "objective/train/theoretical_loss": 3.330315982559516, + "objective/train/tokens_used": 1249258976, + "theoretical_loss": 3.330315982559516, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001263232323232323, + "loss": 2.8095, + "theoretical_loss": 3.330315982559516, + "tokens_seen": 2890399744 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012630303030303032, + "loss": 2.4049, + "theoretical_loss": 3.3303101329047213, + "tokens_seen": 2890465280 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012628282828282828, + "loss": 2.7132, + "theoretical_loss": 3.3303042834196908, + "tokens_seen": 2890530816 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012626262626262626, + "loss": 2.8316, + "theoretical_loss": 3.3302984341044155, + "tokens_seen": 2890596352 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012624242424242425, + "loss": 2.4435, + "theoretical_loss": 3.330292584958887, + "tokens_seen": 2890661888 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012622222222222223, + "loss": 2.713, + "theoretical_loss": 3.3302867359830968, + "tokens_seen": 2890727424 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012620202020202021, + "loss": 2.6798, + "theoretical_loss": 3.330280887177035, + "tokens_seen": 2890792960 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012618181818181817, + "loss": 2.3816, + "theoretical_loss": 3.330275038540694, + "tokens_seen": 2890858496 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012616161616161618, + "loss": 2.7447, + "theoretical_loss": 3.3302691900740644, + "tokens_seen": 2890924032 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012614141414141414, + "loss": 2.7341, + "theoretical_loss": 3.3302633417771377, + "tokens_seen": 2890989568 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012612121212121212, + "loss": 2.5592, + "theoretical_loss": 3.3302574936499045, + "tokens_seen": 2891055104 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001261010101010101, + "loss": 2.6314, + "theoretical_loss": 3.330251645692357, + "tokens_seen": 2891120640 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012608080808080806, + "loss": 2.3254, + "theoretical_loss": 3.3302457979044857, + "tokens_seen": 2891186176 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012606060606060608, + "loss": 2.5275, + "theoretical_loss": 3.3302399502862823, + "tokens_seen": 2891251712 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012604040404040403, + "loss": 2.4073, + "theoretical_loss": 3.3302341028377374, + "tokens_seen": 2891317248 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012602020202020204, + "loss": 2.6654, + "theoretical_loss": 3.3302282555588434, + "tokens_seen": 2891382784 + }, + { + "epoch": 0.75, + "learning_rate": 0.000126, + "loss": 2.4782, + "theoretical_loss": 3.33022240844959, + "tokens_seen": 2891448320 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012597979797979798, + "loss": 2.7301, + "theoretical_loss": 3.3302165615099697, + "tokens_seen": 2891513856 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012595959595959597, + "loss": 2.6353, + "theoretical_loss": 3.330210714739973, + "tokens_seen": 2891579392 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012593939393939392, + "loss": 2.4716, + "theoretical_loss": 3.3302048681395915, + "tokens_seen": 2891644928 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012591919191919194, + "loss": 2.7503, + "theoretical_loss": 3.3301990217088164, + "tokens_seen": 2891710464 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001258989898989899, + "loss": 2.7499, + "theoretical_loss": 3.330193175447639, + "tokens_seen": 2891776000 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012587878787878788, + "loss": 2.6842, + "theoretical_loss": 3.3301873293560504, + "tokens_seen": 2891841536 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012585858585858586, + "loss": 2.4502, + "theoretical_loss": 3.3301814834340417, + "tokens_seen": 2891907072 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012583838383838384, + "loss": 2.6519, + "theoretical_loss": 3.330175637681604, + "tokens_seen": 2891972608 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1628086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.607333183288574, + "objective/train/theoretical_loss": 3.3301697920987294, + "objective/train/tokens_used": 1250897376, + "theoretical_loss": 3.3301697920987294, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012581818181818183, + "loss": 2.637, + "theoretical_loss": 3.3301697920987294, + "tokens_seen": 2892038144 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001257979797979798, + "loss": 2.6525, + "theoretical_loss": 3.330163946685408, + "tokens_seen": 2892103680 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012577777777777777, + "loss": 2.7895, + "theoretical_loss": 3.330158101441632, + "tokens_seen": 2892169216 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012575757575757575, + "loss": 2.3727, + "theoretical_loss": 3.3301522563673918, + "tokens_seen": 2892234752 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012573737373737374, + "loss": 2.6472, + "theoretical_loss": 3.330146411462679, + "tokens_seen": 2892300288 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012571717171717172, + "loss": 2.6272, + "theoretical_loss": 3.3301405667274855, + "tokens_seen": 2892365824 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001256969696969697, + "loss": 2.659, + "theoretical_loss": 3.3301347221618016, + "tokens_seen": 2892431360 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012567676767676766, + "loss": 2.7567, + "theoretical_loss": 3.330128877765619, + "tokens_seen": 2892496896 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012565656565656567, + "loss": 2.6655, + "theoretical_loss": 3.330123033538929, + "tokens_seen": 2892562432 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012563636363636363, + "loss": 2.3995, + "theoretical_loss": 3.3301171894817223, + "tokens_seen": 2892627968 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012561616161616164, + "loss": 2.6952, + "theoretical_loss": 3.330111345593991, + "tokens_seen": 2892693504 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001255959595959596, + "loss": 2.5095, + "theoretical_loss": 3.3301055018757255, + "tokens_seen": 2892759040 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012557575757575755, + "loss": 2.7684, + "theoretical_loss": 3.3300996583269176, + "tokens_seen": 2892824576 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012555555555555557, + "loss": 2.7243, + "theoretical_loss": 3.330093814947558, + "tokens_seen": 2892890112 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012553535353535352, + "loss": 2.6057, + "theoretical_loss": 3.330087971737639, + "tokens_seen": 2892955648 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012551515151515153, + "loss": 2.7536, + "theoretical_loss": 3.3300821286971507, + "tokens_seen": 2893021184 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001254949494949495, + "loss": 2.5772, + "theoretical_loss": 3.330076285826085, + "tokens_seen": 2893086720 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012547474747474747, + "loss": 2.432, + "theoretical_loss": 3.3300704431244332, + "tokens_seen": 2893152256 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012545454545454546, + "loss": 2.4784, + "theoretical_loss": 3.330064600592186, + "tokens_seen": 2893217792 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012543434343434342, + "loss": 2.6503, + "theoretical_loss": 3.330058758229335, + "tokens_seen": 2893283328 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012541414141414143, + "loss": 2.6136, + "theoretical_loss": 3.3300529160358714, + "tokens_seen": 2893348864 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012539393939393938, + "loss": 2.4618, + "theoretical_loss": 3.3300470740117865, + "tokens_seen": 2893414400 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001253737373737374, + "loss": 2.6073, + "theoretical_loss": 3.330041232157072, + "tokens_seen": 2893479936 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012535353535353535, + "loss": 2.6969, + "theoretical_loss": 3.3300353904717177, + "tokens_seen": 2893545472 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012533333333333334, + "loss": 2.4616, + "theoretical_loss": 3.3300295489557166, + "tokens_seen": 2893611008 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1628768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.195849657058716, + "objective/train/theoretical_loss": 3.330023707609059, + "objective/train/tokens_used": 1252535776, + "theoretical_loss": 3.330023707609059, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012531313131313132, + "loss": 2.6024, + "theoretical_loss": 3.330023707609059, + "tokens_seen": 2893676544 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001252929292929293, + "loss": 2.6586, + "theoretical_loss": 3.3300178664317364, + "tokens_seen": 2893742080 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001252727272727273, + "loss": 2.6265, + "theoretical_loss": 3.33001202542374, + "tokens_seen": 2893807616 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012525252525252524, + "loss": 2.6269, + "theoretical_loss": 3.330006184585061, + "tokens_seen": 2893873152 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012523232323232323, + "loss": 2.531, + "theoretical_loss": 3.3300003439156907, + "tokens_seen": 2893938688 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001252121212121212, + "loss": 2.5492, + "theoretical_loss": 3.3299945034156204, + "tokens_seen": 2894004224 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001251919191919192, + "loss": 2.6457, + "theoretical_loss": 3.3299886630848414, + "tokens_seen": 2894069760 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012517171717171718, + "loss": 2.5862, + "theoretical_loss": 3.329982822923345, + "tokens_seen": 2894135296 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012515151515151516, + "loss": 2.6947, + "theoretical_loss": 3.3299769829311217, + "tokens_seen": 2894200832 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012513131313131312, + "loss": 2.5931, + "theoretical_loss": 3.329971143108164, + "tokens_seen": 2894266368 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012511111111111113, + "loss": 2.6276, + "theoretical_loss": 3.3299653034544625, + "tokens_seen": 2894331904 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001250909090909091, + "loss": 2.5578, + "theoretical_loss": 3.3299594639700087, + "tokens_seen": 2894397440 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012507070707070707, + "loss": 2.7047, + "theoretical_loss": 3.3299536246547934, + "tokens_seen": 2894462976 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012505050505050506, + "loss": 2.4756, + "theoretical_loss": 3.3299477855088084, + "tokens_seen": 2894528512 + }, + { + "epoch": 0.75, + "learning_rate": 0.000125030303030303, + "loss": 2.3948, + "theoretical_loss": 3.3299419465320446, + "tokens_seen": 2894594048 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012501010101010102, + "loss": 2.6003, + "theoretical_loss": 3.3299361077244933, + "tokens_seen": 2894659584 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012498989898989898, + "loss": 2.5229, + "theoretical_loss": 3.3299302690861463, + "tokens_seen": 2894725120 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012496969696969696, + "loss": 2.6065, + "theoretical_loss": 3.3299244306169937, + "tokens_seen": 2894790656 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012494949494949495, + "loss": 2.5192, + "theoretical_loss": 3.329918592317028, + "tokens_seen": 2894856192 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012492929292929293, + "loss": 2.5203, + "theoretical_loss": 3.32991275418624, + "tokens_seen": 2894921728 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012490909090909092, + "loss": 2.6274, + "theoretical_loss": 3.329906916224621, + "tokens_seen": 2894987264 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012488888888888887, + "loss": 2.6621, + "theoretical_loss": 3.3299010784321617, + "tokens_seen": 2895052800 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012486868686868686, + "loss": 2.4494, + "theoretical_loss": 3.3298952408088542, + "tokens_seen": 2895118336 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012484848484848484, + "loss": 2.5866, + "theoretical_loss": 3.3298894033546897, + "tokens_seen": 2895183872 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012482828282828283, + "loss": 2.5905, + "theoretical_loss": 3.329883566069659, + "tokens_seen": 2895249408 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1629434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5591092109680176, + "objective/train/theoretical_loss": 3.329877728953753, + "objective/train/tokens_used": 1254174176, + "theoretical_loss": 3.329877728953753, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001248080808080808, + "loss": 2.4312, + "theoretical_loss": 3.329877728953753, + "tokens_seen": 2895314944 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001247878787878788, + "loss": 2.7283, + "theoretical_loss": 3.329871892006964, + "tokens_seen": 2895380480 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012476767676767678, + "loss": 2.6276, + "theoretical_loss": 3.3298660552292834, + "tokens_seen": 2895446016 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012474747474747476, + "loss": 2.6765, + "theoretical_loss": 3.329860218620701, + "tokens_seen": 2895511552 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012472727272727272, + "loss": 2.6264, + "theoretical_loss": 3.3298543821812094, + "tokens_seen": 2895577088 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001247070707070707, + "loss": 2.7552, + "theoretical_loss": 3.3298485459107994, + "tokens_seen": 2895642624 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012468686868686869, + "loss": 2.5612, + "theoretical_loss": 3.3298427098094625, + "tokens_seen": 2895708160 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012466666666666667, + "loss": 2.3501, + "theoretical_loss": 3.3298368738771895, + "tokens_seen": 2895773696 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012464646464646465, + "loss": 2.6507, + "theoretical_loss": 3.329831038113972, + "tokens_seen": 2895839232 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012462626262626264, + "loss": 2.6052, + "theoretical_loss": 3.3298252025198014, + "tokens_seen": 2895904768 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012460606060606062, + "loss": 2.6511, + "theoretical_loss": 3.329819367094669, + "tokens_seen": 2895970304 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001245858585858586, + "loss": 2.6926, + "theoretical_loss": 3.3298135318385653, + "tokens_seen": 2896035840 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012456565656565656, + "loss": 2.6107, + "theoretical_loss": 3.329807696751483, + "tokens_seen": 2896101376 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012454545454545455, + "loss": 2.5567, + "theoretical_loss": 3.3298018618334115, + "tokens_seen": 2896166912 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012452525252525253, + "loss": 2.4211, + "theoretical_loss": 3.329796027084344, + "tokens_seen": 2896232448 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012450505050505051, + "loss": 2.4996, + "theoretical_loss": 3.3297901925042708, + "tokens_seen": 2896297984 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001244848484848485, + "loss": 2.6405, + "theoretical_loss": 3.329784358093183, + "tokens_seen": 2896363520 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012446464646464646, + "loss": 2.5158, + "theoretical_loss": 3.3297785238510724, + "tokens_seen": 2896429056 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012444444444444444, + "loss": 2.3992, + "theoretical_loss": 3.32977268977793, + "tokens_seen": 2896494592 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012442424242424242, + "loss": 2.3982, + "theoretical_loss": 3.329766855873747, + "tokens_seen": 2896560128 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001244040404040404, + "loss": 2.8097, + "theoretical_loss": 3.3297610221385154, + "tokens_seen": 2896625664 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243838383838384, + "loss": 2.4581, + "theoretical_loss": 3.329755188572226, + "tokens_seen": 2896691200 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012436363636363635, + "loss": 2.6113, + "theoretical_loss": 3.329749355174869, + "tokens_seen": 2896756736 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012434343434343433, + "loss": 2.5867, + "theoretical_loss": 3.3297435219464377, + "tokens_seen": 2896822272 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012432323232323232, + "loss": 2.3835, + "theoretical_loss": 3.329737688886922, + "tokens_seen": 2896887808 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1630757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6905438899993896, + "objective/train/theoretical_loss": 3.329731855996313, + "objective/train/tokens_used": 1255812576, + "theoretical_loss": 3.329731855996313, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001243030303030303, + "loss": 2.6937, + "theoretical_loss": 3.329731855996313, + "tokens_seen": 2896953344 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012428282828282828, + "loss": 2.4002, + "theoretical_loss": 3.3297260232746035, + "tokens_seen": 2897018880 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012426262626262627, + "loss": 2.5734, + "theoretical_loss": 3.3297201907217833, + "tokens_seen": 2897084416 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012424242424242425, + "loss": 2.7335, + "theoretical_loss": 3.3297143583378443, + "tokens_seen": 2897149952 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012422222222222224, + "loss": 2.3628, + "theoretical_loss": 3.329708526122778, + "tokens_seen": 2897215488 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001242020202020202, + "loss": 2.5101, + "theoretical_loss": 3.329702694076575, + "tokens_seen": 2897281024 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012418181818181818, + "loss": 2.5669, + "theoretical_loss": 3.3296968621992273, + "tokens_seen": 2897346560 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012416161616161616, + "loss": 2.4686, + "theoretical_loss": 3.329691030490726, + "tokens_seen": 2897412096 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012414141414141414, + "loss": 2.4814, + "theoretical_loss": 3.329685198951062, + "tokens_seen": 2897477632 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012412121212121213, + "loss": 2.5187, + "theoretical_loss": 3.3296793675802268, + "tokens_seen": 2897543168 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001241010101010101, + "loss": 2.4963, + "theoretical_loss": 3.329673536378212, + "tokens_seen": 2897608704 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001240808080808081, + "loss": 2.5387, + "theoretical_loss": 3.3296677053450088, + "tokens_seen": 2897674240 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012406060606060608, + "loss": 2.581, + "theoretical_loss": 3.329661874480608, + "tokens_seen": 2897739776 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012404040404040404, + "loss": 2.6886, + "theoretical_loss": 3.329656043785002, + "tokens_seen": 2897805312 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012402020202020202, + "loss": 2.5334, + "theoretical_loss": 3.3296502132581804, + "tokens_seen": 2897870848 + }, + { + "epoch": 0.75, + "learning_rate": 0.000124, + "loss": 2.2476, + "theoretical_loss": 3.329644382900136, + "tokens_seen": 2897936384 + }, + { + "epoch": 0.75, + "learning_rate": 0.000123979797979798, + "loss": 2.7031, + "theoretical_loss": 3.3296385527108594, + "tokens_seen": 2898001920 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012395959595959597, + "loss": 2.5292, + "theoretical_loss": 3.3296327226903424, + "tokens_seen": 2898067456 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012393939393939393, + "loss": 2.5524, + "theoretical_loss": 3.3296268928385757, + "tokens_seen": 2898132992 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012391919191919191, + "loss": 2.5744, + "theoretical_loss": 3.3296210631555505, + "tokens_seen": 2898198528 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001238989898989899, + "loss": 2.6034, + "theoretical_loss": 3.3296152336412588, + "tokens_seen": 2898264064 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012387878787878788, + "loss": 2.7857, + "theoretical_loss": 3.3296094042956916, + "tokens_seen": 2898329600 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012385858585858587, + "loss": 2.5872, + "theoretical_loss": 3.32960357511884, + "tokens_seen": 2898395136 + }, + { + "epoch": 0.75, + "learning_rate": 0.00012383838383838385, + "loss": 2.4966, + "theoretical_loss": 3.3295977461106956, + "tokens_seen": 2898460672 + }, + { + "epoch": 0.75, + "learning_rate": 0.0001238181818181818, + "loss": 2.5406, + "theoretical_loss": 3.32959191727125, + "tokens_seen": 2898526208 + }, + { + "epoch": 0.75, + "objective/train/docs_used": 1631491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3764522075653076, + "objective/train/theoretical_loss": 3.329586088600493, + "objective/train/tokens_used": 1257450976, + "theoretical_loss": 3.329586088600493, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001237979797979798, + "loss": 2.6617, + "theoretical_loss": 3.329586088600493, + "tokens_seen": 2898591744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012377777777777777, + "loss": 2.4811, + "theoretical_loss": 3.329580260098418, + "tokens_seen": 2898657280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012375757575757576, + "loss": 2.6793, + "theoretical_loss": 3.329574431765015, + "tokens_seen": 2898722816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012373737373737374, + "loss": 2.7945, + "theoretical_loss": 3.329568603600275, + "tokens_seen": 2898788352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012371717171717173, + "loss": 2.5883, + "theoretical_loss": 3.3295627756041903, + "tokens_seen": 2898853888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012369696969696968, + "loss": 2.8118, + "theoretical_loss": 3.329556947776752, + "tokens_seen": 2898919424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012367676767676767, + "loss": 2.752, + "theoretical_loss": 3.3295511201179506, + "tokens_seen": 2898984960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012365656565656565, + "loss": 2.7159, + "theoretical_loss": 3.329545292627779, + "tokens_seen": 2899050496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012363636363636364, + "loss": 2.5321, + "theoretical_loss": 3.329539465306227, + "tokens_seen": 2899116032 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012361616161616162, + "loss": 2.4313, + "theoretical_loss": 3.329533638153286, + "tokens_seen": 2899181568 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235959595959596, + "loss": 2.5447, + "theoretical_loss": 3.329527811168948, + "tokens_seen": 2899247104 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235757575757576, + "loss": 2.7274, + "theoretical_loss": 3.329521984353204, + "tokens_seen": 2899312640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012355555555555557, + "loss": 2.4857, + "theoretical_loss": 3.3295161577060455, + "tokens_seen": 2899378176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012353535353535356, + "loss": 2.5315, + "theoretical_loss": 3.329510331227464, + "tokens_seen": 2899443712 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001235151515151515, + "loss": 2.6706, + "theoretical_loss": 3.32950450491745, + "tokens_seen": 2899509248 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001234949494949495, + "loss": 2.5807, + "theoretical_loss": 3.3294986787759955, + "tokens_seen": 2899574784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012347474747474748, + "loss": 2.6964, + "theoretical_loss": 3.3294928528030914, + "tokens_seen": 2899640320 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012345454545454546, + "loss": 2.3869, + "theoretical_loss": 3.3294870269987293, + "tokens_seen": 2899705856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012343434343434345, + "loss": 2.4319, + "theoretical_loss": 3.329481201362901, + "tokens_seen": 2899771392 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012341414141414143, + "loss": 2.7226, + "theoretical_loss": 3.3294753758955964, + "tokens_seen": 2899836928 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001233939393939394, + "loss": 2.3871, + "theoretical_loss": 3.329469550596808, + "tokens_seen": 2899902464 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012337373737373737, + "loss": 2.8016, + "theoretical_loss": 3.329463725466527, + "tokens_seen": 2899968000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012335353535353536, + "loss": 2.5933, + "theoretical_loss": 3.329457900504744, + "tokens_seen": 2900033536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012333333333333334, + "loss": 2.777, + "theoretical_loss": 3.329452075711451, + "tokens_seen": 2900099072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012331313131313132, + "loss": 2.6788, + "theoretical_loss": 3.32944625108664, + "tokens_seen": 2900164608 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1632851, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.547261953353882, + "objective/train/theoretical_loss": 3.3294404266303004, + "objective/train/tokens_used": 1259089376, + "theoretical_loss": 3.3294404266303004, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012329292929292928, + "loss": 2.3965, + "theoretical_loss": 3.3294404266303004, + "tokens_seen": 2900230144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012327272727272727, + "loss": 2.601, + "theoretical_loss": 3.329434602342425, + "tokens_seen": 2900295680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012325252525252525, + "loss": 2.5523, + "theoretical_loss": 3.3294287782230048, + "tokens_seen": 2900361216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012323232323232323, + "loss": 2.6301, + "theoretical_loss": 3.329422954272031, + "tokens_seen": 2900426752 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012321212121212122, + "loss": 2.5806, + "theoretical_loss": 3.3294171304894946, + "tokens_seen": 2900492288 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001231919191919192, + "loss": 2.4359, + "theoretical_loss": 3.3294113068753877, + "tokens_seen": 2900557824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012317171717171716, + "loss": 2.4689, + "theoretical_loss": 3.3294054834297007, + "tokens_seen": 2900623360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012315151515151514, + "loss": 2.6743, + "theoretical_loss": 3.3293996601524256, + "tokens_seen": 2900688896 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012313131313131313, + "loss": 2.5276, + "theoretical_loss": 3.329393837043554, + "tokens_seen": 2900754432 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001231111111111111, + "loss": 2.6118, + "theoretical_loss": 3.3293880141030763, + "tokens_seen": 2900819968 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001230909090909091, + "loss": 2.4754, + "theoretical_loss": 3.3293821913309847, + "tokens_seen": 2900885504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012307070707070708, + "loss": 2.5264, + "theoretical_loss": 3.3293763687272695, + "tokens_seen": 2900951040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012305050505050506, + "loss": 2.4638, + "theoretical_loss": 3.329370546291923, + "tokens_seen": 2901016576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012303030303030305, + "loss": 2.577, + "theoretical_loss": 3.3293647240249364, + "tokens_seen": 2901082112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012301010101010103, + "loss": 2.6256, + "theoretical_loss": 3.3293589019263004, + "tokens_seen": 2901147648 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122989898989899, + "loss": 2.6414, + "theoretical_loss": 3.329353079996007, + "tokens_seen": 2901213184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012296969696969697, + "loss": 2.8411, + "theoretical_loss": 3.3293472582340473, + "tokens_seen": 2901278720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012294949494949495, + "loss": 2.3437, + "theoretical_loss": 3.329341436640412, + "tokens_seen": 2901344256 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012292929292929294, + "loss": 2.6096, + "theoretical_loss": 3.329335615215094, + "tokens_seen": 2901409792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012290909090909092, + "loss": 2.6012, + "theoretical_loss": 3.329329793958083, + "tokens_seen": 2901475328 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001228888888888889, + "loss": 2.5271, + "theoretical_loss": 3.3293239728693713, + "tokens_seen": 2901540864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012286868686868686, + "loss": 2.5213, + "theoretical_loss": 3.3293181519489496, + "tokens_seen": 2901606400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012284848484848485, + "loss": 2.7228, + "theoretical_loss": 3.32931233119681, + "tokens_seen": 2901671936 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012282828282828283, + "loss": 2.4231, + "theoretical_loss": 3.329306510612943, + "tokens_seen": 2901737472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012280808080808082, + "loss": 2.5147, + "theoretical_loss": 3.3293006901973405, + "tokens_seen": 2901803008 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1633283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.774747371673584, + "objective/train/theoretical_loss": 3.329294869949994, + "objective/train/tokens_used": 1260727776, + "theoretical_loss": 3.329294869949994, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227878787878788, + "loss": 2.486, + "theoretical_loss": 3.329294869949994, + "tokens_seen": 2901868544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012276767676767676, + "loss": 2.6248, + "theoretical_loss": 3.329289049870894, + "tokens_seen": 2901934080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012274747474747474, + "loss": 2.7882, + "theoretical_loss": 3.329283229960032, + "tokens_seen": 2901999616 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012272727272727272, + "loss": 2.3911, + "theoretical_loss": 3.3292774102174003, + "tokens_seen": 2902065152 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001227070707070707, + "loss": 2.6641, + "theoretical_loss": 3.3292715906429895, + "tokens_seen": 2902130688 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226868686868687, + "loss": 2.4561, + "theoretical_loss": 3.3292657712367912, + "tokens_seen": 2902196224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012266666666666668, + "loss": 2.6711, + "theoretical_loss": 3.329259951998796, + "tokens_seen": 2902261760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012264646464646463, + "loss": 2.686, + "theoretical_loss": 3.3292541329289964, + "tokens_seen": 2902327296 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012262626262626262, + "loss": 2.6496, + "theoretical_loss": 3.329248314027383, + "tokens_seen": 2902392832 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001226060606060606, + "loss": 2.7074, + "theoretical_loss": 3.3292424952939474, + "tokens_seen": 2902458368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012258585858585858, + "loss": 2.3643, + "theoretical_loss": 3.329236676728681, + "tokens_seen": 2902523904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012256565656565657, + "loss": 2.6195, + "theoretical_loss": 3.3292308583315746, + "tokens_seen": 2902589440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012254545454545455, + "loss": 2.5374, + "theoretical_loss": 3.32922504010262, + "tokens_seen": 2902654976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012252525252525254, + "loss": 2.4627, + "theoretical_loss": 3.3292192220418086, + "tokens_seen": 2902720512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012250505050505052, + "loss": 2.664, + "theoretical_loss": 3.3292134041491317, + "tokens_seen": 2902786048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012248484848484848, + "loss": 2.5769, + "theoretical_loss": 3.3292075864245803, + "tokens_seen": 2902851584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012246464646464646, + "loss": 2.5133, + "theoretical_loss": 3.3292017688681463, + "tokens_seen": 2902917120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012244444444444445, + "loss": 2.5151, + "theoretical_loss": 3.3291959514798206, + "tokens_seen": 2902982656 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012242424242424243, + "loss": 2.5835, + "theoretical_loss": 3.329190134259595, + "tokens_seen": 2903048192 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001224040404040404, + "loss": 2.6655, + "theoretical_loss": 3.3291843172074604, + "tokens_seen": 2903113728 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001223838383838384, + "loss": 2.5856, + "theoretical_loss": 3.3291785003234082, + "tokens_seen": 2903179264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012236363636363638, + "loss": 2.6883, + "theoretical_loss": 3.32917268360743, + "tokens_seen": 2903244800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012234343434343434, + "loss": 2.4006, + "theoretical_loss": 3.329166867059517, + "tokens_seen": 2903310336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012232323232323232, + "loss": 2.5049, + "theoretical_loss": 3.3291610506796605, + "tokens_seen": 2903375872 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001223030303030303, + "loss": 2.6614, + "theoretical_loss": 3.329155234467852, + "tokens_seen": 2903441408 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1634619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3457143306732178, + "objective/train/theoretical_loss": 3.3291494184240826, + "objective/train/tokens_used": 1262366176, + "theoretical_loss": 3.3291494184240826, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001222828282828283, + "loss": 2.5341, + "theoretical_loss": 3.3291494184240826, + "tokens_seen": 2903506944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012226262626262627, + "loss": 2.5868, + "theoretical_loss": 3.329143602548344, + "tokens_seen": 2903572480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012224242424242423, + "loss": 2.6776, + "theoretical_loss": 3.3291377868406276, + "tokens_seen": 2903638016 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012222222222222221, + "loss": 2.3975, + "theoretical_loss": 3.329131971300924, + "tokens_seen": 2903703552 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001222020202020202, + "loss": 2.6715, + "theoretical_loss": 3.329126155929225, + "tokens_seen": 2903769088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012218181818181818, + "loss": 2.6292, + "theoretical_loss": 3.3291203407255225, + "tokens_seen": 2903834624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012216161616161617, + "loss": 2.6218, + "theoretical_loss": 3.3291145256898074, + "tokens_seen": 2903900160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012214141414141415, + "loss": 2.5479, + "theoretical_loss": 3.3291087108220707, + "tokens_seen": 2903965696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001221212121212121, + "loss": 2.7919, + "theoretical_loss": 3.329102896122304, + "tokens_seen": 2904031232 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001221010101010101, + "loss": 2.6312, + "theoretical_loss": 3.3290970815904997, + "tokens_seen": 2904096768 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012208080808080808, + "loss": 2.5383, + "theoretical_loss": 3.329091267226647, + "tokens_seen": 2904162304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012206060606060607, + "loss": 2.6695, + "theoretical_loss": 3.329085453030739, + "tokens_seen": 2904227840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012204040404040404, + "loss": 2.7156, + "theoretical_loss": 3.329079639002767, + "tokens_seen": 2904293376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012202020202020201, + "loss": 2.7865, + "theoretical_loss": 3.3290738251427214, + "tokens_seen": 2904358912 + }, + { + "epoch": 0.76, + "learning_rate": 0.000122, + "loss": 2.566, + "theoretical_loss": 3.3290680114505937, + "tokens_seen": 2904424448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012197979797979798, + "loss": 2.6556, + "theoretical_loss": 3.329062197926376, + "tokens_seen": 2904489984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012195959595959597, + "loss": 2.7843, + "theoretical_loss": 3.32905638457006, + "tokens_seen": 2904555520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012193939393939395, + "loss": 2.59, + "theoretical_loss": 3.3290505713816354, + "tokens_seen": 2904621056 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012191919191919192, + "loss": 2.6329, + "theoretical_loss": 3.3290447583610945, + "tokens_seen": 2904686592 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001218989898989899, + "loss": 2.6192, + "theoretical_loss": 3.329038945508429, + "tokens_seen": 2904752128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012187878787878787, + "loss": 2.7049, + "theoretical_loss": 3.3290331328236302, + "tokens_seen": 2904817664 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012185858585858586, + "loss": 2.4779, + "theoretical_loss": 3.3290273203066887, + "tokens_seen": 2904883200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012183838383838384, + "loss": 2.5173, + "theoretical_loss": 3.3290215079575964, + "tokens_seen": 2904948736 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012181818181818181, + "loss": 2.6016, + "theoretical_loss": 3.329015695776345, + "tokens_seen": 2905014272 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001217979797979798, + "loss": 2.556, + "theoretical_loss": 3.329009883762925, + "tokens_seen": 2905079808 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1635246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.430767774581909, + "objective/train/theoretical_loss": 3.329004071917329, + "objective/train/tokens_used": 1264004576, + "theoretical_loss": 3.329004071917329, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012177777777777778, + "loss": 2.643, + "theoretical_loss": 3.329004071917329, + "tokens_seen": 2905145344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012175757575757576, + "loss": 2.6314, + "theoretical_loss": 3.3289982602395467, + "tokens_seen": 2905210880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012173737373737375, + "loss": 2.3558, + "theoretical_loss": 3.3289924487295712, + "tokens_seen": 2905276416 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012171717171717173, + "loss": 2.4747, + "theoretical_loss": 3.3289866373873926, + "tokens_seen": 2905341952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012169696969696969, + "loss": 2.5345, + "theoretical_loss": 3.328980826213003, + "tokens_seen": 2905407488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012167676767676767, + "loss": 2.6592, + "theoretical_loss": 3.3289750152063933, + "tokens_seen": 2905473024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012165656565656566, + "loss": 2.6816, + "theoretical_loss": 3.328969204367555, + "tokens_seen": 2905538560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012163636363636364, + "loss": 2.4176, + "theoretical_loss": 3.3289633936964798, + "tokens_seen": 2905604096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012161616161616162, + "loss": 2.4915, + "theoretical_loss": 3.328957583193159, + "tokens_seen": 2905669632 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001215959595959596, + "loss": 2.4981, + "theoretical_loss": 3.3289517728575833, + "tokens_seen": 2905735168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012157575757575758, + "loss": 2.555, + "theoretical_loss": 3.328945962689745, + "tokens_seen": 2905800704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012155555555555556, + "loss": 2.5709, + "theoretical_loss": 3.328940152689635, + "tokens_seen": 2905866240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012153535353535355, + "loss": 2.397, + "theoretical_loss": 3.3289343428572447, + "tokens_seen": 2905931776 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012151515151515152, + "loss": 2.6088, + "theoretical_loss": 3.3289285331925655, + "tokens_seen": 2905997312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012149494949494949, + "loss": 2.6598, + "theoretical_loss": 3.3289227236955887, + "tokens_seen": 2906062848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012147474747474747, + "loss": 2.4202, + "theoretical_loss": 3.3289169143663058, + "tokens_seen": 2906128384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012145454545454546, + "loss": 2.5347, + "theoretical_loss": 3.3289111052047082, + "tokens_seen": 2906193920 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012143434343434344, + "loss": 2.7357, + "theoretical_loss": 3.3289052962107872, + "tokens_seen": 2906259456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012141414141414142, + "loss": 2.7541, + "theoretical_loss": 3.328899487384534, + "tokens_seen": 2906324992 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001213939393939394, + "loss": 2.6216, + "theoretical_loss": 3.32889367872594, + "tokens_seen": 2906390528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012137373737373738, + "loss": 2.5698, + "theoretical_loss": 3.3288878702349973, + "tokens_seen": 2906456064 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012135353535353535, + "loss": 2.6172, + "theoretical_loss": 3.3288820619116963, + "tokens_seen": 2906521600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012133333333333333, + "loss": 2.5751, + "theoretical_loss": 3.3288762537560292, + "tokens_seen": 2906587136 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012131313131313132, + "loss": 2.7532, + "theoretical_loss": 3.328870445767987, + "tokens_seen": 2906652672 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001212929292929293, + "loss": 2.6776, + "theoretical_loss": 3.328864637947561, + "tokens_seen": 2906718208 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1636404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3817529678344727, + "objective/train/theoretical_loss": 3.3288588302947426, + "objective/train/tokens_used": 1265642976, + "theoretical_loss": 3.3288588302947426, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012127272727272727, + "loss": 2.6918, + "theoretical_loss": 3.3288588302947426, + "tokens_seen": 2906783744 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012125252525252525, + "loss": 2.5435, + "theoretical_loss": 3.3288530228095228, + "tokens_seen": 2906849280 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012123232323232324, + "loss": 2.4374, + "theoretical_loss": 3.328847215491894, + "tokens_seen": 2906914816 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012121212121212122, + "loss": 2.6059, + "theoretical_loss": 3.328841408341847, + "tokens_seen": 2906980352 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012119191919191921, + "loss": 2.8353, + "theoretical_loss": 3.328835601359373, + "tokens_seen": 2907045888 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012117171717171716, + "loss": 2.2727, + "theoretical_loss": 3.328829794544464, + "tokens_seen": 2907111424 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012115151515151515, + "loss": 2.4573, + "theoretical_loss": 3.3288239878971106, + "tokens_seen": 2907176960 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012113131313131313, + "loss": 2.4041, + "theoretical_loss": 3.3288181814173043, + "tokens_seen": 2907242496 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012111111111111112, + "loss": 2.4274, + "theoretical_loss": 3.3288123751050374, + "tokens_seen": 2907308032 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001210909090909091, + "loss": 2.6714, + "theoretical_loss": 3.3288065689603004, + "tokens_seen": 2907373568 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012107070707070707, + "loss": 2.717, + "theoretical_loss": 3.328800762983085, + "tokens_seen": 2907439104 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012105050505050505, + "loss": 2.5501, + "theoretical_loss": 3.328794957173382, + "tokens_seen": 2907504640 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012103030303030304, + "loss": 2.7417, + "theoretical_loss": 3.328789151531184, + "tokens_seen": 2907570176 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012101010101010101, + "loss": 2.5076, + "theoretical_loss": 3.3287833460564817, + "tokens_seen": 2907635712 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012098989898989899, + "loss": 2.461, + "theoretical_loss": 3.328777540749266, + "tokens_seen": 2907701248 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012096969696969696, + "loss": 2.7057, + "theoretical_loss": 3.328771735609529, + "tokens_seen": 2907766784 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012094949494949495, + "loss": 2.6654, + "theoretical_loss": 3.328765930637262, + "tokens_seen": 2907832320 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012092929292929293, + "loss": 2.6463, + "theoretical_loss": 3.3287601258324564, + "tokens_seen": 2907897856 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012090909090909091, + "loss": 2.5632, + "theoretical_loss": 3.3287543211951034, + "tokens_seen": 2907963392 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001208888888888889, + "loss": 2.5395, + "theoretical_loss": 3.3287485167251942, + "tokens_seen": 2908028928 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012086868686868688, + "loss": 2.54, + "theoretical_loss": 3.328742712422721, + "tokens_seen": 2908094464 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012084848484848485, + "loss": 2.5359, + "theoretical_loss": 3.328736908287674, + "tokens_seen": 2908160000 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012082828282828282, + "loss": 2.5707, + "theoretical_loss": 3.328731104320046, + "tokens_seen": 2908225536 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012080808080808081, + "loss": 2.5258, + "theoretical_loss": 3.328725300519827, + "tokens_seen": 2908291072 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012078787878787879, + "loss": 2.6512, + "theoretical_loss": 3.3287194968870093, + "tokens_seen": 2908356608 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1636998, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7506494522094727, + "objective/train/theoretical_loss": 3.328713693421584, + "objective/train/tokens_used": 1267281376, + "theoretical_loss": 3.328713693421584, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012076767676767678, + "loss": 2.6574, + "theoretical_loss": 3.328713693421584, + "tokens_seen": 2908422144 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012074747474747475, + "loss": 2.6619, + "theoretical_loss": 3.328707890123543, + "tokens_seen": 2908487680 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012072727272727273, + "loss": 2.7568, + "theoretical_loss": 3.3287020869928767, + "tokens_seen": 2908553216 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012070707070707071, + "loss": 2.6385, + "theoretical_loss": 3.3286962840295775, + "tokens_seen": 2908618752 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001206868686868687, + "loss": 2.4022, + "theoretical_loss": 3.328690481233636, + "tokens_seen": 2908684288 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012066666666666667, + "loss": 2.6303, + "theoretical_loss": 3.328684678605044, + "tokens_seen": 2908749824 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012064646464646464, + "loss": 2.6704, + "theoretical_loss": 3.3286788761437935, + "tokens_seen": 2908815360 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012062626262626262, + "loss": 2.7143, + "theoretical_loss": 3.328673073849875, + "tokens_seen": 2908880896 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001206060606060606, + "loss": 2.7543, + "theoretical_loss": 3.3286672717232797, + "tokens_seen": 2908946432 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012058585858585859, + "loss": 2.5235, + "theoretical_loss": 3.328661469764, + "tokens_seen": 2909011968 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012056565656565657, + "loss": 2.4538, + "theoretical_loss": 3.3286556679720265, + "tokens_seen": 2909077504 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012054545454545454, + "loss": 2.6161, + "theoretical_loss": 3.328649866347351, + "tokens_seen": 2909143040 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012052525252525253, + "loss": 2.6419, + "theoretical_loss": 3.328644064889965, + "tokens_seen": 2909208576 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012050505050505051, + "loss": 2.4131, + "theoretical_loss": 3.3286382635998595, + "tokens_seen": 2909274112 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012048484848484848, + "loss": 2.5211, + "theoretical_loss": 3.3286324624770263, + "tokens_seen": 2909339648 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012046464646464647, + "loss": 2.7236, + "theoretical_loss": 3.328626661521456, + "tokens_seen": 2909405184 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012044444444444445, + "loss": 2.4919, + "theoretical_loss": 3.3286208607331416, + "tokens_seen": 2909470720 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012042424242424242, + "loss": 2.5652, + "theoretical_loss": 3.3286150601120728, + "tokens_seen": 2909536256 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001204040404040404, + "loss": 2.6859, + "theoretical_loss": 3.328609259658242, + "tokens_seen": 2909601792 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012038383838383839, + "loss": 2.4535, + "theoretical_loss": 3.3286034593716405, + "tokens_seen": 2909667328 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012036363636363637, + "loss": 2.6286, + "theoretical_loss": 3.3285976592522597, + "tokens_seen": 2909732864 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012034343434343436, + "loss": 2.5088, + "theoretical_loss": 3.328591859300091, + "tokens_seen": 2909798400 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012032323232323231, + "loss": 2.5938, + "theoretical_loss": 3.3285860595151253, + "tokens_seen": 2909863936 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001203030303030303, + "loss": 2.6055, + "theoretical_loss": 3.328580259897355, + "tokens_seen": 2909929472 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012028282828282828, + "loss": 2.467, + "theoretical_loss": 3.32857446044677, + "tokens_seen": 2909995008 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1638307, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.027076482772827, + "objective/train/theoretical_loss": 3.3285686611633634, + "objective/train/tokens_used": 1268919776, + "theoretical_loss": 3.3285686611633634, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012026262626262627, + "loss": 2.5404, + "theoretical_loss": 3.3285686611633634, + "tokens_seen": 2910060544 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012024242424242425, + "loss": 2.5138, + "theoretical_loss": 3.328562862047126, + "tokens_seen": 2910126080 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012022222222222222, + "loss": 2.4323, + "theoretical_loss": 3.3285570630980486, + "tokens_seen": 2910191616 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001202020202020202, + "loss": 2.3515, + "theoretical_loss": 3.3285512643161232, + "tokens_seen": 2910257152 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012018181818181819, + "loss": 2.5961, + "theoretical_loss": 3.328545465701341, + "tokens_seen": 2910322688 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012016161616161617, + "loss": 2.6591, + "theoretical_loss": 3.328539667253694, + "tokens_seen": 2910388224 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012014141414141414, + "loss": 2.7347, + "theoretical_loss": 3.3285338689731727, + "tokens_seen": 2910453760 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012012121212121211, + "loss": 2.7192, + "theoretical_loss": 3.3285280708597695, + "tokens_seen": 2910519296 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001201010101010101, + "loss": 2.5098, + "theoretical_loss": 3.3285222729134754, + "tokens_seen": 2910584832 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012008080808080808, + "loss": 2.7877, + "theoretical_loss": 3.328516475134281, + "tokens_seen": 2910650368 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012006060606060606, + "loss": 2.7756, + "theoretical_loss": 3.328510677522179, + "tokens_seen": 2910715904 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012004040404040405, + "loss": 2.4982, + "theoretical_loss": 3.32850488007716, + "tokens_seen": 2910781440 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012002020202020203, + "loss": 2.5687, + "theoretical_loss": 3.328499082799216, + "tokens_seen": 2910846976 + }, + { + "epoch": 0.76, + "learning_rate": 0.00012, + "loss": 2.814, + "theoretical_loss": 3.328493285688338, + "tokens_seen": 2910912512 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011997979797979799, + "loss": 2.7312, + "theoretical_loss": 3.3284874887445173, + "tokens_seen": 2910978048 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011995959595959596, + "loss": 2.4789, + "theoretical_loss": 3.3284816919677453, + "tokens_seen": 2911043584 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011993939393939394, + "loss": 2.4523, + "theoretical_loss": 3.3284758953580145, + "tokens_seen": 2911109120 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011991919191919193, + "loss": 2.599, + "theoretical_loss": 3.3284700989153153, + "tokens_seen": 2911174656 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001198989898989899, + "loss": 2.466, + "theoretical_loss": 3.328464302639639, + "tokens_seen": 2911240192 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011987878787878788, + "loss": 2.5882, + "theoretical_loss": 3.3284585065309775, + "tokens_seen": 2911305728 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011985858585858586, + "loss": 2.5113, + "theoretical_loss": 3.328452710589322, + "tokens_seen": 2911371264 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011983838383838385, + "loss": 2.6122, + "theoretical_loss": 3.3284469148146645, + "tokens_seen": 2911436800 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011981818181818183, + "loss": 2.743, + "theoretical_loss": 3.3284411192069956, + "tokens_seen": 2911502336 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011979797979797979, + "loss": 2.616, + "theoretical_loss": 3.3284353237663074, + "tokens_seen": 2911567872 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011977777777777777, + "loss": 2.5117, + "theoretical_loss": 3.328429528492591, + "tokens_seen": 2911633408 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1639054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4445552825927734, + "objective/train/theoretical_loss": 3.3284237333858373, + "objective/train/tokens_used": 1270558176, + "theoretical_loss": 3.3284237333858373, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011975757575757576, + "loss": 2.6462, + "theoretical_loss": 3.3284237333858373, + "tokens_seen": 2911698944 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011973737373737374, + "loss": 2.6348, + "theoretical_loss": 3.3284179384460386, + "tokens_seen": 2911764480 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011971717171717172, + "loss": 2.7456, + "theoretical_loss": 3.3284121436731864, + "tokens_seen": 2911830016 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001196969696969697, + "loss": 2.5104, + "theoretical_loss": 3.3284063490672713, + "tokens_seen": 2911895552 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011967676767676768, + "loss": 2.3019, + "theoretical_loss": 3.3284005546282853, + "tokens_seen": 2911961088 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011965656565656566, + "loss": 2.4052, + "theoretical_loss": 3.3283947603562196, + "tokens_seen": 2912026624 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011963636363636365, + "loss": 2.4633, + "theoretical_loss": 3.328388966251066, + "tokens_seen": 2912092160 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011961616161616162, + "loss": 2.7756, + "theoretical_loss": 3.3283831723128157, + "tokens_seen": 2912157696 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001195959595959596, + "loss": 2.3835, + "theoretical_loss": 3.3283773785414597, + "tokens_seen": 2912223232 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011957575757575757, + "loss": 2.4743, + "theoretical_loss": 3.32837158493699, + "tokens_seen": 2912288768 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011955555555555556, + "loss": 2.6693, + "theoretical_loss": 3.3283657914993983, + "tokens_seen": 2912354304 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011953535353535354, + "loss": 2.5704, + "theoretical_loss": 3.3283599982286756, + "tokens_seen": 2912419840 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011951515151515152, + "loss": 2.5222, + "theoretical_loss": 3.328354205124813, + "tokens_seen": 2912485376 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011949494949494951, + "loss": 2.57, + "theoretical_loss": 3.3283484121878026, + "tokens_seen": 2912550912 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011947474747474748, + "loss": 2.5204, + "theoretical_loss": 3.3283426194176355, + "tokens_seen": 2912616448 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011945454545454545, + "loss": 2.4499, + "theoretical_loss": 3.328336826814303, + "tokens_seen": 2912681984 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011943434343434343, + "loss": 2.4049, + "theoretical_loss": 3.328331034377797, + "tokens_seen": 2912747520 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011941414141414142, + "loss": 2.7115, + "theoretical_loss": 3.3283252421081087, + "tokens_seen": 2912813056 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001193939393939394, + "loss": 2.5431, + "theoretical_loss": 3.3283194500052296, + "tokens_seen": 2912878592 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011937373737373737, + "loss": 2.6461, + "theoretical_loss": 3.328313658069151, + "tokens_seen": 2912944128 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011935353535353535, + "loss": 2.6374, + "theoretical_loss": 3.328307866299864, + "tokens_seen": 2913009664 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011933333333333334, + "loss": 2.483, + "theoretical_loss": 3.3283020746973606, + "tokens_seen": 2913075200 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011931313131313132, + "loss": 2.4789, + "theoretical_loss": 3.3282962832616323, + "tokens_seen": 2913140736 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001192929292929293, + "loss": 2.4869, + "theoretical_loss": 3.3282904919926706, + "tokens_seen": 2913206272 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011927272727272726, + "loss": 2.626, + "theoretical_loss": 3.3282847008904666, + "tokens_seen": 2913271808 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1640560, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2749183177948, + "objective/train/theoretical_loss": 3.3282789099550114, + "objective/train/tokens_used": 1272196576, + "theoretical_loss": 3.3282789099550114, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011925252525252525, + "loss": 2.4436, + "theoretical_loss": 3.3282789099550114, + "tokens_seen": 2913337344 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011923232323232323, + "loss": 2.4639, + "theoretical_loss": 3.328273119186297, + "tokens_seen": 2913402880 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011921212121212121, + "loss": 2.4205, + "theoretical_loss": 3.328267328584315, + "tokens_seen": 2913468416 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001191919191919192, + "loss": 2.4654, + "theoretical_loss": 3.3282615381490563, + "tokens_seen": 2913533952 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011917171717171718, + "loss": 2.6094, + "theoretical_loss": 3.328255747880513, + "tokens_seen": 2913599488 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011915151515151515, + "loss": 2.7647, + "theoretical_loss": 3.328249957778676, + "tokens_seen": 2913665024 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011913131313131314, + "loss": 2.6004, + "theoretical_loss": 3.328244167843537, + "tokens_seen": 2913730560 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011911111111111112, + "loss": 2.6136, + "theoretical_loss": 3.3282383780750875, + "tokens_seen": 2913796096 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011909090909090909, + "loss": 2.5134, + "theoretical_loss": 3.3282325884733184, + "tokens_seen": 2913861632 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011907070707070708, + "loss": 2.5379, + "theoretical_loss": 3.328226799038222, + "tokens_seen": 2913927168 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011905050505050505, + "loss": 2.5843, + "theoretical_loss": 3.328221009769789, + "tokens_seen": 2913992704 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011903030303030303, + "loss": 2.532, + "theoretical_loss": 3.3282152206680116, + "tokens_seen": 2914058240 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011901010101010101, + "loss": 2.5842, + "theoretical_loss": 3.3282094317328808, + "tokens_seen": 2914123776 + }, + { + "epoch": 0.76, + "learning_rate": 0.000118989898989899, + "loss": 2.5156, + "theoretical_loss": 3.3282036429643878, + "tokens_seen": 2914189312 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011896969696969698, + "loss": 2.5979, + "theoretical_loss": 3.3281978543625246, + "tokens_seen": 2914254848 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011894949494949495, + "loss": 2.6998, + "theoretical_loss": 3.3281920659272823, + "tokens_seen": 2914320384 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011892929292929292, + "loss": 2.729, + "theoretical_loss": 3.3281862776586526, + "tokens_seen": 2914385920 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001189090909090909, + "loss": 2.5663, + "theoretical_loss": 3.328180489556627, + "tokens_seen": 2914451456 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011888888888888889, + "loss": 2.5489, + "theoretical_loss": 3.3281747016211964, + "tokens_seen": 2914516992 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011886868686868687, + "loss": 2.6636, + "theoretical_loss": 3.328168913852353, + "tokens_seen": 2914582528 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011884848484848484, + "loss": 2.7044, + "theoretical_loss": 3.3281631262500877, + "tokens_seen": 2914648064 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011882828282828283, + "loss": 2.5756, + "theoretical_loss": 3.3281573388143926, + "tokens_seen": 2914713600 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011880808080808081, + "loss": 2.5171, + "theoretical_loss": 3.328151551545258, + "tokens_seen": 2914779136 + }, + { + "epoch": 0.76, + "learning_rate": 0.0001187878787878788, + "loss": 2.6134, + "theoretical_loss": 3.3281457644426764, + "tokens_seen": 2914844672 + }, + { + "epoch": 0.76, + "learning_rate": 0.00011876767676767678, + "loss": 2.5501, + "theoretical_loss": 3.3281399775066394, + "tokens_seen": 2914910208 + }, + { + "epoch": 0.76, + "objective/train/docs_used": 1641100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6307010650634766, + "objective/train/theoretical_loss": 3.3281341907371376, + "objective/train/tokens_used": 1273834976, + "theoretical_loss": 3.3281341907371376, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011874747474747475, + "loss": 2.6095, + "theoretical_loss": 3.3281341907371376, + "tokens_seen": 2914975744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011872727272727272, + "loss": 2.4336, + "theoretical_loss": 3.328128404134163, + "tokens_seen": 2915041280 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001187070707070707, + "loss": 2.656, + "theoretical_loss": 3.328122617697707, + "tokens_seen": 2915106816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011868686868686869, + "loss": 2.6226, + "theoretical_loss": 3.328116831427761, + "tokens_seen": 2915172352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011866666666666667, + "loss": 2.6518, + "theoretical_loss": 3.328111045324316, + "tokens_seen": 2915237888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011864646464646466, + "loss": 2.6918, + "theoretical_loss": 3.328105259387365, + "tokens_seen": 2915303424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011862626262626263, + "loss": 2.5337, + "theoretical_loss": 3.3280994736168976, + "tokens_seen": 2915368960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011860606060606061, + "loss": 2.5038, + "theoretical_loss": 3.328093688012906, + "tokens_seen": 2915434496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011858585858585858, + "loss": 2.7831, + "theoretical_loss": 3.3280879025753825, + "tokens_seen": 2915500032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011856565656565657, + "loss": 2.8135, + "theoretical_loss": 3.328082117304317, + "tokens_seen": 2915565568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011854545454545455, + "loss": 2.6282, + "theoretical_loss": 3.3280763321997022, + "tokens_seen": 2915631104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011852525252525252, + "loss": 2.5115, + "theoretical_loss": 3.328070547261529, + "tokens_seen": 2915696640 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001185050505050505, + "loss": 2.4096, + "theoretical_loss": 3.328064762489789, + "tokens_seen": 2915762176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011848484848484849, + "loss": 2.258, + "theoretical_loss": 3.3280589778844742, + "tokens_seen": 2915827712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011846464646464647, + "loss": 2.3294, + "theoretical_loss": 3.328053193445575, + "tokens_seen": 2915893248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011844444444444446, + "loss": 2.6884, + "theoretical_loss": 3.3280474091730836, + "tokens_seen": 2915958784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011842424242424243, + "loss": 2.6886, + "theoretical_loss": 3.3280416250669913, + "tokens_seen": 2916024320 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001184040404040404, + "loss": 2.4349, + "theoretical_loss": 3.32803584112729, + "tokens_seen": 2916089856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011838383838383838, + "loss": 2.5063, + "theoretical_loss": 3.32803005735397, + "tokens_seen": 2916155392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011836363636363637, + "loss": 2.4909, + "theoretical_loss": 3.3280242737470243, + "tokens_seen": 2916220928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011834343434343435, + "loss": 2.4312, + "theoretical_loss": 3.328018490306443, + "tokens_seen": 2916286464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011832323232323232, + "loss": 2.4042, + "theoretical_loss": 3.3280127070322187, + "tokens_seen": 2916352000 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001183030303030303, + "loss": 2.5655, + "theoretical_loss": 3.328006923924342, + "tokens_seen": 2916417536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011828282828282829, + "loss": 2.6435, + "theoretical_loss": 3.328001140982805, + "tokens_seen": 2916483072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011826262626262627, + "loss": 2.475, + "theoretical_loss": 3.3279953582075987, + "tokens_seen": 2916548608 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1641732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8531534671783447, + "objective/train/theoretical_loss": 3.327989575598715, + "objective/train/tokens_used": 1275473376, + "theoretical_loss": 3.327989575598715, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011824242424242426, + "loss": 2.6461, + "theoretical_loss": 3.327989575598715, + "tokens_seen": 2916614144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011822222222222223, + "loss": 2.7075, + "theoretical_loss": 3.327983793156145, + "tokens_seen": 2916679680 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001182020202020202, + "loss": 2.5627, + "theoretical_loss": 3.3279780108798804, + "tokens_seen": 2916745216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011818181818181818, + "loss": 2.7254, + "theoretical_loss": 3.3279722287699127, + "tokens_seen": 2916810752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011816161616161616, + "loss": 2.4487, + "theoretical_loss": 3.3279664468262333, + "tokens_seen": 2916876288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011814141414141415, + "loss": 2.5534, + "theoretical_loss": 3.327960665048834, + "tokens_seen": 2916941824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011812121212121213, + "loss": 2.6555, + "theoretical_loss": 3.3279548834377053, + "tokens_seen": 2917007360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001181010101010101, + "loss": 2.5036, + "theoretical_loss": 3.3279491019928398, + "tokens_seen": 2917072896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011808080808080809, + "loss": 2.4623, + "theoretical_loss": 3.3279433207142284, + "tokens_seen": 2917138432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011806060606060606, + "loss": 2.4863, + "theoretical_loss": 3.327937539601863, + "tokens_seen": 2917203968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011804040404040404, + "loss": 2.6084, + "theoretical_loss": 3.3279317586557347, + "tokens_seen": 2917269504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011802020202020202, + "loss": 2.3931, + "theoretical_loss": 3.327925977875835, + "tokens_seen": 2917335040 + }, + { + "epoch": 0.77, + "learning_rate": 0.000118, + "loss": 2.5939, + "theoretical_loss": 3.3279201972621557, + "tokens_seen": 2917400576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011797979797979798, + "loss": 2.4126, + "theoretical_loss": 3.3279144168146875, + "tokens_seen": 2917466112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011795959595959596, + "loss": 2.5968, + "theoretical_loss": 3.327908636533423, + "tokens_seen": 2917531648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011793939393939395, + "loss": 2.7013, + "theoretical_loss": 3.327902856418353, + "tokens_seen": 2917597184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011791919191919193, + "loss": 2.772, + "theoretical_loss": 3.3278970764694695, + "tokens_seen": 2917662720 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001178989898989899, + "loss": 2.4625, + "theoretical_loss": 3.3278912966867633, + "tokens_seen": 2917728256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011787878787878787, + "loss": 2.4673, + "theoretical_loss": 3.3278855170702264, + "tokens_seen": 2917793792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011785858585858586, + "loss": 2.6064, + "theoretical_loss": 3.32787973761985, + "tokens_seen": 2917859328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011783838383838384, + "loss": 2.8013, + "theoretical_loss": 3.3278739583356254, + "tokens_seen": 2917924864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011781818181818182, + "loss": 2.3719, + "theoretical_loss": 3.327868179217545, + "tokens_seen": 2917990400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011779797979797981, + "loss": 2.5525, + "theoretical_loss": 3.327862400265599, + "tokens_seen": 2918055936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011777777777777778, + "loss": 2.2643, + "theoretical_loss": 3.3278566214797802, + "tokens_seen": 2918121472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011775757575757576, + "loss": 2.5462, + "theoretical_loss": 3.3278508428600793, + "tokens_seen": 2918187008 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1643189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.778437852859497, + "objective/train/theoretical_loss": 3.3278450644064876, + "objective/train/tokens_used": 1277111776, + "theoretical_loss": 3.3278450644064876, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011773737373737375, + "loss": 2.843, + "theoretical_loss": 3.3278450644064876, + "tokens_seen": 2918252544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011771717171717172, + "loss": 2.4468, + "theoretical_loss": 3.3278392861189974, + "tokens_seen": 2918318080 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001176969696969697, + "loss": 2.5335, + "theoretical_loss": 3.3278335079975996, + "tokens_seen": 2918383616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011767676767676767, + "loss": 2.6209, + "theoretical_loss": 3.327827730042286, + "tokens_seen": 2918449152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011765656565656565, + "loss": 2.645, + "theoretical_loss": 3.3278219522530477, + "tokens_seen": 2918514688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011763636363636364, + "loss": 2.5308, + "theoretical_loss": 3.3278161746298762, + "tokens_seen": 2918580224 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011761616161616162, + "loss": 2.5399, + "theoretical_loss": 3.327810397172764, + "tokens_seen": 2918645760 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001175959595959596, + "loss": 2.5777, + "theoretical_loss": 3.3278046198817015, + "tokens_seen": 2918711296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011757575757575758, + "loss": 2.4247, + "theoretical_loss": 3.3277988427566805, + "tokens_seen": 2918776832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011755555555555556, + "loss": 2.5673, + "theoretical_loss": 3.3277930657976924, + "tokens_seen": 2918842368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011753535353535353, + "loss": 2.5782, + "theoretical_loss": 3.3277872890047293, + "tokens_seen": 2918907904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011751515151515152, + "loss": 2.6169, + "theoretical_loss": 3.327781512377782, + "tokens_seen": 2918973440 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001174949494949495, + "loss": 2.5692, + "theoretical_loss": 3.327775735916842, + "tokens_seen": 2919038976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011747474747474747, + "loss": 2.4682, + "theoretical_loss": 3.327769959621901, + "tokens_seen": 2919104512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011745454545454545, + "loss": 2.3578, + "theoretical_loss": 3.327764183492951, + "tokens_seen": 2919170048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011743434343434344, + "loss": 2.4357, + "theoretical_loss": 3.327758407529983, + "tokens_seen": 2919235584 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011741414141414142, + "loss": 2.6274, + "theoretical_loss": 3.3277526317329884, + "tokens_seen": 2919301120 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001173939393939394, + "loss": 2.7567, + "theoretical_loss": 3.327746856101959, + "tokens_seen": 2919366656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011737373737373738, + "loss": 2.6214, + "theoretical_loss": 3.327741080636886, + "tokens_seen": 2919432192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011735353535353535, + "loss": 2.5302, + "theoretical_loss": 3.327735305337761, + "tokens_seen": 2919497728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011733333333333333, + "loss": 2.5634, + "theoretical_loss": 3.327729530204576, + "tokens_seen": 2919563264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011731313131313131, + "loss": 2.7403, + "theoretical_loss": 3.327723755237322, + "tokens_seen": 2919628800 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001172929292929293, + "loss": 2.5332, + "theoretical_loss": 3.32771798043599, + "tokens_seen": 2919694336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011727272727272728, + "loss": 2.5978, + "theoretical_loss": 3.3277122058005726, + "tokens_seen": 2919759872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011725252525252525, + "loss": 2.7227, + "theoretical_loss": 3.3277064313310607, + "tokens_seen": 2919825408 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1643758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5002212524414062, + "objective/train/theoretical_loss": 3.327700657027446, + "objective/train/tokens_used": 1278750176, + "theoretical_loss": 3.327700657027446, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011723232323232324, + "loss": 2.513, + "theoretical_loss": 3.327700657027446, + "tokens_seen": 2919890944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011721212121212122, + "loss": 2.3788, + "theoretical_loss": 3.32769488288972, + "tokens_seen": 2919956480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011719191919191919, + "loss": 2.7757, + "theoretical_loss": 3.327689108917874, + "tokens_seen": 2920022016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011717171717171717, + "loss": 2.5599, + "theoretical_loss": 3.3276833351119, + "tokens_seen": 2920087552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011715151515151515, + "loss": 2.4326, + "theoretical_loss": 3.3276775614717886, + "tokens_seen": 2920153088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011713131313131313, + "loss": 2.4039, + "theoretical_loss": 3.3276717879975326, + "tokens_seen": 2920218624 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011711111111111111, + "loss": 2.4107, + "theoretical_loss": 3.327666014689122, + "tokens_seen": 2920284160 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001170909090909091, + "loss": 2.6383, + "theoretical_loss": 3.3276602415465497, + "tokens_seen": 2920349696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011707070707070708, + "loss": 2.7219, + "theoretical_loss": 3.3276544685698064, + "tokens_seen": 2920415232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011705050505050505, + "loss": 2.5217, + "theoretical_loss": 3.3276486957588842, + "tokens_seen": 2920480768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011703030303030302, + "loss": 2.4416, + "theoretical_loss": 3.327642923113774, + "tokens_seen": 2920546304 + }, + { + "epoch": 0.77, + "learning_rate": 0.000117010101010101, + "loss": 2.6189, + "theoretical_loss": 3.3276371506344673, + "tokens_seen": 2920611840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011698989898989899, + "loss": 2.6478, + "theoretical_loss": 3.327631378320956, + "tokens_seen": 2920677376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011696969696969697, + "loss": 2.5886, + "theoretical_loss": 3.327625606173232, + "tokens_seen": 2920742912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011694949494949496, + "loss": 2.5226, + "theoretical_loss": 3.3276198341912857, + "tokens_seen": 2920808448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011692929292929293, + "loss": 2.6312, + "theoretical_loss": 3.3276140623751096, + "tokens_seen": 2920873984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011692929292929293, + "loss": 2.6613, + "theoretical_loss": 3.3276082907246947, + "tokens_seen": 2920939520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011690909090909091, + "loss": 2.5223, + "theoretical_loss": 3.327602519240033, + "tokens_seen": 2921005056 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001168888888888889, + "loss": 2.7189, + "theoretical_loss": 3.3275967479211155, + "tokens_seen": 2921070592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011686868686868688, + "loss": 2.5712, + "theoretical_loss": 3.327590976767934, + "tokens_seen": 2921136128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011684848484848485, + "loss": 2.592, + "theoretical_loss": 3.32758520578048, + "tokens_seen": 2921201664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011682828282828282, + "loss": 2.5518, + "theoretical_loss": 3.3275794349587446, + "tokens_seen": 2921267200 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001168080808080808, + "loss": 2.5302, + "theoretical_loss": 3.3275736643027205, + "tokens_seen": 2921332736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011678787878787879, + "loss": 2.475, + "theoretical_loss": 3.3275678938123976, + "tokens_seen": 2921398272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011676767676767677, + "loss": 2.556, + "theoretical_loss": 3.327562123487769, + "tokens_seen": 2921463808 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1645135, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493926525115967, + "objective/train/theoretical_loss": 3.327556353328825, + "objective/train/tokens_used": 1280388576, + "theoretical_loss": 3.327556353328825, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011674747474747476, + "loss": 2.4736, + "theoretical_loss": 3.327556353328825, + "tokens_seen": 2921529344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011672727272727273, + "loss": 2.6637, + "theoretical_loss": 3.327550583335557, + "tokens_seen": 2921594880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011670707070707071, + "loss": 2.7112, + "theoretical_loss": 3.3275448135079584, + "tokens_seen": 2921660416 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166868686868687, + "loss": 2.4546, + "theoretical_loss": 3.3275390438460186, + "tokens_seen": 2921725952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011666666666666667, + "loss": 2.4889, + "theoretical_loss": 3.3275332743497303, + "tokens_seen": 2921791488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011664646464646465, + "loss": 2.5336, + "theoretical_loss": 3.3275275050190847, + "tokens_seen": 2921857024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011662626262626262, + "loss": 2.6368, + "theoretical_loss": 3.3275217358540736, + "tokens_seen": 2921922560 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001166060606060606, + "loss": 2.5613, + "theoretical_loss": 3.327515966854688, + "tokens_seen": 2921988096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011658585858585859, + "loss": 2.4616, + "theoretical_loss": 3.3275101980209194, + "tokens_seen": 2922053632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011656565656565657, + "loss": 2.8013, + "theoretical_loss": 3.32750442935276, + "tokens_seen": 2922119168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011654545454545456, + "loss": 2.6478, + "theoretical_loss": 3.3274986608502006, + "tokens_seen": 2922184704 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011652525252525254, + "loss": 2.5223, + "theoretical_loss": 3.327492892513234, + "tokens_seen": 2922250240 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001165050505050505, + "loss": 2.6586, + "theoretical_loss": 3.3274871243418502, + "tokens_seen": 2922315776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011648484848484848, + "loss": 2.5097, + "theoretical_loss": 3.3274813563360413, + "tokens_seen": 2922381312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011646464646464646, + "loss": 2.6803, + "theoretical_loss": 3.327475588495799, + "tokens_seen": 2922446848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011644444444444445, + "loss": 2.5853, + "theoretical_loss": 3.3274698208211144, + "tokens_seen": 2922512384 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011642424242424243, + "loss": 2.5186, + "theoretical_loss": 3.32746405331198, + "tokens_seen": 2922577920 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001164040404040404, + "loss": 2.5633, + "theoretical_loss": 3.3274582859683863, + "tokens_seen": 2922643456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011638383838383839, + "loss": 2.6398, + "theoretical_loss": 3.3274525187903254, + "tokens_seen": 2922708992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011636363636363637, + "loss": 2.3977, + "theoretical_loss": 3.3274467517777886, + "tokens_seen": 2922774528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011634343434343435, + "loss": 2.6563, + "theoretical_loss": 3.327440984930768, + "tokens_seen": 2922840064 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011632323232323233, + "loss": 2.5604, + "theoretical_loss": 3.3274352182492537, + "tokens_seen": 2922905600 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001163030303030303, + "loss": 2.5174, + "theoretical_loss": 3.327429451733239, + "tokens_seen": 2922971136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011628282828282828, + "loss": 2.5628, + "theoretical_loss": 3.3274236853827146, + "tokens_seen": 2923036672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011626262626262626, + "loss": 2.4193, + "theoretical_loss": 3.3274179191976714, + "tokens_seen": 2923102208 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1645886, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6343181133270264, + "objective/train/theoretical_loss": 3.327412153178102, + "objective/train/tokens_used": 1282026976, + "theoretical_loss": 3.327412153178102, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011624242424242425, + "loss": 2.5606, + "theoretical_loss": 3.327412153178102, + "tokens_seen": 2923167744 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011622222222222223, + "loss": 2.535, + "theoretical_loss": 3.327406387323998, + "tokens_seen": 2923233280 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001162020202020202, + "loss": 2.3732, + "theoretical_loss": 3.32740062163535, + "tokens_seen": 2923298816 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011618181818181819, + "loss": 2.6888, + "theoretical_loss": 3.32739485611215, + "tokens_seen": 2923364352 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011616161616161616, + "loss": 2.4248, + "theoretical_loss": 3.32738909075439, + "tokens_seen": 2923429888 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011614141414141414, + "loss": 2.5728, + "theoretical_loss": 3.327383325562061, + "tokens_seen": 2923495424 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011612121212121212, + "loss": 2.6446, + "theoretical_loss": 3.3273775605351545, + "tokens_seen": 2923560960 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011610101010101011, + "loss": 2.6405, + "theoretical_loss": 3.3273717956736624, + "tokens_seen": 2923626496 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011608080808080808, + "loss": 2.802, + "theoretical_loss": 3.3273660309775757, + "tokens_seen": 2923692032 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011606060606060606, + "loss": 2.5502, + "theoretical_loss": 3.327360266446887, + "tokens_seen": 2923757568 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011604040404040405, + "loss": 2.5492, + "theoretical_loss": 3.3273545020815867, + "tokens_seen": 2923823104 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011602020202020203, + "loss": 2.5093, + "theoretical_loss": 3.3273487378816666, + "tokens_seen": 2923888640 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011600000000000001, + "loss": 2.6017, + "theoretical_loss": 3.327342973847119, + "tokens_seen": 2923954176 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011597979797979797, + "loss": 2.3247, + "theoretical_loss": 3.3273372099779346, + "tokens_seen": 2924019712 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011595959595959596, + "loss": 2.6537, + "theoretical_loss": 3.3273314462741053, + "tokens_seen": 2924085248 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011593939393939394, + "loss": 2.3852, + "theoretical_loss": 3.327325682735623, + "tokens_seen": 2924150784 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011591919191919192, + "loss": 2.4848, + "theoretical_loss": 3.3273199193624783, + "tokens_seen": 2924216320 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011589898989898991, + "loss": 2.3849, + "theoretical_loss": 3.3273141561546633, + "tokens_seen": 2924281856 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011587878787878788, + "loss": 2.4644, + "theoretical_loss": 3.32730839311217, + "tokens_seen": 2924347392 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011585858585858586, + "loss": 2.286, + "theoretical_loss": 3.327302630234989, + "tokens_seen": 2924412928 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011583838383838385, + "loss": 2.553, + "theoretical_loss": 3.327296867523113, + "tokens_seen": 2924478464 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011581818181818183, + "loss": 2.613, + "theoretical_loss": 3.327291104976532, + "tokens_seen": 2924544000 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001157979797979798, + "loss": 2.3257, + "theoretical_loss": 3.3272853425952396, + "tokens_seen": 2924609536 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011577777777777777, + "loss": 2.5132, + "theoretical_loss": 3.3272795803792254, + "tokens_seen": 2924675072 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011575757575757575, + "loss": 2.5592, + "theoretical_loss": 3.3272738183284822, + "tokens_seen": 2924740608 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1647115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7834601402282715, + "objective/train/theoretical_loss": 3.327268056443001, + "objective/train/tokens_used": 1283665376, + "theoretical_loss": 3.327268056443001, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011573737373737374, + "loss": 2.7906, + "theoretical_loss": 3.327268056443001, + "tokens_seen": 2924806144 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011571717171717172, + "loss": 2.5977, + "theoretical_loss": 3.3272622947227735, + "tokens_seen": 2924871680 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001156969696969697, + "loss": 2.3691, + "theoretical_loss": 3.3272565331677915, + "tokens_seen": 2924937216 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011567676767676769, + "loss": 2.505, + "theoretical_loss": 3.327250771778046, + "tokens_seen": 2925002752 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011565656565656566, + "loss": 2.3388, + "theoretical_loss": 3.327245010553529, + "tokens_seen": 2925068288 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011563636363636363, + "loss": 2.4135, + "theoretical_loss": 3.327239249494232, + "tokens_seen": 2925133824 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011561616161616161, + "loss": 2.2403, + "theoretical_loss": 3.3272334886001467, + "tokens_seen": 2925199360 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001155959595959596, + "loss": 2.7176, + "theoretical_loss": 3.3272277278712643, + "tokens_seen": 2925264896 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011557575757575758, + "loss": 2.5932, + "theoretical_loss": 3.327221967307576, + "tokens_seen": 2925330432 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011555555555555555, + "loss": 2.6039, + "theoretical_loss": 3.3272162069090747, + "tokens_seen": 2925395968 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011553535353535354, + "loss": 2.6184, + "theoretical_loss": 3.3272104466757506, + "tokens_seen": 2925461504 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011551515151515152, + "loss": 2.4378, + "theoretical_loss": 3.327204686607596, + "tokens_seen": 2925527040 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001154949494949495, + "loss": 2.6448, + "theoretical_loss": 3.3271989267046025, + "tokens_seen": 2925592576 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011547474747474749, + "loss": 2.3662, + "theoretical_loss": 3.327193166966761, + "tokens_seen": 2925658112 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011545454545454545, + "loss": 2.502, + "theoretical_loss": 3.3271874073940637, + "tokens_seen": 2925723648 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011543434343434343, + "loss": 2.6889, + "theoretical_loss": 3.327181647986502, + "tokens_seen": 2925789184 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011541414141414141, + "loss": 2.5508, + "theoretical_loss": 3.3271758887440677, + "tokens_seen": 2925854720 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001153939393939394, + "loss": 2.4621, + "theoretical_loss": 3.3271701296667517, + "tokens_seen": 2925920256 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011537373737373738, + "loss": 2.4686, + "theoretical_loss": 3.327164370754546, + "tokens_seen": 2925985792 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011535353535353535, + "loss": 2.54, + "theoretical_loss": 3.3271586120074423, + "tokens_seen": 2926051328 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011533333333333334, + "loss": 2.7744, + "theoretical_loss": 3.327152853425432, + "tokens_seen": 2926116864 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011531313131313132, + "loss": 2.5001, + "theoretical_loss": 3.327147095008507, + "tokens_seen": 2926182400 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011529292929292929, + "loss": 2.5447, + "theoretical_loss": 3.327141336756658, + "tokens_seen": 2926247936 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011527272727272727, + "loss": 2.6239, + "theoretical_loss": 3.327135578669877, + "tokens_seen": 2926313472 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011525252525252526, + "loss": 2.5526, + "theoretical_loss": 3.3271298207481563, + "tokens_seen": 2926379008 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1647827, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.342317819595337, + "objective/train/theoretical_loss": 3.3271240629914867, + "objective/train/tokens_used": 1285303776, + "theoretical_loss": 3.3271240629914867, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011523232323232323, + "loss": 2.3861, + "theoretical_loss": 3.3271240629914867, + "tokens_seen": 2926444544 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011521212121212121, + "loss": 2.536, + "theoretical_loss": 3.3271183053998596, + "tokens_seen": 2926510080 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151919191919192, + "loss": 2.6535, + "theoretical_loss": 3.3271125479732673, + "tokens_seen": 2926575616 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011517171717171718, + "loss": 2.4697, + "theoretical_loss": 3.3271067907117007, + "tokens_seen": 2926641152 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011515151515151516, + "loss": 2.4934, + "theoretical_loss": 3.327101033615152, + "tokens_seen": 2926706688 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011513131313131313, + "loss": 2.3914, + "theoretical_loss": 3.327095276683612, + "tokens_seen": 2926772224 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001151111111111111, + "loss": 2.5578, + "theoretical_loss": 3.327089519917073, + "tokens_seen": 2926837760 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011509090909090909, + "loss": 2.4129, + "theoretical_loss": 3.3270837633155264, + "tokens_seen": 2926903296 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011507070707070707, + "loss": 2.5632, + "theoretical_loss": 3.3270780068789634, + "tokens_seen": 2926968832 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011505050505050506, + "loss": 2.6374, + "theoretical_loss": 3.327072250607376, + "tokens_seen": 2927034368 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011503030303030303, + "loss": 2.548, + "theoretical_loss": 3.3270664945007553, + "tokens_seen": 2927099904 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011501010101010101, + "loss": 2.6215, + "theoretical_loss": 3.3270607385590933, + "tokens_seen": 2927165440 + }, + { + "epoch": 0.77, + "learning_rate": 0.000114989898989899, + "loss": 2.6203, + "theoretical_loss": 3.3270549827823817, + "tokens_seen": 2927230976 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011496969696969698, + "loss": 2.5791, + "theoretical_loss": 3.327049227170612, + "tokens_seen": 2927296512 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011494949494949495, + "loss": 2.514, + "theoretical_loss": 3.3270434717237753, + "tokens_seen": 2927362048 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011492929292929292, + "loss": 2.3878, + "theoretical_loss": 3.327037716441864, + "tokens_seen": 2927427584 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001149090909090909, + "loss": 2.4983, + "theoretical_loss": 3.3270319613248684, + "tokens_seen": 2927493120 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011488888888888889, + "loss": 2.5329, + "theoretical_loss": 3.3270262063727816, + "tokens_seen": 2927558656 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011486868686868687, + "loss": 2.2886, + "theoretical_loss": 3.327020451585594, + "tokens_seen": 2927624192 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011484848484848486, + "loss": 2.575, + "theoretical_loss": 3.327014696963298, + "tokens_seen": 2927689728 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011482828282828284, + "loss": 2.5739, + "theoretical_loss": 3.3270089425058846, + "tokens_seen": 2927755264 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011480808080808081, + "loss": 2.1627, + "theoretical_loss": 3.3270031882133457, + "tokens_seen": 2927820800 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001147878787878788, + "loss": 2.5283, + "theoretical_loss": 3.3269974340856727, + "tokens_seen": 2927886336 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011476767676767676, + "loss": 2.4521, + "theoretical_loss": 3.3269916801228576, + "tokens_seen": 2927951872 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011474747474747475, + "loss": 2.858, + "theoretical_loss": 3.3269859263248915, + "tokens_seen": 2928017408 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1648971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2960734367370605, + "objective/train/theoretical_loss": 3.326980172691766, + "objective/train/tokens_used": 1286942176, + "theoretical_loss": 3.326980172691766, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011472727272727273, + "loss": 2.4445, + "theoretical_loss": 3.326980172691766, + "tokens_seen": 2928082944 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001147070707070707, + "loss": 2.3698, + "theoretical_loss": 3.326974419223473, + "tokens_seen": 2928148480 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011468686868686869, + "loss": 2.411, + "theoretical_loss": 3.3269686659200044, + "tokens_seen": 2928214016 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011466666666666667, + "loss": 2.5367, + "theoretical_loss": 3.326962912781351, + "tokens_seen": 2928279552 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011464646464646466, + "loss": 2.5265, + "theoretical_loss": 3.3269571598075043, + "tokens_seen": 2928345088 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011462626262626264, + "loss": 2.3673, + "theoretical_loss": 3.3269514069984565, + "tokens_seen": 2928410624 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001146060606060606, + "loss": 2.5633, + "theoretical_loss": 3.3269456543541995, + "tokens_seen": 2928476160 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011458585858585858, + "loss": 2.5772, + "theoretical_loss": 3.3269399018747237, + "tokens_seen": 2928541696 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011456565656565656, + "loss": 2.5817, + "theoretical_loss": 3.3269341495600218, + "tokens_seen": 2928607232 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011454545454545455, + "loss": 2.4083, + "theoretical_loss": 3.326928397410085, + "tokens_seen": 2928672768 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011452525252525253, + "loss": 2.6642, + "theoretical_loss": 3.326922645424905, + "tokens_seen": 2928738304 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001145050505050505, + "loss": 2.5363, + "theoretical_loss": 3.326916893604473, + "tokens_seen": 2928803840 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011448484848484849, + "loss": 2.3529, + "theoretical_loss": 3.326911141948781, + "tokens_seen": 2928869376 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011446464646464647, + "loss": 2.55, + "theoretical_loss": 3.3269053904578203, + "tokens_seen": 2928934912 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011444444444444445, + "loss": 2.6739, + "theoretical_loss": 3.326899639131583, + "tokens_seen": 2929000448 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011442424242424242, + "loss": 2.5868, + "theoretical_loss": 3.3268938879700602, + "tokens_seen": 2929065984 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011440404040404041, + "loss": 2.4199, + "theoretical_loss": 3.3268881369732437, + "tokens_seen": 2929131520 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011438383838383838, + "loss": 2.569, + "theoretical_loss": 3.3268823861411247, + "tokens_seen": 2929197056 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011436363636363636, + "loss": 2.6381, + "theoretical_loss": 3.3268766354736954, + "tokens_seen": 2929262592 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011434343434343435, + "loss": 2.5706, + "theoretical_loss": 3.326870884970947, + "tokens_seen": 2929328128 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011432323232323233, + "loss": 2.4637, + "theoretical_loss": 3.3268651346328717, + "tokens_seen": 2929393664 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011430303030303031, + "loss": 2.4382, + "theoretical_loss": 3.32685938445946, + "tokens_seen": 2929459200 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011428282828282829, + "loss": 2.4199, + "theoretical_loss": 3.3268536344507047, + "tokens_seen": 2929524736 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011426262626262627, + "loss": 2.5585, + "theoretical_loss": 3.3268478846065968, + "tokens_seen": 2929590272 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011424242424242424, + "loss": 2.5758, + "theoretical_loss": 3.3268421349271278, + "tokens_seen": 2929655808 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1649763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8904199600219727, + "objective/train/theoretical_loss": 3.3268363854122893, + "objective/train/tokens_used": 1288580576, + "theoretical_loss": 3.3268363854122893, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011422222222222222, + "loss": 2.6136, + "theoretical_loss": 3.3268363854122893, + "tokens_seen": 2929721344 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011420202020202021, + "loss": 2.6195, + "theoretical_loss": 3.3268306360620734, + "tokens_seen": 2929786880 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011418181818181818, + "loss": 2.3823, + "theoretical_loss": 3.326824886876471, + "tokens_seen": 2929852416 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011416161616161616, + "loss": 2.5547, + "theoretical_loss": 3.326819137855474, + "tokens_seen": 2929917952 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011414141414141415, + "loss": 2.836, + "theoretical_loss": 3.3268133889990747, + "tokens_seen": 2929983488 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011412121212121213, + "loss": 2.4586, + "theoretical_loss": 3.3268076403072637, + "tokens_seen": 2930049024 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011410101010101011, + "loss": 2.564, + "theoretical_loss": 3.326801891780033, + "tokens_seen": 2930114560 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011408080808080807, + "loss": 2.4597, + "theoretical_loss": 3.3267961434173743, + "tokens_seen": 2930180096 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011406060606060605, + "loss": 2.6189, + "theoretical_loss": 3.326790395219279, + "tokens_seen": 2930245632 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011404040404040404, + "loss": 2.5007, + "theoretical_loss": 3.3267846471857387, + "tokens_seen": 2930311168 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011402020202020202, + "loss": 2.2672, + "theoretical_loss": 3.3267788993167455, + "tokens_seen": 2930376704 + }, + { + "epoch": 0.77, + "learning_rate": 0.000114, + "loss": 2.3811, + "theoretical_loss": 3.32677315161229, + "tokens_seen": 2930442240 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011397979797979799, + "loss": 2.465, + "theoretical_loss": 3.326767404072365, + "tokens_seen": 2930507776 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011395959595959596, + "loss": 2.5851, + "theoretical_loss": 3.3267616566969616, + "tokens_seen": 2930573312 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011393939393939394, + "loss": 2.5317, + "theoretical_loss": 3.3267559094860712, + "tokens_seen": 2930638848 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011391919191919193, + "loss": 2.6775, + "theoretical_loss": 3.3267501624396854, + "tokens_seen": 2930704384 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001138989898989899, + "loss": 2.4049, + "theoretical_loss": 3.3267444155577963, + "tokens_seen": 2930769920 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011387878787878788, + "loss": 2.6402, + "theoretical_loss": 3.326738668840395, + "tokens_seen": 2930835456 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011385858585858585, + "loss": 2.6442, + "theoretical_loss": 3.3267329222874733, + "tokens_seen": 2930900992 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011383838383838384, + "loss": 2.3752, + "theoretical_loss": 3.326727175899023, + "tokens_seen": 2930966528 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011381818181818182, + "loss": 2.5022, + "theoretical_loss": 3.326721429675035, + "tokens_seen": 2931032064 + }, + { + "epoch": 0.77, + "learning_rate": 0.0001137979797979798, + "loss": 2.3956, + "theoretical_loss": 3.326715683615502, + "tokens_seen": 2931097600 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011377777777777779, + "loss": 2.7124, + "theoretical_loss": 3.3267099377204152, + "tokens_seen": 2931163136 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011375757575757576, + "loss": 2.5307, + "theoretical_loss": 3.3267041919897657, + "tokens_seen": 2931228672 + }, + { + "epoch": 0.77, + "learning_rate": 0.00011373737373737373, + "loss": 2.5394, + "theoretical_loss": 3.3266984464235456, + "tokens_seen": 2931294208 + }, + { + "epoch": 0.77, + "objective/train/docs_used": 1650742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1454708576202393, + "objective/train/theoretical_loss": 3.326692701021747, + "objective/train/tokens_used": 1290218976, + "theoretical_loss": 3.326692701021747, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011371717171717171, + "loss": 2.4037, + "theoretical_loss": 3.326692701021747, + "tokens_seen": 2931359744 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001136969696969697, + "loss": 2.478, + "theoretical_loss": 3.32668695578436, + "tokens_seen": 2931425280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011367676767676768, + "loss": 2.2369, + "theoretical_loss": 3.326681210711378, + "tokens_seen": 2931490816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011365656565656565, + "loss": 2.5053, + "theoretical_loss": 3.3266754658027913, + "tokens_seen": 2931556352 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011363636363636364, + "loss": 2.3737, + "theoretical_loss": 3.3266697210585923, + "tokens_seen": 2931621888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011361616161616162, + "loss": 2.5981, + "theoretical_loss": 3.326663976478772, + "tokens_seen": 2931687424 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001135959595959596, + "loss": 2.5225, + "theoretical_loss": 3.3266582320633225, + "tokens_seen": 2931752960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011357575757575759, + "loss": 2.7154, + "theoretical_loss": 3.326652487812235, + "tokens_seen": 2931818496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011355555555555556, + "loss": 2.6137, + "theoretical_loss": 3.3266467437255023, + "tokens_seen": 2931884032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011353535353535353, + "loss": 2.3203, + "theoretical_loss": 3.3266409998031143, + "tokens_seen": 2931949568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011351515151515151, + "loss": 2.4229, + "theoretical_loss": 3.326635256045064, + "tokens_seen": 2932015104 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001134949494949495, + "loss": 2.4989, + "theoretical_loss": 3.3266295124513423, + "tokens_seen": 2932080640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011347474747474748, + "loss": 2.6069, + "theoretical_loss": 3.326623769021941, + "tokens_seen": 2932146176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011345454545454546, + "loss": 2.5562, + "theoretical_loss": 3.326618025756851, + "tokens_seen": 2932211712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011343434343434344, + "loss": 2.2405, + "theoretical_loss": 3.3266122826560656, + "tokens_seen": 2932277248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011341414141414142, + "loss": 2.5105, + "theoretical_loss": 3.326606539719575, + "tokens_seen": 2932342784 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001133939393939394, + "loss": 2.5135, + "theoretical_loss": 3.3266007969473717, + "tokens_seen": 2932408320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011337373737373737, + "loss": 2.4733, + "theoretical_loss": 3.3265950543394465, + "tokens_seen": 2932473856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011335353535353536, + "loss": 2.4358, + "theoretical_loss": 3.3265893118957917, + "tokens_seen": 2932539392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011333333333333333, + "loss": 2.6239, + "theoretical_loss": 3.326583569616399, + "tokens_seen": 2932604928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011331313131313131, + "loss": 2.4577, + "theoretical_loss": 3.326577827501259, + "tokens_seen": 2932670464 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001132929292929293, + "loss": 2.4497, + "theoretical_loss": 3.3265720855503647, + "tokens_seen": 2932736000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011327272727272728, + "loss": 2.5258, + "theoretical_loss": 3.326566343763707, + "tokens_seen": 2932801536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011325252525252526, + "loss": 2.4654, + "theoretical_loss": 3.326560602141277, + "tokens_seen": 2932867072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011323232323232323, + "loss": 2.3388, + "theoretical_loss": 3.3265548606830677, + "tokens_seen": 2932932608 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1651324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3218421936035156, + "objective/train/theoretical_loss": 3.3265491193890697, + "objective/train/tokens_used": 1291857376, + "theoretical_loss": 3.3265491193890697, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001132121212121212, + "loss": 2.2481, + "theoretical_loss": 3.3265491193890697, + "tokens_seen": 2932998144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011319191919191919, + "loss": 2.4323, + "theoretical_loss": 3.326543378259275, + "tokens_seen": 2933063680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011317171717171717, + "loss": 2.5462, + "theoretical_loss": 3.3265376372936752, + "tokens_seen": 2933129216 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011315151515151516, + "loss": 2.6299, + "theoretical_loss": 3.326531896492262, + "tokens_seen": 2933194752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011313131313131314, + "loss": 2.3835, + "theoretical_loss": 3.3265261558550265, + "tokens_seen": 2933260288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011311111111111111, + "loss": 2.5827, + "theoretical_loss": 3.326520415381961, + "tokens_seen": 2933325824 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001130909090909091, + "loss": 2.5696, + "theoretical_loss": 3.326514675073057, + "tokens_seen": 2933391360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011307070707070708, + "loss": 2.695, + "theoretical_loss": 3.326508934928306, + "tokens_seen": 2933456896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011305050505050506, + "loss": 2.4275, + "theoretical_loss": 3.3265031949476995, + "tokens_seen": 2933522432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011303030303030303, + "loss": 2.5666, + "theoretical_loss": 3.3264974551312294, + "tokens_seen": 2933587968 + }, + { + "epoch": 0.78, + "learning_rate": 0.000113010101010101, + "loss": 2.5157, + "theoretical_loss": 3.3264917154788876, + "tokens_seen": 2933653504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011298989898989899, + "loss": 2.5012, + "theoretical_loss": 3.326485975990665, + "tokens_seen": 2933719040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011296969696969697, + "loss": 2.6455, + "theoretical_loss": 3.326480236666554, + "tokens_seen": 2933784576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011294949494949496, + "loss": 2.6274, + "theoretical_loss": 3.3264744975065454, + "tokens_seen": 2933850112 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011292929292929294, + "loss": 2.5013, + "theoretical_loss": 3.326468758510632, + "tokens_seen": 2933915648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011290909090909091, + "loss": 2.5505, + "theoretical_loss": 3.3264630196788043, + "tokens_seen": 2933981184 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001128888888888889, + "loss": 2.7142, + "theoretical_loss": 3.3264572810110544, + "tokens_seen": 2934046720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011286868686868686, + "loss": 2.27, + "theoretical_loss": 3.326451542507374, + "tokens_seen": 2934112256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011284848484848485, + "loss": 2.5171, + "theoretical_loss": 3.326445804167755, + "tokens_seen": 2934177792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011282828282828283, + "loss": 2.3916, + "theoretical_loss": 3.3264400659921884, + "tokens_seen": 2934243328 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001128080808080808, + "loss": 2.7093, + "theoretical_loss": 3.3264343279806665, + "tokens_seen": 2934308864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011278787878787879, + "loss": 2.6393, + "theoretical_loss": 3.3264285901331805, + "tokens_seen": 2934374400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011276767676767677, + "loss": 2.4998, + "theoretical_loss": 3.3264228524497224, + "tokens_seen": 2934439936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011274747474747475, + "loss": 2.4576, + "theoretical_loss": 3.3264171149302832, + "tokens_seen": 2934505472 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011272727272727274, + "loss": 2.4586, + "theoretical_loss": 3.326411377574855, + "tokens_seen": 2934571008 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1652566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4184749126434326, + "objective/train/theoretical_loss": 3.3264056403834297, + "objective/train/tokens_used": 1293495776, + "theoretical_loss": 3.3264056403834297, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011270707070707072, + "loss": 2.5848, + "theoretical_loss": 3.3264056403834297, + "tokens_seen": 2934636544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011268686868686868, + "loss": 2.676, + "theoretical_loss": 3.3263999033559983, + "tokens_seen": 2934702080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011266666666666666, + "loss": 2.5401, + "theoretical_loss": 3.3263941664925536, + "tokens_seen": 2934767616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011264646464646465, + "loss": 2.6977, + "theoretical_loss": 3.3263884297930857, + "tokens_seen": 2934833152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011262626262626263, + "loss": 2.4382, + "theoretical_loss": 3.3263826932575875, + "tokens_seen": 2934898688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011260606060606062, + "loss": 2.5703, + "theoretical_loss": 3.32637695688605, + "tokens_seen": 2934964224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011258585858585859, + "loss": 2.8037, + "theoretical_loss": 3.3263712206784652, + "tokens_seen": 2935029760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011256565656565657, + "loss": 2.8608, + "theoretical_loss": 3.3263654846348243, + "tokens_seen": 2935095296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011254545454545455, + "loss": 2.4244, + "theoretical_loss": 3.3263597487551193, + "tokens_seen": 2935160832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011252525252525254, + "loss": 2.4467, + "theoretical_loss": 3.3263540130393423, + "tokens_seen": 2935226368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011250505050505051, + "loss": 2.4958, + "theoretical_loss": 3.326348277487484, + "tokens_seen": 2935291904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011248484848484848, + "loss": 2.6211, + "theoretical_loss": 3.3263425420995367, + "tokens_seen": 2935357440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011246464646464646, + "loss": 2.5961, + "theoretical_loss": 3.3263368068754917, + "tokens_seen": 2935422976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011244444444444445, + "loss": 2.6151, + "theoretical_loss": 3.326331071815341, + "tokens_seen": 2935488512 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011242424242424243, + "loss": 2.6796, + "theoretical_loss": 3.326325336919076, + "tokens_seen": 2935554048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011240404040404041, + "loss": 2.3836, + "theoretical_loss": 3.3263196021866883, + "tokens_seen": 2935619584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011238383838383838, + "loss": 2.7159, + "theoretical_loss": 3.32631386761817, + "tokens_seen": 2935685120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011236363636363637, + "loss": 2.5775, + "theoretical_loss": 3.3263081332135123, + "tokens_seen": 2935750656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011234343434343434, + "loss": 2.5876, + "theoretical_loss": 3.326302398972707, + "tokens_seen": 2935816192 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011232323232323232, + "loss": 2.494, + "theoretical_loss": 3.3262966648957457, + "tokens_seen": 2935881728 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001123030303030303, + "loss": 2.5673, + "theoretical_loss": 3.32629093098262, + "tokens_seen": 2935947264 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011228282828282829, + "loss": 2.6719, + "theoretical_loss": 3.326285197233322, + "tokens_seen": 2936012800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011226262626262626, + "loss": 2.6399, + "theoretical_loss": 3.326279463647843, + "tokens_seen": 2936078336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011224242424242424, + "loss": 2.6466, + "theoretical_loss": 3.326273730226175, + "tokens_seen": 2936143872 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011222222222222223, + "loss": 2.666, + "theoretical_loss": 3.326267996968309, + "tokens_seen": 2936209408 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1653144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.965517044067383, + "objective/train/theoretical_loss": 3.3262622638742374, + "objective/train/tokens_used": 1295134176, + "theoretical_loss": 3.3262622638742374, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011220202020202021, + "loss": 2.5653, + "theoretical_loss": 3.3262622638742374, + "tokens_seen": 2936274944 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001121818181818182, + "loss": 2.4406, + "theoretical_loss": 3.3262565309439514, + "tokens_seen": 2936340480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011216161616161615, + "loss": 2.6642, + "theoretical_loss": 3.3262507981774423, + "tokens_seen": 2936406016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011214141414141414, + "loss": 2.649, + "theoretical_loss": 3.326245065574703, + "tokens_seen": 2936471552 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011212121212121212, + "loss": 2.613, + "theoretical_loss": 3.326239333135724, + "tokens_seen": 2936537088 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001121010101010101, + "loss": 2.4232, + "theoretical_loss": 3.3262336008604976, + "tokens_seen": 2936602624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011208080808080809, + "loss": 2.5334, + "theoretical_loss": 3.326227868749015, + "tokens_seen": 2936668160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011206060606060606, + "loss": 2.6365, + "theoretical_loss": 3.3262221368012685, + "tokens_seen": 2936733696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011204040404040404, + "loss": 2.321, + "theoretical_loss": 3.3262164050172487, + "tokens_seen": 2936799232 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011202020202020203, + "loss": 2.7328, + "theoretical_loss": 3.3262106733969485, + "tokens_seen": 2936864768 + }, + { + "epoch": 0.78, + "learning_rate": 0.000112, + "loss": 2.5556, + "theoretical_loss": 3.326204941940359, + "tokens_seen": 2936930304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011197979797979798, + "loss": 2.498, + "theoretical_loss": 3.326199210647472, + "tokens_seen": 2936995840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011195959595959595, + "loss": 2.6715, + "theoretical_loss": 3.3261934795182793, + "tokens_seen": 2937061376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011193939393939394, + "loss": 2.6424, + "theoretical_loss": 3.3261877485527718, + "tokens_seen": 2937126912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011191919191919192, + "loss": 2.7403, + "theoretical_loss": 3.326182017750942, + "tokens_seen": 2937192448 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001118989898989899, + "loss": 2.4883, + "theoretical_loss": 3.326176287112781, + "tokens_seen": 2937257984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011187878787878789, + "loss": 2.3247, + "theoretical_loss": 3.326170556638281, + "tokens_seen": 2937323520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011185858585858586, + "loss": 2.5941, + "theoretical_loss": 3.3261648263274335, + "tokens_seen": 2937389056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011183838383838384, + "loss": 2.7265, + "theoretical_loss": 3.32615909618023, + "tokens_seen": 2937454592 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011181818181818181, + "loss": 2.4176, + "theoretical_loss": 3.3261533661966625, + "tokens_seen": 2937520128 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001117979797979798, + "loss": 2.635, + "theoretical_loss": 3.3261476363767226, + "tokens_seen": 2937585664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011177777777777778, + "loss": 2.5128, + "theoretical_loss": 3.3261419067204017, + "tokens_seen": 2937651200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011175757575757577, + "loss": 2.4423, + "theoretical_loss": 3.326136177227691, + "tokens_seen": 2937716736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011173737373737374, + "loss": 2.5654, + "theoretical_loss": 3.3261304478985836, + "tokens_seen": 2937782272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011171717171717172, + "loss": 2.5985, + "theoretical_loss": 3.32612471873307, + "tokens_seen": 2937847808 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1654272, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.06006121635437, + "objective/train/theoretical_loss": 3.3261189897311425, + "objective/train/tokens_used": 1296772576, + "theoretical_loss": 3.3261189897311425, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001116969696969697, + "loss": 2.9258, + "theoretical_loss": 3.3261189897311425, + "tokens_seen": 2937913344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011167676767676769, + "loss": 2.3102, + "theoretical_loss": 3.326113260892792, + "tokens_seen": 2937978880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011165656565656566, + "loss": 2.6351, + "theoretical_loss": 3.3261075322180114, + "tokens_seen": 2938044416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011163636363636363, + "loss": 2.6943, + "theoretical_loss": 3.3261018037067913, + "tokens_seen": 2938109952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011161616161616161, + "loss": 2.4677, + "theoretical_loss": 3.326096075359124, + "tokens_seen": 2938175488 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001115959595959596, + "loss": 2.4423, + "theoretical_loss": 3.326090347175001, + "tokens_seen": 2938241024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011157575757575758, + "loss": 2.5352, + "theoretical_loss": 3.326084619154414, + "tokens_seen": 2938306560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011155555555555556, + "loss": 2.5401, + "theoretical_loss": 3.3260788912973545, + "tokens_seen": 2938372096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011153535353535353, + "loss": 2.4141, + "theoretical_loss": 3.326073163603814, + "tokens_seen": 2938437632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011151515151515152, + "loss": 2.4288, + "theoretical_loss": 3.326067436073785, + "tokens_seen": 2938503168 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001114949494949495, + "loss": 2.6697, + "theoretical_loss": 3.326061708707259, + "tokens_seen": 2938568704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011147474747474747, + "loss": 2.68, + "theoretical_loss": 3.3260559815042265, + "tokens_seen": 2938634240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011145454545454546, + "loss": 2.5522, + "theoretical_loss": 3.326050254464681, + "tokens_seen": 2938699776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011143434343434343, + "loss": 2.7398, + "theoretical_loss": 3.3260445275886124, + "tokens_seen": 2938765312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011141414141414141, + "loss": 2.4843, + "theoretical_loss": 3.3260388008760136, + "tokens_seen": 2938830848 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001113939393939394, + "loss": 2.6348, + "theoretical_loss": 3.326033074326876, + "tokens_seen": 2938896384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011137373737373738, + "loss": 2.6975, + "theoretical_loss": 3.326027347941191, + "tokens_seen": 2938961920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011135353535353536, + "loss": 2.3567, + "theoretical_loss": 3.326021621718951, + "tokens_seen": 2939027456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011133333333333335, + "loss": 2.3799, + "theoretical_loss": 3.3260158956601464, + "tokens_seen": 2939092992 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001113131313131313, + "loss": 2.6175, + "theoretical_loss": 3.3260101697647704, + "tokens_seen": 2939158528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011129292929292929, + "loss": 2.4094, + "theoretical_loss": 3.3260044440328134, + "tokens_seen": 2939224064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011127272727272727, + "loss": 2.5613, + "theoretical_loss": 3.325998718464268, + "tokens_seen": 2939289600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011125252525252526, + "loss": 2.6417, + "theoretical_loss": 3.325992993059126, + "tokens_seen": 2939355136 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011123232323232324, + "loss": 2.6505, + "theoretical_loss": 3.3259872678173776, + "tokens_seen": 2939420672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011121212121212121, + "loss": 2.5724, + "theoretical_loss": 3.3259815427390165, + "tokens_seen": 2939486208 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1655060, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4067819118499756, + "objective/train/theoretical_loss": 3.325975817824033, + "objective/train/tokens_used": 1298410976, + "theoretical_loss": 3.325975817824033, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001111919191919192, + "loss": 2.591, + "theoretical_loss": 3.325975817824033, + "tokens_seen": 2939551744 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011117171717171718, + "loss": 2.4166, + "theoretical_loss": 3.3259700930724194, + "tokens_seen": 2939617280 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011115151515151516, + "loss": 2.5475, + "theoretical_loss": 3.325964368484167, + "tokens_seen": 2939682816 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011113131313131313, + "loss": 2.4913, + "theoretical_loss": 3.325958644059268, + "tokens_seen": 2939748352 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001111111111111111, + "loss": 2.551, + "theoretical_loss": 3.325952919797714, + "tokens_seen": 2939813888 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011109090909090909, + "loss": 2.3442, + "theoretical_loss": 3.3259471956994964, + "tokens_seen": 2939879424 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011107070707070707, + "loss": 2.5323, + "theoretical_loss": 3.325941471764607, + "tokens_seen": 2939944960 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011105050505050505, + "loss": 2.5017, + "theoretical_loss": 3.3259357479930376, + "tokens_seen": 2940010496 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011103030303030304, + "loss": 2.5178, + "theoretical_loss": 3.3259300243847796, + "tokens_seen": 2940076032 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011101010101010101, + "loss": 2.5255, + "theoretical_loss": 3.325924300939825, + "tokens_seen": 2940141568 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011098989898989899, + "loss": 2.5387, + "theoretical_loss": 3.3259185776581655, + "tokens_seen": 2940207104 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011096969696969698, + "loss": 2.4173, + "theoretical_loss": 3.325912854539793, + "tokens_seen": 2940272640 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011094949494949495, + "loss": 2.5526, + "theoretical_loss": 3.3259071315846986, + "tokens_seen": 2940338176 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011092929292929293, + "loss": 2.6503, + "theoretical_loss": 3.325901408792874, + "tokens_seen": 2940403712 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011090909090909092, + "loss": 2.5499, + "theoretical_loss": 3.325895686164312, + "tokens_seen": 2940469248 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011088888888888889, + "loss": 2.4693, + "theoretical_loss": 3.3258899636990034, + "tokens_seen": 2940534784 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011086868686868687, + "loss": 2.6202, + "theoretical_loss": 3.3258842413969396, + "tokens_seen": 2940600320 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011084848484848485, + "loss": 2.6102, + "theoretical_loss": 3.325878519258113, + "tokens_seen": 2940665856 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011082828282828284, + "loss": 2.3616, + "theoretical_loss": 3.325872797282515, + "tokens_seen": 2940731392 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011080808080808082, + "loss": 2.6588, + "theoretical_loss": 3.3258670754701374, + "tokens_seen": 2940796928 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011078787878787878, + "loss": 2.5331, + "theoretical_loss": 3.325861353820972, + "tokens_seen": 2940862464 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011076767676767676, + "loss": 2.2444, + "theoretical_loss": 3.32585563233501, + "tokens_seen": 2940928000 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011074747474747475, + "loss": 2.4608, + "theoretical_loss": 3.325849911012244, + "tokens_seen": 2940993536 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011072727272727273, + "loss": 2.5875, + "theoretical_loss": 3.325844189852665, + "tokens_seen": 2941059072 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011070707070707071, + "loss": 2.3672, + "theoretical_loss": 3.325838468856265, + "tokens_seen": 2941124608 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1656552, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8251025676727295, + "objective/train/theoretical_loss": 3.325832748023035, + "objective/train/tokens_used": 1300049376, + "theoretical_loss": 3.325832748023035, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011068686868686868, + "loss": 2.6468, + "theoretical_loss": 3.325832748023035, + "tokens_seen": 2941190144 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011066666666666667, + "loss": 2.4242, + "theoretical_loss": 3.325827027352968, + "tokens_seen": 2941255680 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011064646464646465, + "loss": 2.2898, + "theoretical_loss": 3.3258213068460547, + "tokens_seen": 2941321216 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011062626262626264, + "loss": 2.542, + "theoretical_loss": 3.3258155865022876, + "tokens_seen": 2941386752 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011060606060606061, + "loss": 2.6807, + "theoretical_loss": 3.3258098663216575, + "tokens_seen": 2941452288 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011058585858585858, + "loss": 2.6328, + "theoretical_loss": 3.325804146304157, + "tokens_seen": 2941517824 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011056565656565656, + "loss": 2.5944, + "theoretical_loss": 3.325798426449777, + "tokens_seen": 2941583360 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011054545454545455, + "loss": 2.3863, + "theoretical_loss": 3.32579270675851, + "tokens_seen": 2941648896 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011052525252525253, + "loss": 2.6714, + "theoretical_loss": 3.3257869872303467, + "tokens_seen": 2941714432 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011050505050505051, + "loss": 2.6342, + "theoretical_loss": 3.32578126786528, + "tokens_seen": 2941779968 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001104848484848485, + "loss": 2.3921, + "theoretical_loss": 3.3257755486633007, + "tokens_seen": 2941845504 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011046464646464647, + "loss": 2.5612, + "theoretical_loss": 3.325769829624401, + "tokens_seen": 2941911040 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011044444444444444, + "loss": 2.3902, + "theoretical_loss": 3.3257641107485725, + "tokens_seen": 2941976576 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011042424242424242, + "loss": 2.7217, + "theoretical_loss": 3.325758392035807, + "tokens_seen": 2942042112 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001104040404040404, + "loss": 2.5132, + "theoretical_loss": 3.325752673486096, + "tokens_seen": 2942107648 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011038383838383839, + "loss": 2.7821, + "theoretical_loss": 3.325746955099431, + "tokens_seen": 2942173184 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011036363636363636, + "loss": 2.5915, + "theoretical_loss": 3.3257412368758046, + "tokens_seen": 2942238720 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011034343434343434, + "loss": 2.6523, + "theoretical_loss": 3.325735518815208, + "tokens_seen": 2942304256 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011032323232323233, + "loss": 2.5606, + "theoretical_loss": 3.3257298009176326, + "tokens_seen": 2942369792 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011030303030303031, + "loss": 2.5091, + "theoretical_loss": 3.3257240831830703, + "tokens_seen": 2942435328 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001102828282828283, + "loss": 2.6009, + "theoretical_loss": 3.3257183656115132, + "tokens_seen": 2942500864 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011026262626262625, + "loss": 2.3621, + "theoretical_loss": 3.325712648202953, + "tokens_seen": 2942566400 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011024242424242424, + "loss": 2.6317, + "theoretical_loss": 3.325706930957381, + "tokens_seen": 2942631936 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011022222222222222, + "loss": 2.4649, + "theoretical_loss": 3.3257012138747886, + "tokens_seen": 2942697472 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001102020202020202, + "loss": 2.3672, + "theoretical_loss": 3.3256954969551686, + "tokens_seen": 2942763008 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1657166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.474964141845703, + "objective/train/theoretical_loss": 3.325689780198512, + "objective/train/tokens_used": 1301687776, + "theoretical_loss": 3.325689780198512, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011018181818181819, + "loss": 2.4711, + "theoretical_loss": 3.325689780198512, + "tokens_seen": 2942828544 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011016161616161616, + "loss": 2.4446, + "theoretical_loss": 3.3256840636048106, + "tokens_seen": 2942894080 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011014141414141414, + "loss": 2.4233, + "theoretical_loss": 3.3256783471740565, + "tokens_seen": 2942959616 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011012121212121213, + "loss": 2.5989, + "theoretical_loss": 3.325672630906241, + "tokens_seen": 2943025152 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011010101010101011, + "loss": 2.5987, + "theoretical_loss": 3.325666914801356, + "tokens_seen": 2943090688 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011008080808080808, + "loss": 2.719, + "theoretical_loss": 3.325661198859393, + "tokens_seen": 2943156224 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011006060606060607, + "loss": 2.6229, + "theoretical_loss": 3.325655483080344, + "tokens_seen": 2943221760 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011004040404040404, + "loss": 2.5203, + "theoretical_loss": 3.325649767464201, + "tokens_seen": 2943287296 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011002020202020202, + "loss": 2.6911, + "theoretical_loss": 3.3256440520109547, + "tokens_seen": 2943352832 + }, + { + "epoch": 0.78, + "learning_rate": 0.00011, + "loss": 2.3613, + "theoretical_loss": 3.325638336720598, + "tokens_seen": 2943418368 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010997979797979799, + "loss": 2.565, + "theoretical_loss": 3.325632621593122, + "tokens_seen": 2943483904 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010995959595959597, + "loss": 2.4706, + "theoretical_loss": 3.3256269066285187, + "tokens_seen": 2943549440 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010993939393939394, + "loss": 2.6387, + "theoretical_loss": 3.3256211918267793, + "tokens_seen": 2943614976 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010991919191919191, + "loss": 2.3223, + "theoretical_loss": 3.3256154771878963, + "tokens_seen": 2943680512 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001098989898989899, + "loss": 2.5579, + "theoretical_loss": 3.3256097627118613, + "tokens_seen": 2943746048 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010987878787878788, + "loss": 2.5281, + "theoretical_loss": 3.3256040483986657, + "tokens_seen": 2943811584 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010985858585858586, + "loss": 2.6969, + "theoretical_loss": 3.3255983342483013, + "tokens_seen": 2943877120 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010983838383838383, + "loss": 2.5007, + "theoretical_loss": 3.3255926202607595, + "tokens_seen": 2943942656 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010981818181818182, + "loss": 2.7055, + "theoretical_loss": 3.325586906436033, + "tokens_seen": 2944008192 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001097979797979798, + "loss": 2.5172, + "theoretical_loss": 3.3255811927741123, + "tokens_seen": 2944073728 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010977777777777779, + "loss": 2.6539, + "theoretical_loss": 3.32557547927499, + "tokens_seen": 2944139264 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010975757575757577, + "loss": 2.6823, + "theoretical_loss": 3.325569765938658, + "tokens_seen": 2944204800 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010973737373737373, + "loss": 2.4898, + "theoretical_loss": 3.3255640527651074, + "tokens_seen": 2944270336 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010971717171717171, + "loss": 2.4629, + "theoretical_loss": 3.3255583397543305, + "tokens_seen": 2944335872 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001096969696969697, + "loss": 2.4172, + "theoretical_loss": 3.325552626906318, + "tokens_seen": 2944401408 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1658433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.838081121444702, + "objective/train/theoretical_loss": 3.325546914221063, + "objective/train/tokens_used": 1303326176, + "theoretical_loss": 3.325546914221063, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010967676767676768, + "loss": 2.7039, + "theoretical_loss": 3.325546914221063, + "tokens_seen": 2944466944 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010965656565656566, + "loss": 2.6065, + "theoretical_loss": 3.325541201698557, + "tokens_seen": 2944532480 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010963636363636365, + "loss": 2.6299, + "theoretical_loss": 3.3255354893387903, + "tokens_seen": 2944598016 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010961616161616162, + "loss": 2.3869, + "theoretical_loss": 3.3255297771417562, + "tokens_seen": 2944663552 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001095959595959596, + "loss": 2.6421, + "theoretical_loss": 3.3255240651074462, + "tokens_seen": 2944729088 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010957575757575757, + "loss": 2.7559, + "theoretical_loss": 3.325518353235852, + "tokens_seen": 2944794624 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010955555555555556, + "loss": 2.5677, + "theoretical_loss": 3.3255126415269647, + "tokens_seen": 2944860160 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010953535353535354, + "loss": 2.646, + "theoretical_loss": 3.3255069299807762, + "tokens_seen": 2944925696 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010951515151515151, + "loss": 2.3384, + "theoretical_loss": 3.325501218597279, + "tokens_seen": 2944991232 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001094949494949495, + "loss": 2.3998, + "theoretical_loss": 3.325495507376464, + "tokens_seen": 2945056768 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010947474747474748, + "loss": 2.428, + "theoretical_loss": 3.3254897963183234, + "tokens_seen": 2945122304 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010945454545454546, + "loss": 2.4612, + "theoretical_loss": 3.3254840854228496, + "tokens_seen": 2945187840 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010943434343434345, + "loss": 2.3654, + "theoretical_loss": 3.3254783746900327, + "tokens_seen": 2945253376 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010941414141414142, + "loss": 2.4147, + "theoretical_loss": 3.3254726641198658, + "tokens_seen": 2945318912 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010939393939393939, + "loss": 2.5069, + "theoretical_loss": 3.32546695371234, + "tokens_seen": 2945384448 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010937373737373737, + "loss": 2.5741, + "theoretical_loss": 3.3254612434674473, + "tokens_seen": 2945449984 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010935353535353536, + "loss": 2.4954, + "theoretical_loss": 3.3254555333851794, + "tokens_seen": 2945515520 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010933333333333334, + "loss": 2.2803, + "theoretical_loss": 3.325449823465528, + "tokens_seen": 2945581056 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010931313131313131, + "loss": 2.5645, + "theoretical_loss": 3.3254441137084854, + "tokens_seen": 2945646592 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001092929292929293, + "loss": 2.6017, + "theoretical_loss": 3.3254384041140423, + "tokens_seen": 2945712128 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010927272727272728, + "loss": 2.5337, + "theoretical_loss": 3.325432694682191, + "tokens_seen": 2945777664 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010925252525252526, + "loss": 2.4863, + "theoretical_loss": 3.325426985412924, + "tokens_seen": 2945843200 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010923232323232323, + "loss": 2.4718, + "theoretical_loss": 3.3254212763062316, + "tokens_seen": 2945908736 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010921212121212122, + "loss": 2.4284, + "theoretical_loss": 3.325415567362106, + "tokens_seen": 2945974272 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010919191919191919, + "loss": 2.7572, + "theoretical_loss": 3.32540985858054, + "tokens_seen": 2946039808 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1659139, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.635344982147217, + "objective/train/theoretical_loss": 3.3254041499615243, + "objective/train/tokens_used": 1304964576, + "theoretical_loss": 3.3254041499615243, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010917171717171717, + "loss": 2.3755, + "theoretical_loss": 3.3254041499615243, + "tokens_seen": 2946105344 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010915151515151515, + "loss": 2.6694, + "theoretical_loss": 3.325398441505051, + "tokens_seen": 2946170880 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010913131313131314, + "loss": 2.5264, + "theoretical_loss": 3.3253927332111117, + "tokens_seen": 2946236416 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010911111111111112, + "loss": 2.5501, + "theoretical_loss": 3.3253870250796984, + "tokens_seen": 2946301952 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010909090909090909, + "loss": 2.5536, + "theoretical_loss": 3.3253813171108026, + "tokens_seen": 2946367488 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010907070707070708, + "loss": 2.2876, + "theoretical_loss": 3.325375609304416, + "tokens_seen": 2946433024 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010905050505050505, + "loss": 2.4688, + "theoretical_loss": 3.3253699016605305, + "tokens_seen": 2946498560 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010903030303030303, + "loss": 2.5929, + "theoretical_loss": 3.325364194179138, + "tokens_seen": 2946564096 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010901010101010101, + "loss": 2.5399, + "theoretical_loss": 3.3253584868602304, + "tokens_seen": 2946629632 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010898989898989899, + "loss": 2.4968, + "theoretical_loss": 3.325352779703799, + "tokens_seen": 2946695168 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010896969696969697, + "loss": 2.4851, + "theoretical_loss": 3.3253470727098358, + "tokens_seen": 2946760704 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010894949494949495, + "loss": 2.2616, + "theoretical_loss": 3.3253413658783324, + "tokens_seen": 2946826240 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010892929292929294, + "loss": 2.5383, + "theoretical_loss": 3.325335659209281, + "tokens_seen": 2946891776 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010890909090909092, + "loss": 2.397, + "theoretical_loss": 3.3253299527026727, + "tokens_seen": 2946957312 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010888888888888888, + "loss": 2.4959, + "theoretical_loss": 3.3253242463585, + "tokens_seen": 2947022848 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010886868686868686, + "loss": 2.3648, + "theoretical_loss": 3.3253185401767538, + "tokens_seen": 2947088384 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010884848484848485, + "loss": 2.4668, + "theoretical_loss": 3.325312834157427, + "tokens_seen": 2947153920 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010882828282828283, + "loss": 2.408, + "theoretical_loss": 3.3253071283005102, + "tokens_seen": 2947219456 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010880808080808081, + "loss": 2.5994, + "theoretical_loss": 3.3253014226059956, + "tokens_seen": 2947284992 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001087878787878788, + "loss": 2.4376, + "theoretical_loss": 3.3252957170738755, + "tokens_seen": 2947350528 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010876767676767677, + "loss": 2.5654, + "theoretical_loss": 3.3252900117041406, + "tokens_seen": 2947416064 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010874747474747475, + "loss": 2.6076, + "theoretical_loss": 3.325284306496784, + "tokens_seen": 2947481600 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010872727272727274, + "loss": 2.4167, + "theoretical_loss": 3.325278601451796, + "tokens_seen": 2947547136 + }, + { + "epoch": 0.78, + "learning_rate": 0.0001087070707070707, + "loss": 2.671, + "theoretical_loss": 3.3252728965691696, + "tokens_seen": 2947612672 + }, + { + "epoch": 0.78, + "learning_rate": 0.00010868686868686869, + "loss": 2.7609, + "theoretical_loss": 3.325267191848896, + "tokens_seen": 2947678208 + }, + { + "epoch": 0.78, + "objective/train/docs_used": 1660399, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.444298505783081, + "objective/train/theoretical_loss": 3.325261487290967, + "objective/train/tokens_used": 1306602976, + "theoretical_loss": 3.325261487290967, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010866666666666666, + "loss": 2.6091, + "theoretical_loss": 3.325261487290967, + "tokens_seen": 2947743744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010864646464646464, + "loss": 2.4588, + "theoretical_loss": 3.3252557828953746, + "tokens_seen": 2947809280 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010862626262626263, + "loss": 2.5424, + "theoretical_loss": 3.32525007866211, + "tokens_seen": 2947874816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010860606060606061, + "loss": 2.4493, + "theoretical_loss": 3.3252443745911657, + "tokens_seen": 2947940352 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001085858585858586, + "loss": 2.7391, + "theoretical_loss": 3.325238670682533, + "tokens_seen": 2948005888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010856565656565657, + "loss": 2.445, + "theoretical_loss": 3.325232966936204, + "tokens_seen": 2948071424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010854545454545455, + "loss": 2.3828, + "theoretical_loss": 3.3252272633521707, + "tokens_seen": 2948136960 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010852525252525252, + "loss": 2.5012, + "theoretical_loss": 3.325221559930424, + "tokens_seen": 2948202496 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001085050505050505, + "loss": 2.5808, + "theoretical_loss": 3.325215856670956, + "tokens_seen": 2948268032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010848484848484849, + "loss": 2.4682, + "theoretical_loss": 3.3252101535737584, + "tokens_seen": 2948333568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010846464646464646, + "loss": 2.508, + "theoretical_loss": 3.325204450638824, + "tokens_seen": 2948399104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010844444444444444, + "loss": 2.5029, + "theoretical_loss": 3.325198747866143, + "tokens_seen": 2948464640 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010842424242424243, + "loss": 2.6339, + "theoretical_loss": 3.325193045255708, + "tokens_seen": 2948530176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010840404040404041, + "loss": 2.5864, + "theoretical_loss": 3.3251873428075114, + "tokens_seen": 2948595712 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083838383838384, + "loss": 2.3056, + "theoretical_loss": 3.325181640521544, + "tokens_seen": 2948661248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010836363636363637, + "loss": 2.3339, + "theoretical_loss": 3.3251759383977975, + "tokens_seen": 2948726784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010834343434343434, + "loss": 2.552, + "theoretical_loss": 3.3251702364362643, + "tokens_seen": 2948792320 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010832323232323232, + "loss": 2.4206, + "theoretical_loss": 3.325164534636936, + "tokens_seen": 2948857856 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001083030303030303, + "loss": 2.5701, + "theoretical_loss": 3.3251588329998043, + "tokens_seen": 2948923392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010828282828282829, + "loss": 2.4417, + "theoretical_loss": 3.325153131524861, + "tokens_seen": 2948988928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010826262626262627, + "loss": 2.4976, + "theoretical_loss": 3.325147430212098, + "tokens_seen": 2949054464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010824242424242424, + "loss": 2.2516, + "theoretical_loss": 3.3251417290615066, + "tokens_seen": 2949120000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010822222222222223, + "loss": 2.6426, + "theoretical_loss": 3.3251360280730795, + "tokens_seen": 2949185536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010820202020202021, + "loss": 2.4563, + "theoretical_loss": 3.3251303272468076, + "tokens_seen": 2949251072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010818181818181818, + "loss": 2.5374, + "theoretical_loss": 3.3251246265826833, + "tokens_seen": 2949316608 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1661100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9180805683135986, + "objective/train/theoretical_loss": 3.3251189260806977, + "objective/train/tokens_used": 1308241376, + "theoretical_loss": 3.3251189260806977, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010816161616161616, + "loss": 2.6553, + "theoretical_loss": 3.3251189260806977, + "tokens_seen": 2949382144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010814141414141414, + "loss": 2.4157, + "theoretical_loss": 3.325113225740843, + "tokens_seen": 2949447680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010812121212121212, + "loss": 2.2372, + "theoretical_loss": 3.3251075255631113, + "tokens_seen": 2949513216 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001081010101010101, + "loss": 2.618, + "theoretical_loss": 3.325101825547494, + "tokens_seen": 2949578752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010808080808080809, + "loss": 2.4315, + "theoretical_loss": 3.3250961256939826, + "tokens_seen": 2949644288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010806060606060607, + "loss": 2.4507, + "theoretical_loss": 3.3250904260025695, + "tokens_seen": 2949709824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010804040404040404, + "loss": 2.5827, + "theoretical_loss": 3.3250847264732464, + "tokens_seen": 2949775360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010802020202020201, + "loss": 2.4915, + "theoretical_loss": 3.325079027106005, + "tokens_seen": 2949840896 + }, + { + "epoch": 0.79, + "learning_rate": 0.000108, + "loss": 2.4491, + "theoretical_loss": 3.325073327900837, + "tokens_seen": 2949906432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010797979797979798, + "loss": 2.468, + "theoretical_loss": 3.3250676288577337, + "tokens_seen": 2949971968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010795959595959596, + "loss": 2.7037, + "theoretical_loss": 3.325061929976688, + "tokens_seen": 2950037504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010793939393939395, + "loss": 2.4953, + "theoretical_loss": 3.3250562312576903, + "tokens_seen": 2950103040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010791919191919192, + "loss": 2.647, + "theoretical_loss": 3.325050532700734, + "tokens_seen": 2950168576 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001078989898989899, + "loss": 2.1231, + "theoretical_loss": 3.3250448343058094, + "tokens_seen": 2950234112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010787878787878789, + "loss": 2.5181, + "theoretical_loss": 3.3250391360729097, + "tokens_seen": 2950299648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010785858585858587, + "loss": 2.4227, + "theoretical_loss": 3.325033438002025, + "tokens_seen": 2950365184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010783838383838384, + "loss": 2.5969, + "theoretical_loss": 3.325027740093149, + "tokens_seen": 2950430720 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010781818181818181, + "loss": 2.5299, + "theoretical_loss": 3.325022042346272, + "tokens_seen": 2950496256 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001077979797979798, + "loss": 2.4376, + "theoretical_loss": 3.3250163447613867, + "tokens_seen": 2950561792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010777777777777778, + "loss": 2.5822, + "theoretical_loss": 3.3250106473384844, + "tokens_seen": 2950627328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010775757575757576, + "loss": 2.6696, + "theoretical_loss": 3.325004950077557, + "tokens_seen": 2950692864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010773737373737375, + "loss": 2.2542, + "theoretical_loss": 3.3249992529785963, + "tokens_seen": 2950758400 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010771717171717172, + "loss": 2.571, + "theoretical_loss": 3.324993556041594, + "tokens_seen": 2950823936 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001076969696969697, + "loss": 2.6407, + "theoretical_loss": 3.3249878592665425, + "tokens_seen": 2950889472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010767676767676769, + "loss": 2.4354, + "theoretical_loss": 3.3249821626534324, + "tokens_seen": 2950955008 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1661802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.38539981842041, + "objective/train/theoretical_loss": 3.3249764662022567, + "objective/train/tokens_used": 1309879776, + "theoretical_loss": 3.3249764662022567, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010765656565656566, + "loss": 2.3324, + "theoretical_loss": 3.3249764662022567, + "tokens_seen": 2951020544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010763636363636364, + "loss": 2.2722, + "theoretical_loss": 3.3249707699130067, + "tokens_seen": 2951086080 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010761616161616161, + "loss": 2.3903, + "theoretical_loss": 3.324965073785674, + "tokens_seen": 2951151616 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001075959595959596, + "loss": 2.4268, + "theoretical_loss": 3.3249593778202513, + "tokens_seen": 2951217152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010757575757575758, + "loss": 2.6469, + "theoretical_loss": 3.324953682016729, + "tokens_seen": 2951282688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010755555555555556, + "loss": 2.4136, + "theoretical_loss": 3.3249479863751, + "tokens_seen": 2951348224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010753535353535355, + "loss": 2.6949, + "theoretical_loss": 3.324942290895356, + "tokens_seen": 2951413760 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010751515151515153, + "loss": 2.4201, + "theoretical_loss": 3.324936595577488, + "tokens_seen": 2951479296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010749494949494949, + "loss": 2.4954, + "theoretical_loss": 3.3249309004214886, + "tokens_seen": 2951544832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010747474747474747, + "loss": 2.4075, + "theoretical_loss": 3.3249252054273493, + "tokens_seen": 2951610368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010745454545454545, + "loss": 2.545, + "theoretical_loss": 3.324919510595062, + "tokens_seen": 2951675904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010743434343434344, + "loss": 2.5383, + "theoretical_loss": 3.3249138159246185, + "tokens_seen": 2951741440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010741414141414142, + "loss": 2.315, + "theoretical_loss": 3.3249081214160108, + "tokens_seen": 2951806976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010739393939393939, + "loss": 2.3969, + "theoretical_loss": 3.3249024270692304, + "tokens_seen": 2951872512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010737373737373738, + "loss": 2.3544, + "theoretical_loss": 3.3248967328842687, + "tokens_seen": 2951938048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010735353535353536, + "loss": 2.4656, + "theoretical_loss": 3.3248910388611184, + "tokens_seen": 2952003584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010733333333333334, + "loss": 2.3711, + "theoretical_loss": 3.324885344999771, + "tokens_seen": 2952069120 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010731313131313132, + "loss": 2.575, + "theoretical_loss": 3.3248796513002183, + "tokens_seen": 2952134656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010729292929292929, + "loss": 2.4434, + "theoretical_loss": 3.3248739577624518, + "tokens_seen": 2952200192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010727272727272727, + "loss": 2.3934, + "theoretical_loss": 3.3248682643864633, + "tokens_seen": 2952265728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010725252525252525, + "loss": 2.3603, + "theoretical_loss": 3.3248625711722455, + "tokens_seen": 2952331264 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010723232323232324, + "loss": 2.4076, + "theoretical_loss": 3.324856878119789, + "tokens_seen": 2952396800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010721212121212122, + "loss": 2.4806, + "theoretical_loss": 3.3248511852290865, + "tokens_seen": 2952462336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010719191919191919, + "loss": 2.5449, + "theoretical_loss": 3.3248454925001294, + "tokens_seen": 2952527872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010717171717171718, + "loss": 2.6971, + "theoretical_loss": 3.3248397999329096, + "tokens_seen": 2952593408 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1662642, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.936624765396118, + "objective/train/theoretical_loss": 3.324834107527419, + "objective/train/tokens_used": 1311518176, + "theoretical_loss": 3.324834107527419, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010715151515151515, + "loss": 2.5504, + "theoretical_loss": 3.324834107527419, + "tokens_seen": 2952658944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010713131313131313, + "loss": 2.4975, + "theoretical_loss": 3.3248284152836494, + "tokens_seen": 2952724480 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010711111111111111, + "loss": 2.5948, + "theoretical_loss": 3.3248227232015926, + "tokens_seen": 2952790016 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001070909090909091, + "loss": 2.3391, + "theoretical_loss": 3.32481703128124, + "tokens_seen": 2952855552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010707070707070707, + "loss": 2.4971, + "theoretical_loss": 3.3248113395225842, + "tokens_seen": 2952921088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010705050505050505, + "loss": 2.3215, + "theoretical_loss": 3.3248056479256167, + "tokens_seen": 2952986624 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010703030303030304, + "loss": 2.3376, + "theoretical_loss": 3.324799956490329, + "tokens_seen": 2953052160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010701010101010102, + "loss": 2.4886, + "theoretical_loss": 3.324794265216713, + "tokens_seen": 2953117696 + }, + { + "epoch": 0.79, + "learning_rate": 0.000106989898989899, + "loss": 2.535, + "theoretical_loss": 3.324788574104761, + "tokens_seen": 2953183232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010696969696969696, + "loss": 2.5232, + "theoretical_loss": 3.324782883154464, + "tokens_seen": 2953248768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010694949494949495, + "loss": 2.468, + "theoretical_loss": 3.324777192365815, + "tokens_seen": 2953314304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010692929292929293, + "loss": 2.2954, + "theoretical_loss": 3.3247715017388044, + "tokens_seen": 2953379840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010690909090909091, + "loss": 2.5284, + "theoretical_loss": 3.3247658112734255, + "tokens_seen": 2953445376 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001068888888888889, + "loss": 2.4642, + "theoretical_loss": 3.324760120969669, + "tokens_seen": 2953510912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010686868686868687, + "loss": 2.3741, + "theoretical_loss": 3.324754430827527, + "tokens_seen": 2953576448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010684848484848485, + "loss": 2.4842, + "theoretical_loss": 3.3247487408469913, + "tokens_seen": 2953641984 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010682828282828284, + "loss": 2.4942, + "theoretical_loss": 3.3247430510280545, + "tokens_seen": 2953707520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010680808080808082, + "loss": 2.4284, + "theoretical_loss": 3.324737361370707, + "tokens_seen": 2953773056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010678787878787879, + "loss": 2.3534, + "theoretical_loss": 3.324731671874942, + "tokens_seen": 2953838592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010676767676767676, + "loss": 2.4681, + "theoretical_loss": 3.3247259825407505, + "tokens_seen": 2953904128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010674747474747474, + "loss": 2.4784, + "theoretical_loss": 3.3247202933681246, + "tokens_seen": 2953969664 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010672727272727273, + "loss": 2.6044, + "theoretical_loss": 3.324714604357056, + "tokens_seen": 2954035200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010670707070707071, + "loss": 2.566, + "theoretical_loss": 3.3247089155075367, + "tokens_seen": 2954100736 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001066868686868687, + "loss": 2.3984, + "theoretical_loss": 3.324703226819558, + "tokens_seen": 2954166272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010666666666666668, + "loss": 2.3662, + "theoretical_loss": 3.3246975382931128, + "tokens_seen": 2954231808 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1663271, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.544665813446045, + "objective/train/theoretical_loss": 3.3246918499281923, + "objective/train/tokens_used": 1313156576, + "theoretical_loss": 3.3246918499281923, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010664646464646465, + "loss": 2.6879, + "theoretical_loss": 3.3246918499281923, + "tokens_seen": 2954297344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010662626262626262, + "loss": 2.5581, + "theoretical_loss": 3.324686161724788, + "tokens_seen": 2954362880 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001066060606060606, + "loss": 2.4866, + "theoretical_loss": 3.3246804736828923, + "tokens_seen": 2954428416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010658585858585859, + "loss": 2.4798, + "theoretical_loss": 3.3246747858024968, + "tokens_seen": 2954493952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010656565656565657, + "loss": 2.4982, + "theoretical_loss": 3.324669098083593, + "tokens_seen": 2954559488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010654545454545454, + "loss": 2.4308, + "theoretical_loss": 3.324663410526173, + "tokens_seen": 2954625024 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010652525252525253, + "loss": 2.5286, + "theoretical_loss": 3.3246577231302292, + "tokens_seen": 2954690560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010650505050505051, + "loss": 2.5577, + "theoretical_loss": 3.3246520358957525, + "tokens_seen": 2954756096 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001064848484848485, + "loss": 2.6732, + "theoretical_loss": 3.3246463488227356, + "tokens_seen": 2954821632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010646464646464648, + "loss": 2.4853, + "theoretical_loss": 3.32464066191117, + "tokens_seen": 2954887168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010644444444444444, + "loss": 2.7422, + "theoretical_loss": 3.3246349751610467, + "tokens_seen": 2954952704 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010642424242424242, + "loss": 2.5363, + "theoretical_loss": 3.3246292885723587, + "tokens_seen": 2955018240 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001064040404040404, + "loss": 2.5471, + "theoretical_loss": 3.3246236021450972, + "tokens_seen": 2955083776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010638383838383839, + "loss": 2.5529, + "theoretical_loss": 3.3246179158792546, + "tokens_seen": 2955149312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010636363636363637, + "loss": 2.5472, + "theoretical_loss": 3.3246122297748224, + "tokens_seen": 2955214848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010634343434343434, + "loss": 2.6666, + "theoretical_loss": 3.3246065438317918, + "tokens_seen": 2955280384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010632323232323233, + "loss": 2.572, + "theoretical_loss": 3.3246008580501556, + "tokens_seen": 2955345920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010630303030303031, + "loss": 2.5804, + "theoretical_loss": 3.3245951724299054, + "tokens_seen": 2955411456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010628282828282828, + "loss": 2.3339, + "theoretical_loss": 3.324589486971033, + "tokens_seen": 2955476992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010626262626262626, + "loss": 2.5407, + "theoretical_loss": 3.3245838016735303, + "tokens_seen": 2955542528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010624242424242425, + "loss": 2.2829, + "theoretical_loss": 3.3245781165373884, + "tokens_seen": 2955608064 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010622222222222222, + "loss": 2.5967, + "theoretical_loss": 3.3245724315626006, + "tokens_seen": 2955673600 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001062020202020202, + "loss": 2.5895, + "theoretical_loss": 3.3245667467491575, + "tokens_seen": 2955739136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010618181818181819, + "loss": 2.4869, + "theoretical_loss": 3.324561062097051, + "tokens_seen": 2955804672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010616161616161617, + "loss": 2.725, + "theoretical_loss": 3.3245553776062735, + "tokens_seen": 2955870208 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1664408, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6966660022735596, + "objective/train/theoretical_loss": 3.324549693276817, + "objective/train/tokens_used": 1314794976, + "theoretical_loss": 3.324549693276817, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010614141414141415, + "loss": 2.5109, + "theoretical_loss": 3.324549693276817, + "tokens_seen": 2955935744 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010612121212121212, + "loss": 2.4338, + "theoretical_loss": 3.324544009108673, + "tokens_seen": 2956001280 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001061010101010101, + "loss": 2.3747, + "theoretical_loss": 3.324538325101833, + "tokens_seen": 2956066816 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010608080808080808, + "loss": 2.2173, + "theoretical_loss": 3.3245326412562894, + "tokens_seen": 2956132352 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010606060606060606, + "loss": 2.415, + "theoretical_loss": 3.3245269575720338, + "tokens_seen": 2956197888 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010604040404040405, + "loss": 2.5712, + "theoretical_loss": 3.324521274049058, + "tokens_seen": 2956263424 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010602020202020202, + "loss": 2.5388, + "theoretical_loss": 3.3245155906873536, + "tokens_seen": 2956328960 + }, + { + "epoch": 0.79, + "learning_rate": 0.000106, + "loss": 2.5019, + "theoretical_loss": 3.324509907486913, + "tokens_seen": 2956394496 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010597979797979799, + "loss": 2.6495, + "theoretical_loss": 3.324504224447728, + "tokens_seen": 2956460032 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010595959595959597, + "loss": 2.4616, + "theoretical_loss": 3.32449854156979, + "tokens_seen": 2956525568 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010593939393939394, + "loss": 2.3867, + "theoretical_loss": 3.3244928588530915, + "tokens_seen": 2956591104 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010591919191919191, + "loss": 2.4039, + "theoretical_loss": 3.324487176297624, + "tokens_seen": 2956656640 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001058989898989899, + "loss": 2.7044, + "theoretical_loss": 3.324481493903379, + "tokens_seen": 2956722176 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010587878787878788, + "loss": 2.5084, + "theoretical_loss": 3.324475811670349, + "tokens_seen": 2956787712 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010585858585858586, + "loss": 2.4348, + "theoretical_loss": 3.324470129598525, + "tokens_seen": 2956853248 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010583838383838385, + "loss": 2.6116, + "theoretical_loss": 3.3244644476878995, + "tokens_seen": 2956918784 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010581818181818183, + "loss": 2.5807, + "theoretical_loss": 3.3244587659384646, + "tokens_seen": 2956984320 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001057979797979798, + "loss": 2.6235, + "theoretical_loss": 3.324453084350212, + "tokens_seen": 2957049856 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010577777777777778, + "loss": 2.2812, + "theoretical_loss": 3.324447402923133, + "tokens_seen": 2957115392 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010575757575757575, + "loss": 2.4504, + "theoretical_loss": 3.3244417216572195, + "tokens_seen": 2957180928 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010573737373737374, + "loss": 2.7776, + "theoretical_loss": 3.324436040552464, + "tokens_seen": 2957246464 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010571717171717172, + "loss": 2.655, + "theoretical_loss": 3.324430359608858, + "tokens_seen": 2957312000 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010569696969696969, + "loss": 2.5293, + "theoretical_loss": 3.3244246788263934, + "tokens_seen": 2957377536 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010567676767676768, + "loss": 2.3904, + "theoretical_loss": 3.324418998205062, + "tokens_seen": 2957443072 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010565656565656566, + "loss": 2.6468, + "theoretical_loss": 3.3244133177448556, + "tokens_seen": 2957508608 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1665006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4088244438171387, + "objective/train/theoretical_loss": 3.3244076374457663, + "objective/train/tokens_used": 1316433376, + "theoretical_loss": 3.3244076374457663, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010563636363636365, + "loss": 2.4007, + "theoretical_loss": 3.3244076374457663, + "tokens_seen": 2957574144 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010561616161616163, + "loss": 2.4899, + "theoretical_loss": 3.324401957307786, + "tokens_seen": 2957639680 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010559595959595959, + "loss": 2.3924, + "theoretical_loss": 3.324396277330906, + "tokens_seen": 2957705216 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010557575757575757, + "loss": 2.3861, + "theoretical_loss": 3.3243905975151185, + "tokens_seen": 2957770752 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010555555555555555, + "loss": 2.5485, + "theoretical_loss": 3.3243849178604155, + "tokens_seen": 2957836288 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010553535353535354, + "loss": 2.494, + "theoretical_loss": 3.324379238366789, + "tokens_seen": 2957901824 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010551515151515152, + "loss": 2.5801, + "theoretical_loss": 3.32437355903423, + "tokens_seen": 2957967360 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010549494949494949, + "loss": 2.5107, + "theoretical_loss": 3.3243678798627316, + "tokens_seen": 2958032896 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010547474747474748, + "loss": 2.3212, + "theoretical_loss": 3.324362200852285, + "tokens_seen": 2958098432 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010545454545454546, + "loss": 2.6714, + "theoretical_loss": 3.324356522002882, + "tokens_seen": 2958163968 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010543434343434344, + "loss": 2.7677, + "theoretical_loss": 3.3243508433145146, + "tokens_seen": 2958229504 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010541414141414141, + "loss": 2.7196, + "theoretical_loss": 3.3243451647871747, + "tokens_seen": 2958295040 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010539393939393938, + "loss": 2.6429, + "theoretical_loss": 3.324339486420854, + "tokens_seen": 2958360576 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010537373737373737, + "loss": 2.4375, + "theoretical_loss": 3.3243338082155445, + "tokens_seen": 2958426112 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010535353535353535, + "loss": 2.4254, + "theoretical_loss": 3.324328130171238, + "tokens_seen": 2958491648 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010533333333333334, + "loss": 2.3752, + "theoretical_loss": 3.3243224522879267, + "tokens_seen": 2958557184 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010531313131313132, + "loss": 2.4224, + "theoretical_loss": 3.324316774565602, + "tokens_seen": 2958622720 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001052929292929293, + "loss": 2.6658, + "theoretical_loss": 3.324311097004256, + "tokens_seen": 2958688256 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010527272727272728, + "loss": 2.3442, + "theoretical_loss": 3.3243054196038804, + "tokens_seen": 2958753792 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010525252525252526, + "loss": 2.6339, + "theoretical_loss": 3.3242997423644676, + "tokens_seen": 2958819328 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010523232323232323, + "loss": 2.6362, + "theoretical_loss": 3.3242940652860087, + "tokens_seen": 2958884864 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010521212121212121, + "loss": 2.6418, + "theoretical_loss": 3.3242883883684957, + "tokens_seen": 2958950400 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001051919191919192, + "loss": 2.7789, + "theoretical_loss": 3.3242827116119216, + "tokens_seen": 2959015936 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010517171717171717, + "loss": 2.4222, + "theoretical_loss": 3.3242770350162765, + "tokens_seen": 2959081472 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010515151515151515, + "loss": 2.6339, + "theoretical_loss": 3.3242713585815533, + "tokens_seen": 2959147008 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1665825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.455461263656616, + "objective/train/theoretical_loss": 3.324265682307744, + "objective/train/tokens_used": 1318071776, + "theoretical_loss": 3.324265682307744, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010513131313131314, + "loss": 2.4203, + "theoretical_loss": 3.324265682307744, + "tokens_seen": 2959212544 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010511111111111112, + "loss": 2.3592, + "theoretical_loss": 3.3242600061948404, + "tokens_seen": 2959278080 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001050909090909091, + "loss": 2.2966, + "theoretical_loss": 3.324254330242834, + "tokens_seen": 2959343616 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010507070707070706, + "loss": 2.257, + "theoretical_loss": 3.3242486544517167, + "tokens_seen": 2959409152 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010505050505050504, + "loss": 2.7857, + "theoretical_loss": 3.3242429788214807, + "tokens_seen": 2959474688 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010503030303030303, + "loss": 2.6461, + "theoretical_loss": 3.3242373033521173, + "tokens_seen": 2959540224 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010501010101010101, + "loss": 2.4725, + "theoretical_loss": 3.324231628043619, + "tokens_seen": 2959605760 + }, + { + "epoch": 0.79, + "learning_rate": 0.000104989898989899, + "loss": 2.4705, + "theoretical_loss": 3.324225952895978, + "tokens_seen": 2959671296 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010496969696969697, + "loss": 2.609, + "theoretical_loss": 3.324220277909185, + "tokens_seen": 2959736832 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010494949494949495, + "loss": 2.5684, + "theoretical_loss": 3.324214603083233, + "tokens_seen": 2959802368 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010492929292929293, + "loss": 2.7476, + "theoretical_loss": 3.324208928418113, + "tokens_seen": 2959867904 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010490909090909092, + "loss": 2.4483, + "theoretical_loss": 3.3242032539138178, + "tokens_seen": 2959933440 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010488888888888889, + "loss": 2.5433, + "theoretical_loss": 3.3241975795703382, + "tokens_seen": 2959998976 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010486868686868687, + "loss": 2.425, + "theoretical_loss": 3.324191905387667, + "tokens_seen": 2960064512 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010484848484848484, + "loss": 2.5145, + "theoretical_loss": 3.324186231365796, + "tokens_seen": 2960130048 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010482828282828283, + "loss": 2.7131, + "theoretical_loss": 3.3241805575047163, + "tokens_seen": 2960195584 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010480808080808081, + "loss": 2.5171, + "theoretical_loss": 3.3241748838044205, + "tokens_seen": 2960261120 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001047878787878788, + "loss": 2.5742, + "theoretical_loss": 3.3241692102649, + "tokens_seen": 2960326656 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010476767676767678, + "loss": 2.4929, + "theoretical_loss": 3.3241635368861475, + "tokens_seen": 2960392192 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010474747474747475, + "loss": 2.495, + "theoretical_loss": 3.3241578636681544, + "tokens_seen": 2960457728 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010472727272727272, + "loss": 2.6036, + "theoretical_loss": 3.324152190610912, + "tokens_seen": 2960523264 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001047070707070707, + "loss": 2.6225, + "theoretical_loss": 3.3241465177144134, + "tokens_seen": 2960588800 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010468686868686869, + "loss": 2.4679, + "theoretical_loss": 3.324140844978649, + "tokens_seen": 2960654336 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010466666666666667, + "loss": 2.7246, + "theoretical_loss": 3.3241351724036123, + "tokens_seen": 2960719872 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010464646464646464, + "loss": 2.6883, + "theoretical_loss": 3.3241294999892936, + "tokens_seen": 2960785408 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1666526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3725881576538086, + "objective/train/theoretical_loss": 3.3241238277356864, + "objective/train/tokens_used": 1319710176, + "theoretical_loss": 3.3241238277356864, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010462626262626263, + "loss": 2.6529, + "theoretical_loss": 3.3241238277356864, + "tokens_seen": 2960850944 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010460606060606061, + "loss": 2.5872, + "theoretical_loss": 3.3241181556427812, + "tokens_seen": 2960916480 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045858585858586, + "loss": 2.5919, + "theoretical_loss": 3.3241124837105707, + "tokens_seen": 2960982016 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010456565656565658, + "loss": 2.5423, + "theoretical_loss": 3.3241068119390467, + "tokens_seen": 2961047552 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010454545454545454, + "loss": 2.6082, + "theoretical_loss": 3.324101140328201, + "tokens_seen": 2961113088 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010452525252525252, + "loss": 2.6696, + "theoretical_loss": 3.324095468878025, + "tokens_seen": 2961178624 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001045050505050505, + "loss": 2.5802, + "theoretical_loss": 3.3240897975885115, + "tokens_seen": 2961244160 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010448484848484849, + "loss": 2.8121, + "theoretical_loss": 3.3240841264596517, + "tokens_seen": 2961309696 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010446464646464647, + "loss": 2.614, + "theoretical_loss": 3.3240784554914375, + "tokens_seen": 2961375232 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010444444444444445, + "loss": 2.4858, + "theoretical_loss": 3.324072784683861, + "tokens_seen": 2961440768 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010442424242424243, + "loss": 2.4796, + "theoretical_loss": 3.324067114036915, + "tokens_seen": 2961506304 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010440404040404041, + "loss": 2.5045, + "theoretical_loss": 3.3240614435505895, + "tokens_seen": 2961571840 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010438383838383839, + "loss": 2.6715, + "theoretical_loss": 3.324055773224878, + "tokens_seen": 2961637376 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010436363636363636, + "loss": 2.6612, + "theoretical_loss": 3.3240501030597716, + "tokens_seen": 2961702912 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010434343434343435, + "loss": 2.6021, + "theoretical_loss": 3.324044433055262, + "tokens_seen": 2961768448 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010432323232323232, + "loss": 2.5313, + "theoretical_loss": 3.324038763211342, + "tokens_seen": 2961833984 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001043030303030303, + "loss": 2.6035, + "theoretical_loss": 3.324033093528003, + "tokens_seen": 2961899520 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010428282828282829, + "loss": 2.5775, + "theoretical_loss": 3.3240274240052368, + "tokens_seen": 2961965056 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010426262626262627, + "loss": 2.5577, + "theoretical_loss": 3.3240217546430353, + "tokens_seen": 2962030592 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010424242424242425, + "loss": 2.5759, + "theoretical_loss": 3.3240160854413903, + "tokens_seen": 2962096128 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010422222222222222, + "loss": 2.4217, + "theoretical_loss": 3.324010416400294, + "tokens_seen": 2962161664 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001042020202020202, + "loss": 2.8061, + "theoretical_loss": 3.3240047475197385, + "tokens_seen": 2962227200 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010418181818181818, + "loss": 2.377, + "theoretical_loss": 3.3239990787997153, + "tokens_seen": 2962292736 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010416161616161616, + "loss": 2.682, + "theoretical_loss": 3.323993410240216, + "tokens_seen": 2962358272 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010414141414141415, + "loss": 2.4437, + "theoretical_loss": 3.3239877418412336, + "tokens_seen": 2962423808 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1667691, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.248818874359131, + "objective/train/theoretical_loss": 3.323982073602759, + "objective/train/tokens_used": 1321348576, + "theoretical_loss": 3.323982073602759, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010412121212121212, + "loss": 2.6546, + "theoretical_loss": 3.323982073602759, + "tokens_seen": 2962489344 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001041010101010101, + "loss": 2.7465, + "theoretical_loss": 3.3239764055247845, + "tokens_seen": 2962554880 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010408080808080808, + "loss": 2.6127, + "theoretical_loss": 3.3239707376073016, + "tokens_seen": 2962620416 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010406060606060607, + "loss": 2.6977, + "theoretical_loss": 3.3239650698503027, + "tokens_seen": 2962685952 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010404040404040405, + "loss": 2.5242, + "theoretical_loss": 3.3239594022537795, + "tokens_seen": 2962751488 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010402020202020202, + "loss": 2.6005, + "theoretical_loss": 3.323953734817724, + "tokens_seen": 2962817024 + }, + { + "epoch": 0.79, + "learning_rate": 0.000104, + "loss": 2.5559, + "theoretical_loss": 3.323948067542128, + "tokens_seen": 2962882560 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010397979797979798, + "loss": 2.5465, + "theoretical_loss": 3.3239424004269833, + "tokens_seen": 2962948096 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010395959595959596, + "loss": 2.3501, + "theoretical_loss": 3.3239367334722822, + "tokens_seen": 2963013632 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010393939393939395, + "loss": 2.3576, + "theoretical_loss": 3.323931066678016, + "tokens_seen": 2963079168 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010391919191919193, + "loss": 2.4451, + "theoretical_loss": 3.3239254000441774, + "tokens_seen": 2963144704 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001038989898989899, + "loss": 2.7763, + "theoretical_loss": 3.323919733570758, + "tokens_seen": 2963210240 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010387878787878788, + "loss": 2.4763, + "theoretical_loss": 3.323914067257749, + "tokens_seen": 2963275776 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010385858585858585, + "loss": 2.7222, + "theoretical_loss": 3.323908401105143, + "tokens_seen": 2963341312 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010383838383838384, + "loss": 2.5484, + "theoretical_loss": 3.3239027351129327, + "tokens_seen": 2963406848 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010381818181818182, + "loss": 2.587, + "theoretical_loss": 3.3238970692811085, + "tokens_seen": 2963472384 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010379797979797979, + "loss": 2.5028, + "theoretical_loss": 3.3238914036096627, + "tokens_seen": 2963537920 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010377777777777778, + "loss": 2.7452, + "theoretical_loss": 3.323885738098588, + "tokens_seen": 2963603456 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010375757575757576, + "loss": 2.6078, + "theoretical_loss": 3.3238800727478752, + "tokens_seen": 2963668992 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010373737373737374, + "loss": 2.7258, + "theoretical_loss": 3.323874407557517, + "tokens_seen": 2963734528 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010371717171717173, + "loss": 2.8829, + "theoretical_loss": 3.3238687425275053, + "tokens_seen": 2963800064 + }, + { + "epoch": 0.79, + "learning_rate": 0.0001036969696969697, + "loss": 2.5431, + "theoretical_loss": 3.3238630776578315, + "tokens_seen": 2963865600 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010367676767676767, + "loss": 2.5918, + "theoretical_loss": 3.323857412948488, + "tokens_seen": 2963931136 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010365656565656565, + "loss": 2.6819, + "theoretical_loss": 3.3238517483994667, + "tokens_seen": 2963996672 + }, + { + "epoch": 0.79, + "learning_rate": 0.00010363636363636364, + "loss": 2.6343, + "theoretical_loss": 3.3238460840107593, + "tokens_seen": 2964062208 + }, + { + "epoch": 0.79, + "objective/train/docs_used": 1668204, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.308716297149658, + "objective/train/theoretical_loss": 3.323840419782358, + "objective/train/tokens_used": 1322986976, + "theoretical_loss": 3.323840419782358, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010361616161616162, + "loss": 2.5655, + "theoretical_loss": 3.323840419782358, + "tokens_seen": 2964127744 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001035959595959596, + "loss": 2.6323, + "theoretical_loss": 3.3238347557142545, + "tokens_seen": 2964193280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010357575757575758, + "loss": 2.6764, + "theoretical_loss": 3.32382909180644, + "tokens_seen": 2964258816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010355555555555556, + "loss": 2.389, + "theoretical_loss": 3.323823428058908, + "tokens_seen": 2964324352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010353535353535354, + "loss": 2.5114, + "theoretical_loss": 3.323817764471649, + "tokens_seen": 2964389888 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010351515151515151, + "loss": 2.46, + "theoretical_loss": 3.323812101044656, + "tokens_seen": 2964455424 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001034949494949495, + "loss": 2.6184, + "theoretical_loss": 3.32380643777792, + "tokens_seen": 2964520960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010347474747474747, + "loss": 2.7357, + "theoretical_loss": 3.3238007746714335, + "tokens_seen": 2964586496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010345454545454545, + "loss": 2.6091, + "theoretical_loss": 3.3237951117251883, + "tokens_seen": 2964652032 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010343434343434344, + "loss": 2.4866, + "theoretical_loss": 3.323789448939176, + "tokens_seen": 2964717568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010341414141414142, + "loss": 2.3517, + "theoretical_loss": 3.323783786313389, + "tokens_seen": 2964783104 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001033939393939394, + "loss": 2.6528, + "theoretical_loss": 3.3237781238478195, + "tokens_seen": 2964848640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010337373737373737, + "loss": 2.6909, + "theoretical_loss": 3.3237724615424584, + "tokens_seen": 2964914176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010335353535353536, + "loss": 2.4248, + "theoretical_loss": 3.323766799397298, + "tokens_seen": 2964979712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010333333333333333, + "loss": 2.5217, + "theoretical_loss": 3.323761137412331, + "tokens_seen": 2965045248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010331313131313131, + "loss": 2.6006, + "theoretical_loss": 3.3237554755875487, + "tokens_seen": 2965110784 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001032929292929293, + "loss": 2.3729, + "theoretical_loss": 3.3237498139229427, + "tokens_seen": 2965176320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010327272727272727, + "loss": 2.7243, + "theoretical_loss": 3.3237441524185054, + "tokens_seen": 2965241856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010325252525252525, + "loss": 2.6806, + "theoretical_loss": 3.323738491074229, + "tokens_seen": 2965307392 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010323232323232324, + "loss": 2.5053, + "theoretical_loss": 3.3237328298901048, + "tokens_seen": 2965372928 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010321212121212122, + "loss": 2.5561, + "theoretical_loss": 3.3237271688661245, + "tokens_seen": 2965438464 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001031919191919192, + "loss": 2.5533, + "theoretical_loss": 3.323721508002281, + "tokens_seen": 2965504000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010317171717171719, + "loss": 2.5362, + "theoretical_loss": 3.3237158472985655, + "tokens_seen": 2965569536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010315151515151514, + "loss": 2.4021, + "theoretical_loss": 3.3237101867549708, + "tokens_seen": 2965635072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010313131313131313, + "loss": 2.6501, + "theoretical_loss": 3.3237045263714875, + "tokens_seen": 2965700608 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1669270, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728689432144165, + "objective/train/theoretical_loss": 3.3236988661481086, + "objective/train/tokens_used": 1324625376, + "theoretical_loss": 3.3236988661481086, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010311111111111111, + "loss": 2.7283, + "theoretical_loss": 3.3236988661481086, + "tokens_seen": 2965766144 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001030909090909091, + "loss": 2.5059, + "theoretical_loss": 3.3236932060848257, + "tokens_seen": 2965831680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010307070707070708, + "loss": 2.6849, + "theoretical_loss": 3.3236875461816306, + "tokens_seen": 2965897216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010305050505050505, + "loss": 2.566, + "theoretical_loss": 3.3236818864385156, + "tokens_seen": 2965962752 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010303030303030303, + "loss": 2.5871, + "theoretical_loss": 3.323676226855472, + "tokens_seen": 2966028288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010301010101010102, + "loss": 2.634, + "theoretical_loss": 3.3236705674324925, + "tokens_seen": 2966093824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010298989898989899, + "loss": 2.543, + "theoretical_loss": 3.323664908169568, + "tokens_seen": 2966159360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010296969696969697, + "loss": 2.4668, + "theoretical_loss": 3.323659249066692, + "tokens_seen": 2966224896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010294949494949494, + "loss": 2.3878, + "theoretical_loss": 3.323653590123855, + "tokens_seen": 2966290432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010292929292929293, + "loss": 2.6451, + "theoretical_loss": 3.3236479313410494, + "tokens_seen": 2966355968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010290909090909091, + "loss": 2.5083, + "theoretical_loss": 3.3236422727182675, + "tokens_seen": 2966421504 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001028888888888889, + "loss": 2.5378, + "theoretical_loss": 3.3236366142555007, + "tokens_seen": 2966487040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010286868686868688, + "loss": 2.4576, + "theoretical_loss": 3.323630955952741, + "tokens_seen": 2966552576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010284848484848485, + "loss": 2.5148, + "theoretical_loss": 3.323625297809981, + "tokens_seen": 2966618112 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010282828282828283, + "loss": 2.6796, + "theoretical_loss": 3.323619639827212, + "tokens_seen": 2966683648 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001028080808080808, + "loss": 2.459, + "theoretical_loss": 3.323613982004426, + "tokens_seen": 2966749184 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010278787878787879, + "loss": 2.4514, + "theoretical_loss": 3.323608324341615, + "tokens_seen": 2966814720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010276767676767677, + "loss": 2.6919, + "theoretical_loss": 3.3236026668387715, + "tokens_seen": 2966880256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010274747474747476, + "loss": 2.6098, + "theoretical_loss": 3.323597009495886, + "tokens_seen": 2966945792 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010272727272727273, + "loss": 2.6766, + "theoretical_loss": 3.3235913523129526, + "tokens_seen": 2967011328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010270707070707071, + "loss": 2.6003, + "theoretical_loss": 3.323585695289961, + "tokens_seen": 2967076864 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026868686868687, + "loss": 2.5456, + "theoretical_loss": 3.323580038426905, + "tokens_seen": 2967142400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010266666666666668, + "loss": 2.4922, + "theoretical_loss": 3.323574381723775, + "tokens_seen": 2967207936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010264646464646465, + "loss": 2.5653, + "theoretical_loss": 3.3235687251805643, + "tokens_seen": 2967273472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010262626262626262, + "loss": 2.5423, + "theoretical_loss": 3.323563068797264, + "tokens_seen": 2967339008 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1669763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7289767265319824, + "objective/train/theoretical_loss": 3.323557412573866, + "objective/train/tokens_used": 1326263776, + "theoretical_loss": 3.323557412573866, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001026060606060606, + "loss": 2.884, + "theoretical_loss": 3.323557412573866, + "tokens_seen": 2967404544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010258585858585859, + "loss": 2.5609, + "theoretical_loss": 3.3235517565103625, + "tokens_seen": 2967470080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010256565656565657, + "loss": 2.5846, + "theoretical_loss": 3.3235461006067455, + "tokens_seen": 2967535616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010254545454545455, + "loss": 2.6979, + "theoretical_loss": 3.3235404448630073, + "tokens_seen": 2967601152 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010252525252525252, + "loss": 2.6854, + "theoretical_loss": 3.323534789279139, + "tokens_seen": 2967666688 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010250505050505051, + "loss": 2.4684, + "theoretical_loss": 3.3235291338551334, + "tokens_seen": 2967732224 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010248484848484849, + "loss": 2.6013, + "theoretical_loss": 3.3235234785909817, + "tokens_seen": 2967797760 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010246464646464646, + "loss": 2.6695, + "theoretical_loss": 3.3235178234866765, + "tokens_seen": 2967863296 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010244444444444445, + "loss": 2.724, + "theoretical_loss": 3.3235121685422095, + "tokens_seen": 2967928832 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010242424242424242, + "loss": 2.6108, + "theoretical_loss": 3.3235065137575726, + "tokens_seen": 2967994368 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001024040404040404, + "loss": 2.6388, + "theoretical_loss": 3.323500859132757, + "tokens_seen": 2968059904 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010238383838383839, + "loss": 2.584, + "theoretical_loss": 3.3234952046677564, + "tokens_seen": 2968125440 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010236363636363637, + "loss": 2.2309, + "theoretical_loss": 3.3234895503625617, + "tokens_seen": 2968190976 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010234343434343435, + "loss": 2.4686, + "theoretical_loss": 3.3234838962171644, + "tokens_seen": 2968256512 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010232323232323234, + "loss": 2.7065, + "theoretical_loss": 3.323478242231557, + "tokens_seen": 2968322048 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001023030303030303, + "loss": 2.5273, + "theoretical_loss": 3.323472588405732, + "tokens_seen": 2968387584 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010228282828282828, + "loss": 2.6381, + "theoretical_loss": 3.3234669347396806, + "tokens_seen": 2968453120 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010226262626262626, + "loss": 2.6664, + "theoretical_loss": 3.323461281233395, + "tokens_seen": 2968518656 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010224242424242425, + "loss": 2.4417, + "theoretical_loss": 3.323455627886867, + "tokens_seen": 2968584192 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010222222222222223, + "loss": 2.527, + "theoretical_loss": 3.3234499747000887, + "tokens_seen": 2968649728 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001022020202020202, + "loss": 2.6753, + "theoretical_loss": 3.323444321673052, + "tokens_seen": 2968715264 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010218181818181818, + "loss": 2.3429, + "theoretical_loss": 3.323438668805749, + "tokens_seen": 2968780800 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010216161616161617, + "loss": 2.6575, + "theoretical_loss": 3.3234330160981713, + "tokens_seen": 2968846336 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010214141414141415, + "loss": 2.5547, + "theoretical_loss": 3.3234273635503118, + "tokens_seen": 2968911872 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010212121212121212, + "loss": 2.5479, + "theoretical_loss": 3.323421711162161, + "tokens_seen": 2968977408 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1671048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.66095232963562, + "objective/train/theoretical_loss": 3.323416058933712, + "objective/train/tokens_used": 1327902176, + "theoretical_loss": 3.323416058933712, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010210101010101009, + "loss": 2.4214, + "theoretical_loss": 3.323416058933712, + "tokens_seen": 2969042944 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010208080808080808, + "loss": 2.489, + "theoretical_loss": 3.3234104068649564, + "tokens_seen": 2969108480 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010206060606060606, + "loss": 2.8509, + "theoretical_loss": 3.3234047549558863, + "tokens_seen": 2969174016 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010204040404040404, + "loss": 2.5728, + "theoretical_loss": 3.3233991032064933, + "tokens_seen": 2969239552 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010202020202020203, + "loss": 2.5214, + "theoretical_loss": 3.3233934516167696, + "tokens_seen": 2969305088 + }, + { + "epoch": 0.8, + "learning_rate": 0.000102, + "loss": 2.529, + "theoretical_loss": 3.3233878001867074, + "tokens_seen": 2969370624 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010197979797979798, + "loss": 2.539, + "theoretical_loss": 3.3233821489162985, + "tokens_seen": 2969436160 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010195959595959597, + "loss": 2.4751, + "theoretical_loss": 3.3233764978055342, + "tokens_seen": 2969501696 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010193939393939394, + "loss": 2.6988, + "theoretical_loss": 3.3233708468544076, + "tokens_seen": 2969567232 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010191919191919192, + "loss": 2.5651, + "theoretical_loss": 3.32336519606291, + "tokens_seen": 2969632768 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001018989898989899, + "loss": 2.4948, + "theoretical_loss": 3.3233595454310336, + "tokens_seen": 2969698304 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010187878787878788, + "loss": 2.5509, + "theoretical_loss": 3.32335389495877, + "tokens_seen": 2969763840 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010185858585858586, + "loss": 2.5554, + "theoretical_loss": 3.3233482446461116, + "tokens_seen": 2969829376 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010183838383838384, + "loss": 2.5817, + "theoretical_loss": 3.32334259449305, + "tokens_seen": 2969894912 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010181818181818183, + "loss": 2.6989, + "theoretical_loss": 3.323336944499578, + "tokens_seen": 2969960448 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010179797979797981, + "loss": 2.7115, + "theoretical_loss": 3.323331294665686, + "tokens_seen": 2970025984 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010177777777777777, + "loss": 2.6749, + "theoretical_loss": 3.3233256449913675, + "tokens_seen": 2970091520 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010175757575757575, + "loss": 2.5861, + "theoretical_loss": 3.323319995476614, + "tokens_seen": 2970157056 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010173737373737374, + "loss": 2.7493, + "theoretical_loss": 3.323314346121417, + "tokens_seen": 2970222592 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010171717171717172, + "loss": 2.6306, + "theoretical_loss": 3.3233086969257695, + "tokens_seen": 2970288128 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001016969696969697, + "loss": 2.5712, + "theoretical_loss": 3.323303047889662, + "tokens_seen": 2970353664 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010167676767676767, + "loss": 2.6929, + "theoretical_loss": 3.323297399013088, + "tokens_seen": 2970419200 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010165656565656566, + "loss": 2.5092, + "theoretical_loss": 3.3232917502960384, + "tokens_seen": 2970484736 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010163636363636364, + "loss": 2.4265, + "theoretical_loss": 3.3232861017385056, + "tokens_seen": 2970550272 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010161616161616163, + "loss": 2.6963, + "theoretical_loss": 3.323280453340481, + "tokens_seen": 2970615808 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1671649, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.975634813308716, + "objective/train/theoretical_loss": 3.323274805101958, + "objective/train/tokens_used": 1329540576, + "theoretical_loss": 3.323274805101958, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001015959595959596, + "loss": 2.6819, + "theoretical_loss": 3.323274805101958, + "tokens_seen": 2970681344 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010157575757575757, + "loss": 2.7384, + "theoretical_loss": 3.3232691570229274, + "tokens_seen": 2970746880 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010155555555555555, + "loss": 2.655, + "theoretical_loss": 3.323263509103381, + "tokens_seen": 2970812416 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010153535353535354, + "loss": 2.6619, + "theoretical_loss": 3.3232578613433117, + "tokens_seen": 2970877952 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010151515151515152, + "loss": 2.4088, + "theoretical_loss": 3.3232522137427107, + "tokens_seen": 2970943488 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001014949494949495, + "loss": 2.5469, + "theoretical_loss": 3.3232465663015702, + "tokens_seen": 2971009024 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010147474747474749, + "loss": 2.3756, + "theoretical_loss": 3.3232409190198826, + "tokens_seen": 2971074560 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010145454545454546, + "loss": 2.5971, + "theoretical_loss": 3.3232352718976395, + "tokens_seen": 2971140096 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010143434343434343, + "loss": 2.702, + "theoretical_loss": 3.3232296249348328, + "tokens_seen": 2971205632 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010141414141414141, + "loss": 2.5501, + "theoretical_loss": 3.3232239781314545, + "tokens_seen": 2971271168 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001013939393939394, + "loss": 2.8633, + "theoretical_loss": 3.3232183314874972, + "tokens_seen": 2971336704 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010137373737373738, + "loss": 2.4677, + "theoretical_loss": 3.323212685002952, + "tokens_seen": 2971402240 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010135353535353535, + "loss": 2.6257, + "theoretical_loss": 3.323207038677811, + "tokens_seen": 2971467776 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010133333333333333, + "loss": 2.6443, + "theoretical_loss": 3.323201392512067, + "tokens_seen": 2971533312 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010131313131313132, + "loss": 2.7244, + "theoretical_loss": 3.323195746505711, + "tokens_seen": 2971598848 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001012929292929293, + "loss": 2.6799, + "theoretical_loss": 3.323190100658736, + "tokens_seen": 2971664384 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010127272727272729, + "loss": 2.5521, + "theoretical_loss": 3.3231844549711327, + "tokens_seen": 2971729920 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010125252525252524, + "loss": 2.6893, + "theoretical_loss": 3.323178809442894, + "tokens_seen": 2971795456 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010123232323232323, + "loss": 2.4559, + "theoretical_loss": 3.323173164074012, + "tokens_seen": 2971860992 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010121212121212121, + "loss": 2.6436, + "theoretical_loss": 3.323167518864478, + "tokens_seen": 2971926528 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001011919191919192, + "loss": 2.6829, + "theoretical_loss": 3.3231618738142843, + "tokens_seen": 2971992064 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010117171717171718, + "loss": 2.4746, + "theoretical_loss": 3.3231562289234233, + "tokens_seen": 2972057600 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010115151515151515, + "loss": 2.7578, + "theoretical_loss": 3.323150584191886, + "tokens_seen": 2972123136 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010113131313131313, + "loss": 2.5878, + "theoretical_loss": 3.3231449396196657, + "tokens_seen": 2972188672 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010111111111111112, + "loss": 2.6825, + "theoretical_loss": 3.3231392952067536, + "tokens_seen": 2972254208 + }, + { + "debugging/Self-BLEU-5": 0.29714909839637155, + "debugging/distinct-1-grams": 0.7583905665726303, + "debugging/distinct-2-grams": 0.9373430156197423, + "debugging/entropy-1-grams": 5.38260279658836, + "debugging/entropy-2-grams": 6.052160411565846, + "debugging/length": 452.2857142857143, + "debugging/num_segments": 7, + "epoch": 0.8, + "objective/train/docs_used": 1672806, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7355499267578125, + "objective/train/theoretical_loss": 3.3231336509531415, + "objective/train/tokens_used": 1331178976, + "theoretical_loss": 3.3231336509531415, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010109090909090909, + "loss": 2.5585, + "theoretical_loss": 3.3231336509531415, + "tokens_seen": 2972319744 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010107070707070707, + "loss": 2.5229, + "theoretical_loss": 3.3231280068588216, + "tokens_seen": 2972385280 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010105050505050506, + "loss": 2.6257, + "theoretical_loss": 3.3231223629237863, + "tokens_seen": 2972450816 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010103030303030303, + "loss": 2.4833, + "theoretical_loss": 3.323116719148027, + "tokens_seen": 2972516352 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010101010101010101, + "loss": 2.5936, + "theoretical_loss": 3.3231110755315365, + "tokens_seen": 2972581888 + }, + { + "epoch": 0.8, + "learning_rate": 0.000100989898989899, + "loss": 2.8912, + "theoretical_loss": 3.323105432074306, + "tokens_seen": 2972647424 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010096969696969698, + "loss": 2.4786, + "theoretical_loss": 3.323099788776328, + "tokens_seen": 2972712960 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010094949494949496, + "loss": 2.2685, + "theoretical_loss": 3.3230941456375938, + "tokens_seen": 2972778496 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010092929292929293, + "loss": 2.5294, + "theoretical_loss": 3.323088502658096, + "tokens_seen": 2972844032 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001009090909090909, + "loss": 2.703, + "theoretical_loss": 3.323082859837826, + "tokens_seen": 2972909568 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010088888888888889, + "loss": 2.6632, + "theoretical_loss": 3.323077217176777, + "tokens_seen": 2972975104 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010086868686868687, + "loss": 2.4276, + "theoretical_loss": 3.32307157467494, + "tokens_seen": 2973040640 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010084848484848485, + "loss": 2.6273, + "theoretical_loss": 3.323065932332307, + "tokens_seen": 2973106176 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010082828282828283, + "loss": 2.5027, + "theoretical_loss": 3.3230602901488706, + "tokens_seen": 2973171712 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010080808080808081, + "loss": 2.5579, + "theoretical_loss": 3.3230546481246224, + "tokens_seen": 2973237248 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010078787878787879, + "loss": 2.7858, + "theoretical_loss": 3.3230490062595543, + "tokens_seen": 2973302784 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010076767676767678, + "loss": 2.6521, + "theoretical_loss": 3.3230433645536586, + "tokens_seen": 2973368320 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010074747474747476, + "loss": 2.5494, + "theoretical_loss": 3.3230377230069266, + "tokens_seen": 2973433856 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010072727272727272, + "loss": 2.4217, + "theoretical_loss": 3.3230320816193517, + "tokens_seen": 2973499392 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001007070707070707, + "loss": 2.4761, + "theoretical_loss": 3.323026440390924, + "tokens_seen": 2973564928 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010068686868686869, + "loss": 2.5427, + "theoretical_loss": 3.3230207993216374, + "tokens_seen": 2973630464 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010066666666666667, + "loss": 2.3989, + "theoretical_loss": 3.323015158411483, + "tokens_seen": 2973696000 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010064646464646465, + "loss": 2.7547, + "theoretical_loss": 3.3230095176604526, + "tokens_seen": 2973761536 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010062626262626264, + "loss": 2.7107, + "theoretical_loss": 3.3230038770685386, + "tokens_seen": 2973827072 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010060606060606061, + "loss": 2.7287, + "theoretical_loss": 3.3229982366357325, + "tokens_seen": 2973892608 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1673458, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8385441303253174, + "objective/train/theoretical_loss": 3.322992596362027, + "objective/train/tokens_used": 1332817376, + "theoretical_loss": 3.322992596362027, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010058585858585859, + "loss": 2.7315, + "theoretical_loss": 3.322992596362027, + "tokens_seen": 2973958144 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010056565656565656, + "loss": 2.636, + "theoretical_loss": 3.322986956247414, + "tokens_seen": 2974023680 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010054545454545455, + "loss": 2.4192, + "theoretical_loss": 3.3229813162918846, + "tokens_seen": 2974089216 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010052525252525253, + "loss": 2.5398, + "theoretical_loss": 3.322975676495432, + "tokens_seen": 2974154752 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001005050505050505, + "loss": 2.6226, + "theoretical_loss": 3.322970036858048, + "tokens_seen": 2974220288 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010048484848484848, + "loss": 2.4983, + "theoretical_loss": 3.3229643973797236, + "tokens_seen": 2974285824 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010046464646464647, + "loss": 2.678, + "theoretical_loss": 3.322958758060452, + "tokens_seen": 2974351360 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010044444444444445, + "loss": 2.4063, + "theoretical_loss": 3.3229531189002244, + "tokens_seen": 2974416896 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010042424242424244, + "loss": 2.4918, + "theoretical_loss": 3.3229474798990335, + "tokens_seen": 2974482432 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010040404040404041, + "loss": 2.5625, + "theoretical_loss": 3.3229418410568705, + "tokens_seen": 2974547968 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010038383838383838, + "loss": 2.5451, + "theoretical_loss": 3.3229362023737283, + "tokens_seen": 2974613504 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010036363636363636, + "loss": 2.5267, + "theoretical_loss": 3.3229305638495985, + "tokens_seen": 2974679040 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010034343434343435, + "loss": 2.408, + "theoretical_loss": 3.3229249254844726, + "tokens_seen": 2974744576 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010032323232323233, + "loss": 2.5864, + "theoretical_loss": 3.3229192872783435, + "tokens_seen": 2974810112 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001003030303030303, + "loss": 2.5035, + "theoretical_loss": 3.322913649231203, + "tokens_seen": 2974875648 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010028282828282828, + "loss": 2.5994, + "theoretical_loss": 3.3229080113430425, + "tokens_seen": 2974941184 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010026262626262627, + "loss": 2.4382, + "theoretical_loss": 3.3229023736138545, + "tokens_seen": 2975006720 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010024242424242425, + "loss": 2.5133, + "theoretical_loss": 3.322896736043631, + "tokens_seen": 2975072256 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010022222222222222, + "loss": 2.5281, + "theoretical_loss": 3.322891098632364, + "tokens_seen": 2975137792 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001002020202020202, + "loss": 2.6836, + "theoretical_loss": 3.3228854613800456, + "tokens_seen": 2975203328 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010018181818181818, + "loss": 2.6878, + "theoretical_loss": 3.3228798242866677, + "tokens_seen": 2975268864 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010016161616161616, + "loss": 2.6103, + "theoretical_loss": 3.3228741873522223, + "tokens_seen": 2975334400 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010014141414141414, + "loss": 2.6613, + "theoretical_loss": 3.3228685505767013, + "tokens_seen": 2975399936 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010012121212121213, + "loss": 2.4506, + "theoretical_loss": 3.3228629139600967, + "tokens_seen": 2975465472 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010010101010101011, + "loss": 2.4968, + "theoretical_loss": 3.322857277502401, + "tokens_seen": 2975531008 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1674243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.555476188659668, + "objective/train/theoretical_loss": 3.3228516412036058, + "objective/train/tokens_used": 1334455776, + "theoretical_loss": 3.3228516412036058, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010008080808080808, + "loss": 2.7896, + "theoretical_loss": 3.3228516412036058, + "tokens_seen": 2975596544 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010006060606060607, + "loss": 2.5235, + "theoretical_loss": 3.3228460050637034, + "tokens_seen": 2975662080 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010004040404040404, + "loss": 2.6473, + "theoretical_loss": 3.3228403690826855, + "tokens_seen": 2975727616 + }, + { + "epoch": 0.8, + "learning_rate": 0.00010002020202020202, + "loss": 2.6389, + "theoretical_loss": 3.322834733260544, + "tokens_seen": 2975793152 + }, + { + "epoch": 0.8, + "learning_rate": 0.0001, + "loss": 2.5185, + "theoretical_loss": 3.3228290975972716, + "tokens_seen": 2975858688 + }, + { + "epoch": 0.8, + "learning_rate": 9.997979797979798e-05, + "loss": 2.7796, + "theoretical_loss": 3.32282346209286, + "tokens_seen": 2975924224 + }, + { + "epoch": 0.8, + "learning_rate": 9.995959595959596e-05, + "loss": 2.8198, + "theoretical_loss": 3.322817826747301, + "tokens_seen": 2975989760 + }, + { + "epoch": 0.8, + "learning_rate": 9.993939393939394e-05, + "loss": 2.6972, + "theoretical_loss": 3.3228121915605864, + "tokens_seen": 2976055296 + }, + { + "epoch": 0.8, + "learning_rate": 9.991919191919193e-05, + "loss": 2.6768, + "theoretical_loss": 3.3228065565327087, + "tokens_seen": 2976120832 + }, + { + "epoch": 0.8, + "learning_rate": 9.989898989898991e-05, + "loss": 2.6145, + "theoretical_loss": 3.3228009216636605, + "tokens_seen": 2976186368 + }, + { + "epoch": 0.8, + "learning_rate": 9.987878787878787e-05, + "loss": 2.5898, + "theoretical_loss": 3.3227952869534323, + "tokens_seen": 2976251904 + }, + { + "epoch": 0.8, + "learning_rate": 9.985858585858585e-05, + "loss": 2.4448, + "theoretical_loss": 3.3227896524020175, + "tokens_seen": 2976317440 + }, + { + "epoch": 0.8, + "learning_rate": 9.983838383838384e-05, + "loss": 2.5609, + "theoretical_loss": 3.3227840180094073, + "tokens_seen": 2976382976 + }, + { + "epoch": 0.8, + "learning_rate": 9.981818181818182e-05, + "loss": 2.7242, + "theoretical_loss": 3.322778383775594, + "tokens_seen": 2976448512 + }, + { + "epoch": 0.8, + "learning_rate": 9.97979797979798e-05, + "loss": 2.5754, + "theoretical_loss": 3.3227727497005697, + "tokens_seen": 2976514048 + }, + { + "epoch": 0.8, + "learning_rate": 9.977777777777779e-05, + "loss": 2.6346, + "theoretical_loss": 3.3227671157843264, + "tokens_seen": 2976579584 + }, + { + "epoch": 0.8, + "learning_rate": 9.975757575757576e-05, + "loss": 2.5032, + "theoretical_loss": 3.3227614820268565, + "tokens_seen": 2976645120 + }, + { + "epoch": 0.8, + "learning_rate": 9.973737373737374e-05, + "loss": 2.8591, + "theoretical_loss": 3.3227558484281507, + "tokens_seen": 2976710656 + }, + { + "epoch": 0.8, + "learning_rate": 9.971717171717173e-05, + "loss": 2.6271, + "theoretical_loss": 3.322750214988203, + "tokens_seen": 2976776192 + }, + { + "epoch": 0.8, + "learning_rate": 9.96969696969697e-05, + "loss": 2.5829, + "theoretical_loss": 3.3227445817070036, + "tokens_seen": 2976841728 + }, + { + "epoch": 0.8, + "learning_rate": 9.967676767676768e-05, + "loss": 2.4196, + "theoretical_loss": 3.3227389485845458, + "tokens_seen": 2976907264 + }, + { + "epoch": 0.8, + "learning_rate": 9.965656565656565e-05, + "loss": 2.6858, + "theoretical_loss": 3.3227333156208214, + "tokens_seen": 2976972800 + }, + { + "epoch": 0.8, + "learning_rate": 9.963636363636363e-05, + "loss": 2.6653, + "theoretical_loss": 3.3227276828158216, + "tokens_seen": 2977038336 + }, + { + "epoch": 0.8, + "learning_rate": 9.961616161616162e-05, + "loss": 2.5257, + "theoretical_loss": 3.3227220501695394, + "tokens_seen": 2977103872 + }, + { + "epoch": 0.8, + "learning_rate": 9.95959595959596e-05, + "loss": 2.4482, + "theoretical_loss": 3.322716417681966, + "tokens_seen": 2977169408 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1674993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.76416015625, + "objective/train/theoretical_loss": 3.3227107853530944, + "objective/train/tokens_used": 1336094176, + "theoretical_loss": 3.3227107853530944, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 9.957575757575759e-05, + "loss": 2.5647, + "theoretical_loss": 3.3227107853530944, + "tokens_seen": 2977234944 + }, + { + "epoch": 0.8, + "learning_rate": 9.955555555555556e-05, + "loss": 2.3845, + "theoretical_loss": 3.3227051531829157, + "tokens_seen": 2977300480 + }, + { + "epoch": 0.8, + "learning_rate": 9.953535353535354e-05, + "loss": 2.54, + "theoretical_loss": 3.3226995211714225, + "tokens_seen": 2977366016 + }, + { + "epoch": 0.8, + "learning_rate": 9.951515151515151e-05, + "loss": 2.5724, + "theoretical_loss": 3.322693889318607, + "tokens_seen": 2977431552 + }, + { + "epoch": 0.8, + "learning_rate": 9.94949494949495e-05, + "loss": 2.7315, + "theoretical_loss": 3.3226882576244607, + "tokens_seen": 2977497088 + }, + { + "epoch": 0.8, + "learning_rate": 9.947474747474748e-05, + "loss": 2.4932, + "theoretical_loss": 3.322682626088976, + "tokens_seen": 2977562624 + }, + { + "epoch": 0.8, + "learning_rate": 9.945454545454545e-05, + "loss": 2.5777, + "theoretical_loss": 3.3226769947121446, + "tokens_seen": 2977628160 + }, + { + "epoch": 0.8, + "learning_rate": 9.943434343434343e-05, + "loss": 2.3917, + "theoretical_loss": 3.322671363493959, + "tokens_seen": 2977693696 + }, + { + "epoch": 0.8, + "learning_rate": 9.941414141414142e-05, + "loss": 2.6613, + "theoretical_loss": 3.322665732434411, + "tokens_seen": 2977759232 + }, + { + "epoch": 0.8, + "learning_rate": 9.93939393939394e-05, + "loss": 2.6782, + "theoretical_loss": 3.3226601015334927, + "tokens_seen": 2977824768 + }, + { + "epoch": 0.8, + "learning_rate": 9.937373737373739e-05, + "loss": 2.6141, + "theoretical_loss": 3.322654470791196, + "tokens_seen": 2977890304 + }, + { + "epoch": 0.8, + "learning_rate": 9.935353535353534e-05, + "loss": 2.8254, + "theoretical_loss": 3.3226488402075125, + "tokens_seen": 2977955840 + }, + { + "epoch": 0.8, + "learning_rate": 9.933333333333333e-05, + "loss": 2.7736, + "theoretical_loss": 3.322643209782435, + "tokens_seen": 2978021376 + }, + { + "epoch": 0.8, + "learning_rate": 9.931313131313131e-05, + "loss": 2.7007, + "theoretical_loss": 3.322637579515956, + "tokens_seen": 2978086912 + }, + { + "epoch": 0.8, + "learning_rate": 9.92929292929293e-05, + "loss": 2.7373, + "theoretical_loss": 3.3226319494080663, + "tokens_seen": 2978152448 + }, + { + "epoch": 0.8, + "learning_rate": 9.927272727272728e-05, + "loss": 2.6898, + "theoretical_loss": 3.3226263194587586, + "tokens_seen": 2978217984 + }, + { + "epoch": 0.8, + "learning_rate": 9.925252525252526e-05, + "loss": 2.6583, + "theoretical_loss": 3.3226206896680246, + "tokens_seen": 2978283520 + }, + { + "epoch": 0.8, + "learning_rate": 9.923232323232323e-05, + "loss": 2.7128, + "theoretical_loss": 3.3226150600358566, + "tokens_seen": 2978349056 + }, + { + "epoch": 0.8, + "learning_rate": 9.921212121212122e-05, + "loss": 2.7286, + "theoretical_loss": 3.3226094305622467, + "tokens_seen": 2978414592 + }, + { + "epoch": 0.8, + "learning_rate": 9.91919191919192e-05, + "loss": 2.6478, + "theoretical_loss": 3.322603801247187, + "tokens_seen": 2978480128 + }, + { + "epoch": 0.8, + "learning_rate": 9.917171717171717e-05, + "loss": 2.8304, + "theoretical_loss": 3.3225981720906694, + "tokens_seen": 2978545664 + }, + { + "epoch": 0.8, + "learning_rate": 9.915151515151515e-05, + "loss": 2.6444, + "theoretical_loss": 3.3225925430926857, + "tokens_seen": 2978611200 + }, + { + "epoch": 0.8, + "learning_rate": 9.913131313131313e-05, + "loss": 2.4657, + "theoretical_loss": 3.3225869142532285, + "tokens_seen": 2978676736 + }, + { + "epoch": 0.8, + "learning_rate": 9.911111111111111e-05, + "loss": 2.5986, + "theoretical_loss": 3.3225812855722894, + "tokens_seen": 2978742272 + }, + { + "epoch": 0.8, + "learning_rate": 9.90909090909091e-05, + "loss": 2.346, + "theoretical_loss": 3.3225756570498604, + "tokens_seen": 2978807808 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1676283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.743748188018799, + "objective/train/theoretical_loss": 3.3225700286859343, + "objective/train/tokens_used": 1337732576, + "theoretical_loss": 3.3225700286859343, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.8, + "learning_rate": 9.907070707070708e-05, + "loss": 2.4969, + "theoretical_loss": 3.3225700286859343, + "tokens_seen": 2978873344 + }, + { + "epoch": 0.8, + "learning_rate": 9.905050505050506e-05, + "loss": 2.6063, + "theoretical_loss": 3.322564400480502, + "tokens_seen": 2978938880 + }, + { + "epoch": 0.8, + "learning_rate": 9.903030303030303e-05, + "loss": 2.5371, + "theoretical_loss": 3.3225587724335566, + "tokens_seen": 2979004416 + }, + { + "epoch": 0.8, + "learning_rate": 9.9010101010101e-05, + "loss": 2.9183, + "theoretical_loss": 3.3225531445450893, + "tokens_seen": 2979069952 + }, + { + "epoch": 0.8, + "learning_rate": 9.898989898989899e-05, + "loss": 2.4371, + "theoretical_loss": 3.322547516815093, + "tokens_seen": 2979135488 + }, + { + "epoch": 0.8, + "learning_rate": 9.896969696969697e-05, + "loss": 2.5764, + "theoretical_loss": 3.322541889243559, + "tokens_seen": 2979201024 + }, + { + "epoch": 0.8, + "learning_rate": 9.894949494949495e-05, + "loss": 2.3889, + "theoretical_loss": 3.3225362618304795, + "tokens_seen": 2979266560 + }, + { + "epoch": 0.8, + "learning_rate": 9.892929292929292e-05, + "loss": 2.5092, + "theoretical_loss": 3.322530634575847, + "tokens_seen": 2979332096 + }, + { + "epoch": 0.8, + "learning_rate": 9.890909090909091e-05, + "loss": 2.5499, + "theoretical_loss": 3.322525007479653, + "tokens_seen": 2979397632 + }, + { + "epoch": 0.8, + "learning_rate": 9.888888888888889e-05, + "loss": 2.4119, + "theoretical_loss": 3.32251938054189, + "tokens_seen": 2979463168 + }, + { + "epoch": 0.8, + "learning_rate": 9.886868686868688e-05, + "loss": 2.485, + "theoretical_loss": 3.32251375376255, + "tokens_seen": 2979528704 + }, + { + "epoch": 0.8, + "learning_rate": 9.884848484848486e-05, + "loss": 2.4776, + "theoretical_loss": 3.322508127141625, + "tokens_seen": 2979594240 + }, + { + "epoch": 0.8, + "learning_rate": 9.882828282828283e-05, + "loss": 2.5003, + "theoretical_loss": 3.3225025006791067, + "tokens_seen": 2979659776 + }, + { + "epoch": 0.8, + "learning_rate": 9.88080808080808e-05, + "loss": 2.6061, + "theoretical_loss": 3.3224968743749876, + "tokens_seen": 2979725312 + }, + { + "epoch": 0.8, + "learning_rate": 9.878787878787878e-05, + "loss": 2.4966, + "theoretical_loss": 3.3224912482292597, + "tokens_seen": 2979790848 + }, + { + "epoch": 0.8, + "learning_rate": 9.876767676767677e-05, + "loss": 2.5525, + "theoretical_loss": 3.3224856222419143, + "tokens_seen": 2979856384 + }, + { + "epoch": 0.8, + "learning_rate": 9.874747474747475e-05, + "loss": 2.5634, + "theoretical_loss": 3.3224799964129446, + "tokens_seen": 2979921920 + }, + { + "epoch": 0.8, + "learning_rate": 9.872727272727274e-05, + "loss": 2.464, + "theoretical_loss": 3.3224743707423423, + "tokens_seen": 2979987456 + }, + { + "epoch": 0.8, + "learning_rate": 9.870707070707071e-05, + "loss": 2.6527, + "theoretical_loss": 3.322468745230099, + "tokens_seen": 2980052992 + }, + { + "epoch": 0.8, + "learning_rate": 9.868686868686869e-05, + "loss": 2.6003, + "theoretical_loss": 3.3224631198762076, + "tokens_seen": 2980118528 + }, + { + "epoch": 0.8, + "learning_rate": 9.866666666666668e-05, + "loss": 2.4626, + "theoretical_loss": 3.322457494680659, + "tokens_seen": 2980184064 + }, + { + "epoch": 0.8, + "learning_rate": 9.864646464646465e-05, + "loss": 2.6986, + "theoretical_loss": 3.322451869643446, + "tokens_seen": 2980249600 + }, + { + "epoch": 0.8, + "learning_rate": 9.862626262626263e-05, + "loss": 2.5271, + "theoretical_loss": 3.3224462447645613, + "tokens_seen": 2980315136 + }, + { + "epoch": 0.8, + "learning_rate": 9.86060606060606e-05, + "loss": 2.6189, + "theoretical_loss": 3.3224406200439955, + "tokens_seen": 2980380672 + }, + { + "epoch": 0.8, + "learning_rate": 9.858585858585858e-05, + "loss": 2.4347, + "theoretical_loss": 3.322434995481742, + "tokens_seen": 2980446208 + }, + { + "epoch": 0.8, + "objective/train/docs_used": 1676817, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4090921878814697, + "objective/train/theoretical_loss": 3.3224293710777917, + "objective/train/tokens_used": 1339370976, + "theoretical_loss": 3.3224293710777917, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 9.856565656565657e-05, + "loss": 2.4331, + "theoretical_loss": 3.3224293710777917, + "tokens_seen": 2980511744 + }, + { + "epoch": 0.81, + "learning_rate": 9.854545454545455e-05, + "loss": 2.6434, + "theoretical_loss": 3.322423746832137, + "tokens_seen": 2980577280 + }, + { + "epoch": 0.81, + "learning_rate": 9.852525252525254e-05, + "loss": 2.5943, + "theoretical_loss": 3.322418122744771, + "tokens_seen": 2980642816 + }, + { + "epoch": 0.81, + "learning_rate": 9.85050505050505e-05, + "loss": 2.7117, + "theoretical_loss": 3.3224124988156847, + "tokens_seen": 2980708352 + }, + { + "epoch": 0.81, + "learning_rate": 9.848484848484848e-05, + "loss": 2.5416, + "theoretical_loss": 3.3224068750448703, + "tokens_seen": 2980773888 + }, + { + "epoch": 0.81, + "learning_rate": 9.846464646464646e-05, + "loss": 2.3858, + "theoretical_loss": 3.3224012514323196, + "tokens_seen": 2980839424 + }, + { + "epoch": 0.81, + "learning_rate": 9.844444444444444e-05, + "loss": 2.4854, + "theoretical_loss": 3.3223956279780253, + "tokens_seen": 2980904960 + }, + { + "epoch": 0.81, + "learning_rate": 9.842424242424243e-05, + "loss": 2.4429, + "theoretical_loss": 3.3223900046819796, + "tokens_seen": 2980970496 + }, + { + "epoch": 0.81, + "learning_rate": 9.840404040404041e-05, + "loss": 2.5937, + "theoretical_loss": 3.3223843815441736, + "tokens_seen": 2981036032 + }, + { + "epoch": 0.81, + "learning_rate": 9.838383838383838e-05, + "loss": 2.5855, + "theoretical_loss": 3.3223787585646005, + "tokens_seen": 2981101568 + }, + { + "epoch": 0.81, + "learning_rate": 9.836363636363637e-05, + "loss": 2.4767, + "theoretical_loss": 3.3223731357432515, + "tokens_seen": 2981167104 + }, + { + "epoch": 0.81, + "learning_rate": 9.834343434343435e-05, + "loss": 2.5885, + "theoretical_loss": 3.322367513080119, + "tokens_seen": 2981232640 + }, + { + "epoch": 0.81, + "learning_rate": 9.832323232323233e-05, + "loss": 2.5268, + "theoretical_loss": 3.3223618905751953, + "tokens_seen": 2981298176 + }, + { + "epoch": 0.81, + "learning_rate": 9.83030303030303e-05, + "loss": 2.5126, + "theoretical_loss": 3.322356268228472, + "tokens_seen": 2981363712 + }, + { + "epoch": 0.81, + "learning_rate": 9.828282828282828e-05, + "loss": 2.5098, + "theoretical_loss": 3.3223506460399417, + "tokens_seen": 2981429248 + }, + { + "epoch": 0.81, + "learning_rate": 9.826262626262626e-05, + "loss": 2.584, + "theoretical_loss": 3.3223450240095955, + "tokens_seen": 2981494784 + }, + { + "epoch": 0.81, + "learning_rate": 9.824242424242424e-05, + "loss": 2.7202, + "theoretical_loss": 3.322339402137427, + "tokens_seen": 2981560320 + }, + { + "epoch": 0.81, + "learning_rate": 9.822222222222223e-05, + "loss": 2.6796, + "theoretical_loss": 3.3223337804234268, + "tokens_seen": 2981625856 + }, + { + "epoch": 0.81, + "learning_rate": 9.820202020202021e-05, + "loss": 2.3173, + "theoretical_loss": 3.322328158867588, + "tokens_seen": 2981691392 + }, + { + "epoch": 0.81, + "learning_rate": 9.818181818181818e-05, + "loss": 2.4681, + "theoretical_loss": 3.322322537469902, + "tokens_seen": 2981756928 + }, + { + "epoch": 0.81, + "learning_rate": 9.816161616161617e-05, + "loss": 2.7147, + "theoretical_loss": 3.322316916230361, + "tokens_seen": 2981822464 + }, + { + "epoch": 0.81, + "learning_rate": 9.814141414141414e-05, + "loss": 2.5942, + "theoretical_loss": 3.3223112951489577, + "tokens_seen": 2981888000 + }, + { + "epoch": 0.81, + "learning_rate": 9.812121212121212e-05, + "loss": 2.6278, + "theoretical_loss": 3.3223056742256833, + "tokens_seen": 2981953536 + }, + { + "epoch": 0.81, + "learning_rate": 9.81010101010101e-05, + "loss": 2.6104, + "theoretical_loss": 3.3223000534605305, + "tokens_seen": 2982019072 + }, + { + "epoch": 0.81, + "learning_rate": 9.808080808080807e-05, + "loss": 2.5453, + "theoretical_loss": 3.322294432853491, + "tokens_seen": 2982084608 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1677869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.416811943054199, + "objective/train/theoretical_loss": 3.3222888124045573, + "objective/train/tokens_used": 1341009376, + "theoretical_loss": 3.3222888124045573, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.81, + "learning_rate": 9.806060606060606e-05, + "loss": 2.4866, + "theoretical_loss": 3.3222888124045573, + "tokens_seen": 2982150144 + }, + { + "epoch": 0.81, + "learning_rate": 9.804040404040404e-05, + "loss": 2.5114, + "theoretical_loss": 3.322283192113721, + "tokens_seen": 2982215680 + }, + { + "epoch": 0.81, + "learning_rate": 9.802020202020203e-05, + "loss": 2.6051, + "theoretical_loss": 3.3222775719809743, + "tokens_seen": 2982281216 + }, + { + "epoch": 0.81, + "learning_rate": 9.800000000000001e-05, + "loss": 2.3983, + "theoretical_loss": 3.3222719520063095, + "tokens_seen": 2982346752 + }, + { + "epoch": 0.81, + "learning_rate": 9.7979797979798e-05, + "loss": 2.4966, + "theoretical_loss": 3.3222663321897183, + "tokens_seen": 2982412288 + }, + { + "epoch": 0.81, + "learning_rate": 9.795959595959595e-05, + "loss": 2.6691, + "theoretical_loss": 3.3222607125311936, + "tokens_seen": 2982477824 + }, + { + "epoch": 0.81, + "learning_rate": 9.793939393939394e-05, + "loss": 2.6091, + "theoretical_loss": 3.3222550930307264, + "tokens_seen": 2982543360 + }, + { + "epoch": 0.81, + "learning_rate": 9.791919191919192e-05, + "loss": 2.6279, + "theoretical_loss": 3.3222494736883093, + "tokens_seen": 2982608896 + }, + { + "epoch": 0.81, + "learning_rate": 9.78989898989899e-05, + "loss": 2.6723, + "theoretical_loss": 3.3222438545039346, + "tokens_seen": 2982674432 + }, + { + "epoch": 0.81, + "learning_rate": 9.787878787878789e-05, + "loss": 2.6775, + "theoretical_loss": 3.3222382354775943, + "tokens_seen": 2982739968 + }, + { + "epoch": 0.81, + "learning_rate": 9.785858585858586e-05, + "loss": 2.6183, + "theoretical_loss": 3.3222326166092797, + "tokens_seen": 2982805504 + }, + { + "epoch": 0.81, + "learning_rate": 9.783838383838384e-05, + "loss": 2.5185, + "theoretical_loss": 3.322226997898984, + "tokens_seen": 2982871040 + }, + { + "epoch": 0.81, + "learning_rate": 9.781818181818183e-05, + "loss": 2.5758, + "theoretical_loss": 3.322221379346699, + "tokens_seen": 2982936576 + }, + { + "epoch": 0.81, + "learning_rate": 9.77979797979798e-05, + "loss": 2.7549, + "theoretical_loss": 3.322215760952416, + "tokens_seen": 2983002112 + }, + { + "epoch": 0.81, + "learning_rate": 9.777777777777778e-05, + "loss": 2.5156, + "theoretical_loss": 3.322210142716128, + "tokens_seen": 2983067648 + }, + { + "epoch": 0.81, + "learning_rate": 9.775757575757575e-05, + "loss": 2.5628, + "theoretical_loss": 3.3222045246378267, + "tokens_seen": 2983133184 + }, + { + "epoch": 0.81, + "learning_rate": 9.773737373737373e-05, + "loss": 2.758, + "theoretical_loss": 3.3221989067175044, + "tokens_seen": 2983198720 + }, + { + "epoch": 0.81, + "learning_rate": 9.771717171717172e-05, + "loss": 2.6118, + "theoretical_loss": 3.3221932889551526, + "tokens_seen": 2983264256 + }, + { + "epoch": 0.81, + "learning_rate": 9.76969696969697e-05, + "loss": 2.3756, + "theoretical_loss": 3.3221876713507643, + "tokens_seen": 2983329792 + }, + { + "epoch": 0.81, + "learning_rate": 9.767676767676769e-05, + "loss": 2.4936, + "theoretical_loss": 3.322182053904331, + "tokens_seen": 2983395328 + }, + { + "epoch": 0.81, + "learning_rate": 9.765656565656566e-05, + "loss": 2.7012, + "theoretical_loss": 3.3221764366158446, + "tokens_seen": 2983460864 + }, + { + "epoch": 0.81, + "learning_rate": 9.763636363636364e-05, + "loss": 2.6204, + "theoretical_loss": 3.322170819485298, + "tokens_seen": 2983526400 + }, + { + "epoch": 0.81, + "learning_rate": 9.761616161616161e-05, + "loss": 2.6503, + "theoretical_loss": 3.322165202512682, + "tokens_seen": 2983591936 + }, + { + "epoch": 0.81, + "learning_rate": 9.75959595959596e-05, + "loss": 2.708, + "theoretical_loss": 3.3221595856979906, + "tokens_seen": 2983657472 + }, + { + "epoch": 0.81, + "learning_rate": 9.757575757575758e-05, + "loss": 2.5985, + "theoretical_loss": 3.322153969041214, + "tokens_seen": 2983723008 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1678369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.963561773300171, + "objective/train/theoretical_loss": 3.322148352542345, + "objective/train/tokens_used": 1342647776, + "theoretical_loss": 3.322148352542345, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 9.755555555555556e-05, + "loss": 2.4244, + "theoretical_loss": 3.322148352542345, + "tokens_seen": 2983788544 + }, + { + "epoch": 0.81, + "learning_rate": 9.753535353535353e-05, + "loss": 2.5215, + "theoretical_loss": 3.322142736201376, + "tokens_seen": 2983854080 + }, + { + "epoch": 0.81, + "learning_rate": 9.751515151515152e-05, + "loss": 2.5054, + "theoretical_loss": 3.322137120018299, + "tokens_seen": 2983919616 + }, + { + "epoch": 0.81, + "learning_rate": 9.74949494949495e-05, + "loss": 2.6563, + "theoretical_loss": 3.3221315039931056, + "tokens_seen": 2983985152 + }, + { + "epoch": 0.81, + "learning_rate": 9.747474747474748e-05, + "loss": 2.6983, + "theoretical_loss": 3.3221258881257882, + "tokens_seen": 2984050688 + }, + { + "epoch": 0.81, + "learning_rate": 9.745454545454547e-05, + "loss": 2.6697, + "theoretical_loss": 3.3221202724163392, + "tokens_seen": 2984116224 + }, + { + "epoch": 0.81, + "learning_rate": 9.743434343434343e-05, + "loss": 2.4803, + "theoretical_loss": 3.32211465686475, + "tokens_seen": 2984181760 + }, + { + "epoch": 0.81, + "learning_rate": 9.741414141414141e-05, + "loss": 2.6697, + "theoretical_loss": 3.3221090414710135, + "tokens_seen": 2984247296 + }, + { + "epoch": 0.81, + "learning_rate": 9.73939393939394e-05, + "loss": 2.7278, + "theoretical_loss": 3.3221034262351212, + "tokens_seen": 2984312832 + }, + { + "epoch": 0.81, + "learning_rate": 9.737373737373738e-05, + "loss": 2.5038, + "theoretical_loss": 3.3220978111570654, + "tokens_seen": 2984378368 + }, + { + "epoch": 0.81, + "learning_rate": 9.735353535353536e-05, + "loss": 2.4213, + "theoretical_loss": 3.3220921962368384, + "tokens_seen": 2984443904 + }, + { + "epoch": 0.81, + "learning_rate": 9.733333333333333e-05, + "loss": 2.5318, + "theoretical_loss": 3.3220865814744323, + "tokens_seen": 2984509440 + }, + { + "epoch": 0.81, + "learning_rate": 9.731313131313132e-05, + "loss": 2.7245, + "theoretical_loss": 3.3220809668698386, + "tokens_seen": 2984574976 + }, + { + "epoch": 0.81, + "learning_rate": 9.72929292929293e-05, + "loss": 2.5419, + "theoretical_loss": 3.3220753524230497, + "tokens_seen": 2984640512 + }, + { + "epoch": 0.81, + "learning_rate": 9.727272727272727e-05, + "loss": 2.7319, + "theoretical_loss": 3.322069738134058, + "tokens_seen": 2984706048 + }, + { + "epoch": 0.81, + "learning_rate": 9.725252525252525e-05, + "loss": 2.5806, + "theoretical_loss": 3.3220641240028552, + "tokens_seen": 2984771584 + }, + { + "epoch": 0.81, + "learning_rate": 9.723232323232322e-05, + "loss": 2.5367, + "theoretical_loss": 3.322058510029434, + "tokens_seen": 2984837120 + }, + { + "epoch": 0.81, + "learning_rate": 9.721212121212121e-05, + "loss": 2.7044, + "theoretical_loss": 3.322052896213786, + "tokens_seen": 2984902656 + }, + { + "epoch": 0.81, + "learning_rate": 9.719191919191919e-05, + "loss": 2.5506, + "theoretical_loss": 3.322047282555903, + "tokens_seen": 2984968192 + }, + { + "epoch": 0.81, + "learning_rate": 9.717171717171718e-05, + "loss": 2.5311, + "theoretical_loss": 3.322041669055778, + "tokens_seen": 2985033728 + }, + { + "epoch": 0.81, + "learning_rate": 9.715151515151516e-05, + "loss": 2.5272, + "theoretical_loss": 3.3220360557134025, + "tokens_seen": 2985099264 + }, + { + "epoch": 0.81, + "learning_rate": 9.713131313131314e-05, + "loss": 2.6687, + "theoretical_loss": 3.3220304425287686, + "tokens_seen": 2985164800 + }, + { + "epoch": 0.81, + "learning_rate": 9.711111111111111e-05, + "loss": 2.6479, + "theoretical_loss": 3.3220248295018684, + "tokens_seen": 2985230336 + }, + { + "epoch": 0.81, + "learning_rate": 9.709090909090909e-05, + "loss": 2.7451, + "theoretical_loss": 3.3220192166326945, + "tokens_seen": 2985295872 + }, + { + "epoch": 0.81, + "learning_rate": 9.707070707070707e-05, + "loss": 2.446, + "theoretical_loss": 3.322013603921238, + "tokens_seen": 2985361408 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1679677, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1515989303588867, + "objective/train/theoretical_loss": 3.3220079913674923, + "objective/train/tokens_used": 1344286176, + "theoretical_loss": 3.3220079913674923, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.81, + "learning_rate": 9.705050505050505e-05, + "loss": 2.364, + "theoretical_loss": 3.3220079913674923, + "tokens_seen": 2985426944 + }, + { + "epoch": 0.81, + "learning_rate": 9.703030303030304e-05, + "loss": 2.4876, + "theoretical_loss": 3.3220023789714483, + "tokens_seen": 2985492480 + }, + { + "epoch": 0.81, + "learning_rate": 9.701010101010101e-05, + "loss": 2.5021, + "theoretical_loss": 3.321996766733099, + "tokens_seen": 2985558016 + }, + { + "epoch": 0.81, + "learning_rate": 9.698989898989899e-05, + "loss": 2.4486, + "theoretical_loss": 3.3219911546524363, + "tokens_seen": 2985623552 + }, + { + "epoch": 0.81, + "learning_rate": 9.696969696969698e-05, + "loss": 2.5229, + "theoretical_loss": 3.321985542729452, + "tokens_seen": 2985689088 + }, + { + "epoch": 0.81, + "learning_rate": 9.694949494949496e-05, + "loss": 2.4459, + "theoretical_loss": 3.321979930964138, + "tokens_seen": 2985754624 + }, + { + "epoch": 0.81, + "learning_rate": 9.692929292929293e-05, + "loss": 2.5413, + "theoretical_loss": 3.321974319356487, + "tokens_seen": 2985820160 + }, + { + "epoch": 0.81, + "learning_rate": 9.69090909090909e-05, + "loss": 2.4923, + "theoretical_loss": 3.3219687079064912, + "tokens_seen": 2985885696 + }, + { + "epoch": 0.81, + "learning_rate": 9.688888888888888e-05, + "loss": 2.7826, + "theoretical_loss": 3.3219630966141422, + "tokens_seen": 2985951232 + }, + { + "epoch": 0.81, + "learning_rate": 9.686868686868687e-05, + "loss": 2.6405, + "theoretical_loss": 3.321957485479432, + "tokens_seen": 2986016768 + }, + { + "epoch": 0.81, + "learning_rate": 9.684848484848485e-05, + "loss": 2.8431, + "theoretical_loss": 3.3219518745023535, + "tokens_seen": 2986082304 + }, + { + "epoch": 0.81, + "learning_rate": 9.682828282828284e-05, + "loss": 2.7309, + "theoretical_loss": 3.3219462636828982, + "tokens_seen": 2986147840 + }, + { + "epoch": 0.81, + "learning_rate": 9.68080808080808e-05, + "loss": 2.4618, + "theoretical_loss": 3.3219406530210582, + "tokens_seen": 2986213376 + }, + { + "epoch": 0.81, + "learning_rate": 9.678787878787879e-05, + "loss": 2.5485, + "theoretical_loss": 3.321935042516826, + "tokens_seen": 2986278912 + }, + { + "epoch": 0.81, + "learning_rate": 9.676767676767677e-05, + "loss": 2.6353, + "theoretical_loss": 3.3219294321701933, + "tokens_seen": 2986344448 + }, + { + "epoch": 0.81, + "learning_rate": 9.674747474747474e-05, + "loss": 2.582, + "theoretical_loss": 3.3219238219811524, + "tokens_seen": 2986409984 + }, + { + "epoch": 0.81, + "learning_rate": 9.672727272727273e-05, + "loss": 2.4981, + "theoretical_loss": 3.3219182119496953, + "tokens_seen": 2986475520 + }, + { + "epoch": 0.81, + "learning_rate": 9.670707070707071e-05, + "loss": 2.4883, + "theoretical_loss": 3.3219126020758143, + "tokens_seen": 2986541056 + }, + { + "epoch": 0.81, + "learning_rate": 9.668686868686868e-05, + "loss": 2.3157, + "theoretical_loss": 3.321906992359502, + "tokens_seen": 2986606592 + }, + { + "epoch": 0.81, + "learning_rate": 9.666666666666667e-05, + "loss": 2.4675, + "theoretical_loss": 3.3219013828007493, + "tokens_seen": 2986672128 + }, + { + "epoch": 0.81, + "learning_rate": 9.664646464646465e-05, + "loss": 2.4855, + "theoretical_loss": 3.3218957733995493, + "tokens_seen": 2986737664 + }, + { + "epoch": 0.81, + "learning_rate": 9.662626262626264e-05, + "loss": 2.6707, + "theoretical_loss": 3.3218901641558936, + "tokens_seen": 2986803200 + }, + { + "epoch": 0.81, + "learning_rate": 9.660606060606062e-05, + "loss": 2.439, + "theoretical_loss": 3.3218845550697744, + "tokens_seen": 2986868736 + }, + { + "epoch": 0.81, + "learning_rate": 9.658585858585858e-05, + "loss": 2.7504, + "theoretical_loss": 3.321878946141184, + "tokens_seen": 2986934272 + }, + { + "epoch": 0.81, + "learning_rate": 9.656565656565656e-05, + "loss": 2.4204, + "theoretical_loss": 3.3218733373701146, + "tokens_seen": 2986999808 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1680304, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.889124631881714, + "objective/train/theoretical_loss": 3.3218677287565583, + "objective/train/tokens_used": 1345924576, + "theoretical_loss": 3.3218677287565583, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 9.654545454545454e-05, + "loss": 2.6731, + "theoretical_loss": 3.3218677287565583, + "tokens_seen": 2987065344 + }, + { + "epoch": 0.81, + "learning_rate": 9.652525252525253e-05, + "loss": 2.7173, + "theoretical_loss": 3.3218621203005068, + "tokens_seen": 2987130880 + }, + { + "epoch": 0.81, + "learning_rate": 9.650505050505051e-05, + "loss": 2.3605, + "theoretical_loss": 3.3218565120019523, + "tokens_seen": 2987196416 + }, + { + "epoch": 0.81, + "learning_rate": 9.648484848484848e-05, + "loss": 2.4024, + "theoretical_loss": 3.321850903860888, + "tokens_seen": 2987261952 + }, + { + "epoch": 0.81, + "learning_rate": 9.646464646464647e-05, + "loss": 2.5515, + "theoretical_loss": 3.321845295877304, + "tokens_seen": 2987327488 + }, + { + "epoch": 0.81, + "learning_rate": 9.644444444444445e-05, + "loss": 2.5801, + "theoretical_loss": 3.3218396880511945, + "tokens_seen": 2987393024 + }, + { + "epoch": 0.81, + "learning_rate": 9.642424242424243e-05, + "loss": 2.6829, + "theoretical_loss": 3.3218340803825503, + "tokens_seen": 2987458560 + }, + { + "epoch": 0.81, + "learning_rate": 9.64040404040404e-05, + "loss": 2.6144, + "theoretical_loss": 3.321828472871364, + "tokens_seen": 2987524096 + }, + { + "epoch": 0.81, + "learning_rate": 9.638383838383837e-05, + "loss": 2.574, + "theoretical_loss": 3.3218228655176274, + "tokens_seen": 2987589632 + }, + { + "epoch": 0.81, + "learning_rate": 9.636363636363636e-05, + "loss": 2.6233, + "theoretical_loss": 3.3218172583213335, + "tokens_seen": 2987655168 + }, + { + "epoch": 0.81, + "learning_rate": 9.634343434343434e-05, + "loss": 2.6029, + "theoretical_loss": 3.3218116512824736, + "tokens_seen": 2987720704 + }, + { + "epoch": 0.81, + "learning_rate": 9.632323232323233e-05, + "loss": 2.7178, + "theoretical_loss": 3.3218060444010398, + "tokens_seen": 2987786240 + }, + { + "epoch": 0.81, + "learning_rate": 9.630303030303031e-05, + "loss": 2.5694, + "theoretical_loss": 3.3218004376770245, + "tokens_seen": 2987851776 + }, + { + "epoch": 0.81, + "learning_rate": 9.62828282828283e-05, + "loss": 2.5357, + "theoretical_loss": 3.3217948311104197, + "tokens_seen": 2987917312 + }, + { + "epoch": 0.81, + "learning_rate": 9.626262626262627e-05, + "loss": 2.7276, + "theoretical_loss": 3.3217892247012175, + "tokens_seen": 2987982848 + }, + { + "epoch": 0.81, + "learning_rate": 9.624242424242425e-05, + "loss": 2.5981, + "theoretical_loss": 3.3217836184494107, + "tokens_seen": 2988048384 + }, + { + "epoch": 0.81, + "learning_rate": 9.622222222222222e-05, + "loss": 2.5311, + "theoretical_loss": 3.3217780123549905, + "tokens_seen": 2988113920 + }, + { + "epoch": 0.81, + "learning_rate": 9.62020202020202e-05, + "loss": 2.5134, + "theoretical_loss": 3.3217724064179492, + "tokens_seen": 2988179456 + }, + { + "epoch": 0.81, + "learning_rate": 9.618181818181819e-05, + "loss": 2.5082, + "theoretical_loss": 3.3217668006382794, + "tokens_seen": 2988244992 + }, + { + "epoch": 0.81, + "learning_rate": 9.616161616161616e-05, + "loss": 2.4654, + "theoretical_loss": 3.321761195015973, + "tokens_seen": 2988310528 + }, + { + "epoch": 0.81, + "learning_rate": 9.614141414141414e-05, + "loss": 2.6619, + "theoretical_loss": 3.3217555895510222, + "tokens_seen": 2988376064 + }, + { + "epoch": 0.81, + "learning_rate": 9.612121212121213e-05, + "loss": 2.5184, + "theoretical_loss": 3.321749984243419, + "tokens_seen": 2988441600 + }, + { + "epoch": 0.81, + "learning_rate": 9.610101010101011e-05, + "loss": 2.4863, + "theoretical_loss": 3.3217443790931553, + "tokens_seen": 2988507136 + }, + { + "epoch": 0.81, + "learning_rate": 9.60808080808081e-05, + "loss": 2.5036, + "theoretical_loss": 3.321738774100224, + "tokens_seen": 2988572672 + }, + { + "epoch": 0.81, + "learning_rate": 9.606060606060605e-05, + "loss": 2.4408, + "theoretical_loss": 3.3217331692646157, + "tokens_seen": 2988638208 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1681687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6515755653381348, + "objective/train/theoretical_loss": 3.3217275645863245, + "objective/train/tokens_used": 1347562976, + "theoretical_loss": 3.3217275645863245, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.81, + "learning_rate": 9.604040404040403e-05, + "loss": 2.5147, + "theoretical_loss": 3.3217275645863245, + "tokens_seen": 2988703744 + }, + { + "epoch": 0.81, + "learning_rate": 9.602020202020202e-05, + "loss": 2.6113, + "theoretical_loss": 3.321721960065341, + "tokens_seen": 2988769280 + }, + { + "epoch": 0.81, + "learning_rate": 9.6e-05, + "loss": 2.4276, + "theoretical_loss": 3.3217163557016582, + "tokens_seen": 2988834816 + }, + { + "epoch": 0.81, + "learning_rate": 9.597979797979799e-05, + "loss": 2.7394, + "theoretical_loss": 3.321710751495268, + "tokens_seen": 2988900352 + }, + { + "epoch": 0.81, + "learning_rate": 9.595959595959596e-05, + "loss": 2.5659, + "theoretical_loss": 3.3217051474461625, + "tokens_seen": 2988965888 + }, + { + "epoch": 0.81, + "learning_rate": 9.593939393939394e-05, + "loss": 2.5479, + "theoretical_loss": 3.321699543554334, + "tokens_seen": 2989031424 + }, + { + "epoch": 0.81, + "learning_rate": 9.591919191919192e-05, + "loss": 2.6326, + "theoretical_loss": 3.3216939398197747, + "tokens_seen": 2989096960 + }, + { + "epoch": 0.81, + "learning_rate": 9.589898989898991e-05, + "loss": 2.5379, + "theoretical_loss": 3.3216883362424756, + "tokens_seen": 2989162496 + }, + { + "epoch": 0.81, + "learning_rate": 9.587878787878788e-05, + "loss": 2.5467, + "theoretical_loss": 3.3216827328224303, + "tokens_seen": 2989228032 + }, + { + "epoch": 0.81, + "learning_rate": 9.585858585858586e-05, + "loss": 2.7621, + "theoretical_loss": 3.321677129559631, + "tokens_seen": 2989293568 + }, + { + "epoch": 0.81, + "learning_rate": 9.583838383838383e-05, + "loss": 2.5177, + "theoretical_loss": 3.321671526454068, + "tokens_seen": 2989359104 + }, + { + "epoch": 0.81, + "learning_rate": 9.581818181818182e-05, + "loss": 2.4579, + "theoretical_loss": 3.3216659235057353, + "tokens_seen": 2989424640 + }, + { + "epoch": 0.81, + "learning_rate": 9.57979797979798e-05, + "loss": 2.6351, + "theoretical_loss": 3.3216603207146247, + "tokens_seen": 2989490176 + }, + { + "epoch": 0.81, + "learning_rate": 9.577777777777779e-05, + "loss": 2.603, + "theoretical_loss": 3.321654718080728, + "tokens_seen": 2989555712 + }, + { + "epoch": 0.81, + "learning_rate": 9.575757575757577e-05, + "loss": 2.777, + "theoretical_loss": 3.321649115604037, + "tokens_seen": 2989621248 + }, + { + "epoch": 0.81, + "learning_rate": 9.573737373737374e-05, + "loss": 2.5287, + "theoretical_loss": 3.3216435132845445, + "tokens_seen": 2989686784 + }, + { + "epoch": 0.81, + "learning_rate": 9.571717171717171e-05, + "loss": 2.7293, + "theoretical_loss": 3.321637911122242, + "tokens_seen": 2989752320 + }, + { + "epoch": 0.81, + "learning_rate": 9.56969696969697e-05, + "loss": 2.4257, + "theoretical_loss": 3.3216323091171227, + "tokens_seen": 2989817856 + }, + { + "epoch": 0.81, + "learning_rate": 9.567676767676768e-05, + "loss": 2.611, + "theoretical_loss": 3.3216267072691776, + "tokens_seen": 2989883392 + }, + { + "epoch": 0.81, + "learning_rate": 9.565656565656566e-05, + "loss": 2.6203, + "theoretical_loss": 3.3216211055783997, + "tokens_seen": 2989948928 + }, + { + "epoch": 0.81, + "learning_rate": 9.563636363636363e-05, + "loss": 2.5155, + "theoretical_loss": 3.3216155040447806, + "tokens_seen": 2990014464 + }, + { + "epoch": 0.81, + "learning_rate": 9.561616161616162e-05, + "loss": 2.5814, + "theoretical_loss": 3.321609902668312, + "tokens_seen": 2990080000 + }, + { + "epoch": 0.81, + "learning_rate": 9.55959595959596e-05, + "loss": 2.5818, + "theoretical_loss": 3.3216043014489873, + "tokens_seen": 2990145536 + }, + { + "epoch": 0.81, + "learning_rate": 9.557575757575758e-05, + "loss": 2.4606, + "theoretical_loss": 3.321598700386798, + "tokens_seen": 2990211072 + }, + { + "epoch": 0.81, + "learning_rate": 9.555555555555557e-05, + "loss": 2.4861, + "theoretical_loss": 3.321593099481736, + "tokens_seen": 2990276608 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1682396, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4805214405059814, + "objective/train/theoretical_loss": 3.321587498733794, + "objective/train/tokens_used": 1349201376, + "theoretical_loss": 3.321587498733794, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 9.553535353535353e-05, + "loss": 2.6965, + "theoretical_loss": 3.321587498733794, + "tokens_seen": 2990342144 + }, + { + "epoch": 0.81, + "learning_rate": 9.551515151515151e-05, + "loss": 2.4113, + "theoretical_loss": 3.3215818981429637, + "tokens_seen": 2990407680 + }, + { + "epoch": 0.81, + "learning_rate": 9.549494949494949e-05, + "loss": 2.5326, + "theoretical_loss": 3.3215762977092376, + "tokens_seen": 2990473216 + }, + { + "epoch": 0.81, + "learning_rate": 9.547474747474748e-05, + "loss": 2.4309, + "theoretical_loss": 3.3215706974326076, + "tokens_seen": 2990538752 + }, + { + "epoch": 0.81, + "learning_rate": 9.545454545454546e-05, + "loss": 2.4241, + "theoretical_loss": 3.3215650973130657, + "tokens_seen": 2990604288 + }, + { + "epoch": 0.81, + "learning_rate": 9.543434343434344e-05, + "loss": 2.5543, + "theoretical_loss": 3.3215594973506044, + "tokens_seen": 2990669824 + }, + { + "epoch": 0.81, + "learning_rate": 9.541414141414142e-05, + "loss": 2.5618, + "theoretical_loss": 3.3215538975452157, + "tokens_seen": 2990735360 + }, + { + "epoch": 0.81, + "learning_rate": 9.53939393939394e-05, + "loss": 2.6063, + "theoretical_loss": 3.3215482978968915, + "tokens_seen": 2990800896 + }, + { + "epoch": 0.81, + "learning_rate": 9.537373737373737e-05, + "loss": 2.3272, + "theoretical_loss": 3.3215426984056244, + "tokens_seen": 2990866432 + }, + { + "epoch": 0.81, + "learning_rate": 9.535353535353535e-05, + "loss": 2.7711, + "theoretical_loss": 3.3215370990714064, + "tokens_seen": 2990931968 + }, + { + "epoch": 0.81, + "learning_rate": 9.533333333333334e-05, + "loss": 2.4907, + "theoretical_loss": 3.32153149989423, + "tokens_seen": 2990997504 + }, + { + "epoch": 0.81, + "learning_rate": 9.531313131313131e-05, + "loss": 2.2739, + "theoretical_loss": 3.3215259008740863, + "tokens_seen": 2991063040 + }, + { + "epoch": 0.81, + "learning_rate": 9.529292929292929e-05, + "loss": 2.5085, + "theoretical_loss": 3.3215203020109687, + "tokens_seen": 2991128576 + }, + { + "epoch": 0.81, + "learning_rate": 9.527272727272728e-05, + "loss": 2.6392, + "theoretical_loss": 3.321514703304868, + "tokens_seen": 2991194112 + }, + { + "epoch": 0.81, + "learning_rate": 9.525252525252526e-05, + "loss": 2.7478, + "theoretical_loss": 3.321509104755778, + "tokens_seen": 2991259648 + }, + { + "epoch": 0.81, + "learning_rate": 9.523232323232324e-05, + "loss": 2.473, + "theoretical_loss": 3.3215035063636895, + "tokens_seen": 2991325184 + }, + { + "epoch": 0.81, + "learning_rate": 9.521212121212121e-05, + "loss": 2.4358, + "theoretical_loss": 3.3214979081285954, + "tokens_seen": 2991390720 + }, + { + "epoch": 0.81, + "learning_rate": 9.519191919191918e-05, + "loss": 2.3981, + "theoretical_loss": 3.3214923100504876, + "tokens_seen": 2991456256 + }, + { + "epoch": 0.81, + "learning_rate": 9.517171717171717e-05, + "loss": 2.5752, + "theoretical_loss": 3.3214867121293583, + "tokens_seen": 2991521792 + }, + { + "epoch": 0.81, + "learning_rate": 9.515151515151515e-05, + "loss": 2.7512, + "theoretical_loss": 3.3214811143651994, + "tokens_seen": 2991587328 + }, + { + "epoch": 0.81, + "learning_rate": 9.513131313131314e-05, + "loss": 2.4353, + "theoretical_loss": 3.3214755167580035, + "tokens_seen": 2991652864 + }, + { + "epoch": 0.81, + "learning_rate": 9.511111111111111e-05, + "loss": 2.674, + "theoretical_loss": 3.321469919307763, + "tokens_seen": 2991718400 + }, + { + "epoch": 0.81, + "learning_rate": 9.509090909090909e-05, + "loss": 2.5325, + "theoretical_loss": 3.3214643220144686, + "tokens_seen": 2991783936 + }, + { + "epoch": 0.81, + "learning_rate": 9.507070707070707e-05, + "loss": 2.5045, + "theoretical_loss": 3.321458724878114, + "tokens_seen": 2991849472 + }, + { + "epoch": 0.81, + "learning_rate": 9.505050505050506e-05, + "loss": 2.6524, + "theoretical_loss": 3.3214531278986907, + "tokens_seen": 2991915008 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1683453, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.60022234916687, + "objective/train/theoretical_loss": 3.321447531076191, + "objective/train/tokens_used": 1350839776, + "theoretical_loss": 3.321447531076191, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.81, + "learning_rate": 9.503030303030304e-05, + "loss": 2.6359, + "theoretical_loss": 3.321447531076191, + "tokens_seen": 2991980544 + }, + { + "epoch": 0.81, + "learning_rate": 9.501010101010101e-05, + "loss": 2.7728, + "theoretical_loss": 3.3214419344106076, + "tokens_seen": 2992046080 + }, + { + "epoch": 0.81, + "learning_rate": 9.498989898989898e-05, + "loss": 2.561, + "theoretical_loss": 3.3214363379019316, + "tokens_seen": 2992111616 + }, + { + "epoch": 0.81, + "learning_rate": 9.496969696969697e-05, + "loss": 2.4623, + "theoretical_loss": 3.321430741550156, + "tokens_seen": 2992177152 + }, + { + "epoch": 0.81, + "learning_rate": 9.494949494949495e-05, + "loss": 2.6689, + "theoretical_loss": 3.3214251453552723, + "tokens_seen": 2992242688 + }, + { + "epoch": 0.81, + "learning_rate": 9.492929292929294e-05, + "loss": 2.7136, + "theoretical_loss": 3.321419549317273, + "tokens_seen": 2992308224 + }, + { + "epoch": 0.81, + "learning_rate": 9.490909090909092e-05, + "loss": 2.7696, + "theoretical_loss": 3.321413953436151, + "tokens_seen": 2992373760 + }, + { + "epoch": 0.81, + "learning_rate": 9.488888888888889e-05, + "loss": 2.6202, + "theoretical_loss": 3.3214083577118974, + "tokens_seen": 2992439296 + }, + { + "epoch": 0.81, + "learning_rate": 9.486868686868687e-05, + "loss": 2.5354, + "theoretical_loss": 3.3214027621445044, + "tokens_seen": 2992504832 + }, + { + "epoch": 0.81, + "learning_rate": 9.484848484848484e-05, + "loss": 2.7283, + "theoretical_loss": 3.3213971667339646, + "tokens_seen": 2992570368 + }, + { + "epoch": 0.81, + "learning_rate": 9.482828282828283e-05, + "loss": 2.6369, + "theoretical_loss": 3.32139157148027, + "tokens_seen": 2992635904 + }, + { + "epoch": 0.81, + "learning_rate": 9.480808080808081e-05, + "loss": 2.4035, + "theoretical_loss": 3.321385976383413, + "tokens_seen": 2992701440 + }, + { + "epoch": 0.81, + "learning_rate": 9.478787878787878e-05, + "loss": 2.4445, + "theoretical_loss": 3.3213803814433858, + "tokens_seen": 2992766976 + }, + { + "epoch": 0.81, + "learning_rate": 9.476767676767677e-05, + "loss": 2.5289, + "theoretical_loss": 3.32137478666018, + "tokens_seen": 2992832512 + }, + { + "epoch": 0.81, + "learning_rate": 9.474747474747475e-05, + "loss": 2.4965, + "theoretical_loss": 3.321369192033788, + "tokens_seen": 2992898048 + }, + { + "epoch": 0.81, + "learning_rate": 9.472727272727273e-05, + "loss": 2.5786, + "theoretical_loss": 3.3213635975642024, + "tokens_seen": 2992963584 + }, + { + "epoch": 0.81, + "learning_rate": 9.470707070707072e-05, + "loss": 2.7154, + "theoretical_loss": 3.321358003251415, + "tokens_seen": 2993029120 + }, + { + "epoch": 0.81, + "learning_rate": 9.468686868686869e-05, + "loss": 2.7823, + "theoretical_loss": 3.3213524090954185, + "tokens_seen": 2993094656 + }, + { + "epoch": 0.81, + "learning_rate": 9.466666666666666e-05, + "loss": 2.734, + "theoretical_loss": 3.321346815096204, + "tokens_seen": 2993160192 + }, + { + "epoch": 0.81, + "learning_rate": 9.464646464646464e-05, + "loss": 2.5293, + "theoretical_loss": 3.3213412212537645, + "tokens_seen": 2993225728 + }, + { + "epoch": 0.81, + "learning_rate": 9.462626262626263e-05, + "loss": 2.4929, + "theoretical_loss": 3.321335627568092, + "tokens_seen": 2993291264 + }, + { + "epoch": 0.81, + "learning_rate": 9.460606060606061e-05, + "loss": 2.6844, + "theoretical_loss": 3.321330034039179, + "tokens_seen": 2993356800 + }, + { + "epoch": 0.81, + "learning_rate": 9.45858585858586e-05, + "loss": 2.387, + "theoretical_loss": 3.321324440667017, + "tokens_seen": 2993422336 + }, + { + "epoch": 0.81, + "learning_rate": 9.456565656565657e-05, + "loss": 2.5057, + "theoretical_loss": 3.3213188474515984, + "tokens_seen": 2993487872 + }, + { + "epoch": 0.81, + "learning_rate": 9.454545454545455e-05, + "loss": 2.5902, + "theoretical_loss": 3.3213132543929156, + "tokens_seen": 2993553408 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1684023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7854576110839844, + "objective/train/theoretical_loss": 3.3213076614909607, + "objective/train/tokens_used": 1352478176, + "theoretical_loss": 3.3213076614909607, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 9.452525252525253e-05, + "loss": 2.4975, + "theoretical_loss": 3.3213076614909607, + "tokens_seen": 2993618944 + }, + { + "epoch": 0.81, + "learning_rate": 9.45050505050505e-05, + "loss": 2.4802, + "theoretical_loss": 3.3213020687457258, + "tokens_seen": 2993684480 + }, + { + "epoch": 0.81, + "learning_rate": 9.448484848484849e-05, + "loss": 2.5133, + "theoretical_loss": 3.321296476157203, + "tokens_seen": 2993750016 + }, + { + "epoch": 0.81, + "learning_rate": 9.446464646464646e-05, + "loss": 2.6743, + "theoretical_loss": 3.3212908837253847, + "tokens_seen": 2993815552 + }, + { + "epoch": 0.81, + "learning_rate": 9.444444444444444e-05, + "loss": 2.5028, + "theoretical_loss": 3.3212852914502626, + "tokens_seen": 2993881088 + }, + { + "epoch": 0.81, + "learning_rate": 9.442424242424243e-05, + "loss": 2.7117, + "theoretical_loss": 3.3212796993318294, + "tokens_seen": 2993946624 + }, + { + "epoch": 0.81, + "learning_rate": 9.440404040404041e-05, + "loss": 2.6858, + "theoretical_loss": 3.3212741073700776, + "tokens_seen": 2994012160 + }, + { + "epoch": 0.81, + "learning_rate": 9.43838383838384e-05, + "loss": 2.6803, + "theoretical_loss": 3.3212685155649986, + "tokens_seen": 2994077696 + }, + { + "epoch": 0.81, + "learning_rate": 9.436363636363636e-05, + "loss": 2.7001, + "theoretical_loss": 3.3212629239165845, + "tokens_seen": 2994143232 + }, + { + "epoch": 0.81, + "learning_rate": 9.434343434343435e-05, + "loss": 2.705, + "theoretical_loss": 3.3212573324248282, + "tokens_seen": 2994208768 + }, + { + "epoch": 0.81, + "learning_rate": 9.432323232323232e-05, + "loss": 2.7608, + "theoretical_loss": 3.3212517410897218, + "tokens_seen": 2994274304 + }, + { + "epoch": 0.81, + "learning_rate": 9.43030303030303e-05, + "loss": 2.4835, + "theoretical_loss": 3.321246149911257, + "tokens_seen": 2994339840 + }, + { + "epoch": 0.81, + "learning_rate": 9.428282828282829e-05, + "loss": 2.5722, + "theoretical_loss": 3.3212405588894263, + "tokens_seen": 2994405376 + }, + { + "epoch": 0.81, + "learning_rate": 9.426262626262626e-05, + "loss": 2.6979, + "theoretical_loss": 3.3212349680242217, + "tokens_seen": 2994470912 + }, + { + "epoch": 0.81, + "learning_rate": 9.424242424242424e-05, + "loss": 2.6475, + "theoretical_loss": 3.3212293773156354, + "tokens_seen": 2994536448 + }, + { + "epoch": 0.81, + "learning_rate": 9.422222222222223e-05, + "loss": 2.5108, + "theoretical_loss": 3.32122378676366, + "tokens_seen": 2994601984 + }, + { + "epoch": 0.81, + "learning_rate": 9.420202020202021e-05, + "loss": 2.6098, + "theoretical_loss": 3.321218196368287, + "tokens_seen": 2994667520 + }, + { + "epoch": 0.81, + "learning_rate": 9.418181818181819e-05, + "loss": 2.7265, + "theoretical_loss": 3.321212606129509, + "tokens_seen": 2994733056 + }, + { + "epoch": 0.81, + "learning_rate": 9.416161616161618e-05, + "loss": 2.6776, + "theoretical_loss": 3.3212070160473184, + "tokens_seen": 2994798592 + }, + { + "epoch": 0.81, + "learning_rate": 9.414141414141413e-05, + "loss": 2.6792, + "theoretical_loss": 3.3212014261217067, + "tokens_seen": 2994864128 + }, + { + "epoch": 0.81, + "learning_rate": 9.412121212121212e-05, + "loss": 2.6448, + "theoretical_loss": 3.3211958363526666, + "tokens_seen": 2994929664 + }, + { + "epoch": 0.81, + "learning_rate": 9.41010101010101e-05, + "loss": 2.6994, + "theoretical_loss": 3.3211902467401906, + "tokens_seen": 2994995200 + }, + { + "epoch": 0.81, + "learning_rate": 9.408080808080809e-05, + "loss": 2.5111, + "theoretical_loss": 3.3211846572842703, + "tokens_seen": 2995060736 + }, + { + "epoch": 0.81, + "learning_rate": 9.406060606060607e-05, + "loss": 2.5136, + "theoretical_loss": 3.3211790679848976, + "tokens_seen": 2995126272 + }, + { + "epoch": 0.81, + "learning_rate": 9.404040404040404e-05, + "loss": 2.6595, + "theoretical_loss": 3.3211734788420655, + "tokens_seen": 2995191808 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1684913, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.284437656402588, + "objective/train/theoretical_loss": 3.321167889855766, + "objective/train/tokens_used": 1354116576, + "theoretical_loss": 3.321167889855766, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.81, + "learning_rate": 9.402020202020202e-05, + "loss": 2.4901, + "theoretical_loss": 3.321167889855766, + "tokens_seen": 2995257344 + }, + { + "epoch": 0.81, + "learning_rate": 9.400000000000001e-05, + "loss": 2.6634, + "theoretical_loss": 3.321162301025991, + "tokens_seen": 2995322880 + }, + { + "epoch": 0.81, + "learning_rate": 9.397979797979798e-05, + "loss": 2.6421, + "theoretical_loss": 3.321156712352733, + "tokens_seen": 2995388416 + }, + { + "epoch": 0.81, + "learning_rate": 9.395959595959596e-05, + "loss": 2.5296, + "theoretical_loss": 3.3211511238359837, + "tokens_seen": 2995453952 + }, + { + "epoch": 0.81, + "learning_rate": 9.393939393939393e-05, + "loss": 2.6238, + "theoretical_loss": 3.3211455354757358, + "tokens_seen": 2995519488 + }, + { + "epoch": 0.81, + "learning_rate": 9.391919191919192e-05, + "loss": 2.6214, + "theoretical_loss": 3.3211399472719814, + "tokens_seen": 2995585024 + }, + { + "epoch": 0.81, + "learning_rate": 9.38989898989899e-05, + "loss": 2.6729, + "theoretical_loss": 3.3211343592247125, + "tokens_seen": 2995650560 + }, + { + "epoch": 0.81, + "learning_rate": 9.387878787878788e-05, + "loss": 2.4886, + "theoretical_loss": 3.3211287713339215, + "tokens_seen": 2995716096 + }, + { + "epoch": 0.81, + "learning_rate": 9.385858585858587e-05, + "loss": 2.6084, + "theoretical_loss": 3.3211231835996005, + "tokens_seen": 2995781632 + }, + { + "epoch": 0.81, + "learning_rate": 9.383838383838384e-05, + "loss": 2.7705, + "theoretical_loss": 3.3211175960217414, + "tokens_seen": 2995847168 + }, + { + "epoch": 0.81, + "learning_rate": 9.381818181818182e-05, + "loss": 2.574, + "theoretical_loss": 3.321112008600337, + "tokens_seen": 2995912704 + }, + { + "epoch": 0.81, + "learning_rate": 9.37979797979798e-05, + "loss": 2.5572, + "theoretical_loss": 3.3211064213353794, + "tokens_seen": 2995978240 + }, + { + "epoch": 0.81, + "learning_rate": 9.377777777777778e-05, + "loss": 2.7203, + "theoretical_loss": 3.32110083422686, + "tokens_seen": 2996043776 + }, + { + "epoch": 0.81, + "learning_rate": 9.375757575757576e-05, + "loss": 2.5896, + "theoretical_loss": 3.321095247274772, + "tokens_seen": 2996109312 + }, + { + "epoch": 0.81, + "learning_rate": 9.373737373737375e-05, + "loss": 2.6915, + "theoretical_loss": 3.3210896604791067, + "tokens_seen": 2996174848 + }, + { + "epoch": 0.81, + "learning_rate": 9.371717171717172e-05, + "loss": 2.6815, + "theoretical_loss": 3.3210840738398573, + "tokens_seen": 2996240384 + }, + { + "epoch": 0.81, + "learning_rate": 9.36969696969697e-05, + "loss": 2.8684, + "theoretical_loss": 3.321078487357015, + "tokens_seen": 2996305920 + }, + { + "epoch": 0.81, + "learning_rate": 9.367676767676768e-05, + "loss": 2.9407, + "theoretical_loss": 3.3210729010305733, + "tokens_seen": 2996371456 + }, + { + "epoch": 0.81, + "learning_rate": 9.365656565656567e-05, + "loss": 2.4079, + "theoretical_loss": 3.3210673148605228, + "tokens_seen": 2996436992 + }, + { + "epoch": 0.81, + "learning_rate": 9.363636363636364e-05, + "loss": 2.8086, + "theoretical_loss": 3.321061728846857, + "tokens_seen": 2996502528 + }, + { + "epoch": 0.81, + "learning_rate": 9.361616161616161e-05, + "loss": 2.655, + "theoretical_loss": 3.3210561429895673, + "tokens_seen": 2996568064 + }, + { + "epoch": 0.81, + "learning_rate": 9.359595959595959e-05, + "loss": 2.5992, + "theoretical_loss": 3.321050557288646, + "tokens_seen": 2996633600 + }, + { + "epoch": 0.81, + "learning_rate": 9.357575757575758e-05, + "loss": 2.4848, + "theoretical_loss": 3.3210449717440857, + "tokens_seen": 2996699136 + }, + { + "epoch": 0.81, + "learning_rate": 9.355555555555556e-05, + "loss": 2.6201, + "theoretical_loss": 3.3210393863558787, + "tokens_seen": 2996764672 + }, + { + "epoch": 0.81, + "learning_rate": 9.353535353535354e-05, + "loss": 2.687, + "theoretical_loss": 3.3210338011240164, + "tokens_seen": 2996830208 + }, + { + "epoch": 0.81, + "objective/train/docs_used": 1685661, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812835454940796, + "objective/train/theoretical_loss": 3.3210282160484916, + "objective/train/tokens_used": 1355754976, + "theoretical_loss": 3.3210282160484916, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.81, + "learning_rate": 9.351515151515151e-05, + "loss": 2.7378, + "theoretical_loss": 3.3210282160484916, + "tokens_seen": 2996895744 + }, + { + "epoch": 0.82, + "learning_rate": 9.34949494949495e-05, + "loss": 2.5808, + "theoretical_loss": 3.3210226311292965, + "tokens_seen": 2996961280 + }, + { + "epoch": 0.82, + "learning_rate": 9.347474747474748e-05, + "loss": 2.4924, + "theoretical_loss": 3.3210170463664235, + "tokens_seen": 2997026816 + }, + { + "epoch": 0.82, + "learning_rate": 9.345454545454545e-05, + "loss": 2.7183, + "theoretical_loss": 3.321011461759864, + "tokens_seen": 2997092352 + }, + { + "epoch": 0.82, + "learning_rate": 9.343434343434344e-05, + "loss": 2.6016, + "theoretical_loss": 3.321005877309611, + "tokens_seen": 2997157888 + }, + { + "epoch": 0.82, + "learning_rate": 9.341414141414141e-05, + "loss": 2.5967, + "theoretical_loss": 3.3210002930156564, + "tokens_seen": 2997223424 + }, + { + "epoch": 0.82, + "learning_rate": 9.339393939393939e-05, + "loss": 2.4918, + "theoretical_loss": 3.3209947088779925, + "tokens_seen": 2997288960 + }, + { + "epoch": 0.82, + "learning_rate": 9.337373737373738e-05, + "loss": 2.6318, + "theoretical_loss": 3.3209891248966112, + "tokens_seen": 2997354496 + }, + { + "epoch": 0.82, + "learning_rate": 9.335353535353536e-05, + "loss": 2.5383, + "theoretical_loss": 3.3209835410715054, + "tokens_seen": 2997420032 + }, + { + "epoch": 0.82, + "learning_rate": 9.333333333333334e-05, + "loss": 2.5995, + "theoretical_loss": 3.3209779574026665, + "tokens_seen": 2997485568 + }, + { + "epoch": 0.82, + "learning_rate": 9.331313131313133e-05, + "loss": 2.6923, + "theoretical_loss": 3.320972373890087, + "tokens_seen": 2997551104 + }, + { + "epoch": 0.82, + "learning_rate": 9.329292929292928e-05, + "loss": 2.5789, + "theoretical_loss": 3.3209667905337596, + "tokens_seen": 2997616640 + }, + { + "epoch": 0.82, + "learning_rate": 9.327272727272727e-05, + "loss": 2.7497, + "theoretical_loss": 3.320961207333676, + "tokens_seen": 2997682176 + }, + { + "epoch": 0.82, + "learning_rate": 9.325252525252525e-05, + "loss": 2.648, + "theoretical_loss": 3.3209556242898284, + "tokens_seen": 2997747712 + }, + { + "epoch": 0.82, + "learning_rate": 9.323232323232324e-05, + "loss": 2.6488, + "theoretical_loss": 3.320950041402209, + "tokens_seen": 2997813248 + }, + { + "epoch": 0.82, + "learning_rate": 9.321212121212122e-05, + "loss": 2.5975, + "theoretical_loss": 3.3209444586708106, + "tokens_seen": 2997878784 + }, + { + "epoch": 0.82, + "learning_rate": 9.319191919191919e-05, + "loss": 2.6357, + "theoretical_loss": 3.3209388760956244, + "tokens_seen": 2997944320 + }, + { + "epoch": 0.82, + "learning_rate": 9.317171717171717e-05, + "loss": 2.8216, + "theoretical_loss": 3.3209332936766436, + "tokens_seen": 2998009856 + }, + { + "epoch": 0.82, + "learning_rate": 9.315151515151516e-05, + "loss": 2.62, + "theoretical_loss": 3.3209277114138596, + "tokens_seen": 2998075392 + }, + { + "epoch": 0.82, + "learning_rate": 9.313131313131314e-05, + "loss": 2.4821, + "theoretical_loss": 3.3209221293072653, + "tokens_seen": 2998140928 + }, + { + "epoch": 0.82, + "learning_rate": 9.311111111111111e-05, + "loss": 2.5984, + "theoretical_loss": 3.3209165473568523, + "tokens_seen": 2998206464 + }, + { + "epoch": 0.82, + "learning_rate": 9.309090909090908e-05, + "loss": 2.7693, + "theoretical_loss": 3.3209109655626134, + "tokens_seen": 2998272000 + }, + { + "epoch": 0.82, + "learning_rate": 9.307070707070707e-05, + "loss": 2.41, + "theoretical_loss": 3.3209053839245404, + "tokens_seen": 2998337536 + }, + { + "epoch": 0.82, + "learning_rate": 9.305050505050505e-05, + "loss": 2.832, + "theoretical_loss": 3.320899802442626, + "tokens_seen": 2998403072 + }, + { + "epoch": 0.82, + "learning_rate": 9.303030303030303e-05, + "loss": 2.58, + "theoretical_loss": 3.3208942211168617, + "tokens_seen": 2998468608 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1687315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8215982913970947, + "objective/train/theoretical_loss": 3.32088863994724, + "objective/train/tokens_used": 1357393376, + "theoretical_loss": 3.32088863994724, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.82, + "learning_rate": 9.301010101010102e-05, + "loss": 2.6921, + "theoretical_loss": 3.32088863994724, + "tokens_seen": 2998534144 + }, + { + "epoch": 0.82, + "learning_rate": 9.298989898989899e-05, + "loss": 2.5174, + "theoretical_loss": 3.3208830589337537, + "tokens_seen": 2998599680 + }, + { + "epoch": 0.82, + "learning_rate": 9.296969696969697e-05, + "loss": 2.6124, + "theoretical_loss": 3.3208774780763943, + "tokens_seen": 2998665216 + }, + { + "epoch": 0.82, + "learning_rate": 9.294949494949496e-05, + "loss": 2.5553, + "theoretical_loss": 3.320871897375154, + "tokens_seen": 2998730752 + }, + { + "epoch": 0.82, + "learning_rate": 9.292929292929293e-05, + "loss": 2.6001, + "theoretical_loss": 3.320866316830026, + "tokens_seen": 2998796288 + }, + { + "epoch": 0.82, + "learning_rate": 9.290909090909091e-05, + "loss": 2.6719, + "theoretical_loss": 3.320860736441001, + "tokens_seen": 2998861824 + }, + { + "epoch": 0.82, + "learning_rate": 9.288888888888888e-05, + "loss": 2.6694, + "theoretical_loss": 3.320855156208072, + "tokens_seen": 2998927360 + }, + { + "epoch": 0.82, + "learning_rate": 9.286868686868687e-05, + "loss": 2.5954, + "theoretical_loss": 3.320849576131232, + "tokens_seen": 2998992896 + }, + { + "epoch": 0.82, + "learning_rate": 9.284848484848485e-05, + "loss": 2.7475, + "theoretical_loss": 3.3208439962104723, + "tokens_seen": 2999058432 + }, + { + "epoch": 0.82, + "learning_rate": 9.282828282828283e-05, + "loss": 2.6013, + "theoretical_loss": 3.3208384164457847, + "tokens_seen": 2999123968 + }, + { + "epoch": 0.82, + "learning_rate": 9.280808080808082e-05, + "loss": 2.7161, + "theoretical_loss": 3.3208328368371625, + "tokens_seen": 2999189504 + }, + { + "epoch": 0.82, + "learning_rate": 9.27878787878788e-05, + "loss": 2.5225, + "theoretical_loss": 3.3208272573845976, + "tokens_seen": 2999255040 + }, + { + "epoch": 0.82, + "learning_rate": 9.276767676767676e-05, + "loss": 2.5233, + "theoretical_loss": 3.3208216780880817, + "tokens_seen": 2999320576 + }, + { + "epoch": 0.82, + "learning_rate": 9.274747474747474e-05, + "loss": 2.7495, + "theoretical_loss": 3.3208160989476077, + "tokens_seen": 2999386112 + }, + { + "epoch": 0.82, + "learning_rate": 9.272727272727273e-05, + "loss": 2.9613, + "theoretical_loss": 3.320810519963167, + "tokens_seen": 2999451648 + }, + { + "epoch": 0.82, + "learning_rate": 9.270707070707071e-05, + "loss": 2.6609, + "theoretical_loss": 3.3208049411347527, + "tokens_seen": 2999517184 + }, + { + "epoch": 0.82, + "learning_rate": 9.26868686868687e-05, + "loss": 2.6768, + "theoretical_loss": 3.3207993624623566, + "tokens_seen": 2999582720 + }, + { + "epoch": 0.82, + "learning_rate": 9.266666666666666e-05, + "loss": 2.655, + "theoretical_loss": 3.3207937839459714, + "tokens_seen": 2999648256 + }, + { + "epoch": 0.82, + "learning_rate": 9.264646464646465e-05, + "loss": 2.9401, + "theoretical_loss": 3.3207882055855884, + "tokens_seen": 2999713792 + }, + { + "epoch": 0.82, + "learning_rate": 9.262626262626263e-05, + "loss": 2.5496, + "theoretical_loss": 3.3207826273812007, + "tokens_seen": 2999779328 + }, + { + "epoch": 0.82, + "learning_rate": 9.260606060606062e-05, + "loss": 2.7215, + "theoretical_loss": 3.3207770493327997, + "tokens_seen": 2999844864 + }, + { + "epoch": 0.82, + "learning_rate": 9.258585858585859e-05, + "loss": 2.5991, + "theoretical_loss": 3.320771471440379, + "tokens_seen": 2999910400 + }, + { + "epoch": 0.82, + "learning_rate": 9.256565656565656e-05, + "loss": 2.4961, + "theoretical_loss": 3.3207658937039293, + "tokens_seen": 2999975936 + }, + { + "epoch": 0.82, + "learning_rate": 9.254545454545454e-05, + "loss": 2.8608, + "theoretical_loss": 3.3207603161234434, + "tokens_seen": 3000041472 + }, + { + "epoch": 0.82, + "learning_rate": 9.252525252525253e-05, + "loss": 2.5373, + "theoretical_loss": 3.320754738698914, + "tokens_seen": 3000107008 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1688067, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7751805782318115, + "objective/train/theoretical_loss": 3.3207491614303324, + "objective/train/tokens_used": 1359031776, + "theoretical_loss": 3.3207491614303324, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 9.250505050505051e-05, + "loss": 2.6125, + "theoretical_loss": 3.3207491614303324, + "tokens_seen": 3000172544 + }, + { + "epoch": 0.82, + "learning_rate": 9.24848484848485e-05, + "loss": 2.6206, + "theoretical_loss": 3.3207435843176922, + "tokens_seen": 3000238080 + }, + { + "epoch": 0.82, + "learning_rate": 9.246464646464646e-05, + "loss": 2.4838, + "theoretical_loss": 3.3207380073609842, + "tokens_seen": 3000303616 + }, + { + "epoch": 0.82, + "learning_rate": 9.244444444444445e-05, + "loss": 2.786, + "theoretical_loss": 3.3207324305602013, + "tokens_seen": 3000369152 + }, + { + "epoch": 0.82, + "learning_rate": 9.242424242424242e-05, + "loss": 2.5359, + "theoretical_loss": 3.3207268539153363, + "tokens_seen": 3000434688 + }, + { + "epoch": 0.82, + "learning_rate": 9.24040404040404e-05, + "loss": 2.4309, + "theoretical_loss": 3.32072127742638, + "tokens_seen": 3000500224 + }, + { + "epoch": 0.82, + "learning_rate": 9.238383838383839e-05, + "loss": 2.4865, + "theoretical_loss": 3.320715701093326, + "tokens_seen": 3000565760 + }, + { + "epoch": 0.82, + "learning_rate": 9.236363636363637e-05, + "loss": 2.7399, + "theoretical_loss": 3.3207101249161655, + "tokens_seen": 3000631296 + }, + { + "epoch": 0.82, + "learning_rate": 9.234343434343434e-05, + "loss": 2.5617, + "theoretical_loss": 3.3207045488948914, + "tokens_seen": 3000696832 + }, + { + "epoch": 0.82, + "learning_rate": 9.232323232323232e-05, + "loss": 2.3847, + "theoretical_loss": 3.3206989730294962, + "tokens_seen": 3000762368 + }, + { + "epoch": 0.82, + "learning_rate": 9.230303030303031e-05, + "loss": 2.6087, + "theoretical_loss": 3.3206933973199715, + "tokens_seen": 3000827904 + }, + { + "epoch": 0.82, + "learning_rate": 9.228282828282829e-05, + "loss": 2.4821, + "theoretical_loss": 3.3206878217663096, + "tokens_seen": 3000893440 + }, + { + "epoch": 0.82, + "learning_rate": 9.226262626262628e-05, + "loss": 2.4503, + "theoretical_loss": 3.320682246368503, + "tokens_seen": 3000958976 + }, + { + "epoch": 0.82, + "learning_rate": 9.224242424242423e-05, + "loss": 2.7414, + "theoretical_loss": 3.3206766711265434, + "tokens_seen": 3001024512 + }, + { + "epoch": 0.82, + "learning_rate": 9.222222222222222e-05, + "loss": 2.4251, + "theoretical_loss": 3.320671096040424, + "tokens_seen": 3001090048 + }, + { + "epoch": 0.82, + "learning_rate": 9.22020202020202e-05, + "loss": 2.6889, + "theoretical_loss": 3.3206655211101364, + "tokens_seen": 3001155584 + }, + { + "epoch": 0.82, + "learning_rate": 9.218181818181819e-05, + "loss": 2.6735, + "theoretical_loss": 3.320659946335673, + "tokens_seen": 3001221120 + }, + { + "epoch": 0.82, + "learning_rate": 9.216161616161617e-05, + "loss": 2.4625, + "theoretical_loss": 3.320654371717026, + "tokens_seen": 3001286656 + }, + { + "epoch": 0.82, + "learning_rate": 9.214141414141414e-05, + "loss": 2.5292, + "theoretical_loss": 3.3206487972541874, + "tokens_seen": 3001352192 + }, + { + "epoch": 0.82, + "learning_rate": 9.212121212121212e-05, + "loss": 2.5221, + "theoretical_loss": 3.3206432229471496, + "tokens_seen": 3001417728 + }, + { + "epoch": 0.82, + "learning_rate": 9.210101010101011e-05, + "loss": 2.7744, + "theoretical_loss": 3.320637648795905, + "tokens_seen": 3001483264 + }, + { + "epoch": 0.82, + "learning_rate": 9.208080808080808e-05, + "loss": 2.6952, + "theoretical_loss": 3.320632074800446, + "tokens_seen": 3001548800 + }, + { + "epoch": 0.82, + "learning_rate": 9.206060606060606e-05, + "loss": 2.5394, + "theoretical_loss": 3.3206265009607643, + "tokens_seen": 3001614336 + }, + { + "epoch": 0.82, + "learning_rate": 9.204040404040403e-05, + "loss": 2.6339, + "theoretical_loss": 3.320620927276853, + "tokens_seen": 3001679872 + }, + { + "epoch": 0.82, + "learning_rate": 9.202020202020202e-05, + "loss": 2.6019, + "theoretical_loss": 3.3206153537487033, + "tokens_seen": 3001745408 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1689141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3990423679351807, + "objective/train/theoretical_loss": 3.320609780376308, + "objective/train/tokens_used": 1360670176, + "theoretical_loss": 3.320609780376308, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.82, + "learning_rate": 9.2e-05, + "loss": 2.4322, + "theoretical_loss": 3.320609780376308, + "tokens_seen": 3001810944 + }, + { + "epoch": 0.82, + "learning_rate": 9.197979797979798e-05, + "loss": 2.6085, + "theoretical_loss": 3.3206042071596595, + "tokens_seen": 3001876480 + }, + { + "epoch": 0.82, + "learning_rate": 9.195959595959597e-05, + "loss": 2.51, + "theoretical_loss": 3.3205986340987494, + "tokens_seen": 3001942016 + }, + { + "epoch": 0.82, + "learning_rate": 9.193939393939395e-05, + "loss": 2.2831, + "theoretical_loss": 3.320593061193571, + "tokens_seen": 3002007552 + }, + { + "epoch": 0.82, + "learning_rate": 9.191919191919192e-05, + "loss": 2.563, + "theoretical_loss": 3.3205874884441156, + "tokens_seen": 3002073088 + }, + { + "epoch": 0.82, + "learning_rate": 9.189898989898989e-05, + "loss": 2.4786, + "theoretical_loss": 3.3205819158503758, + "tokens_seen": 3002138624 + }, + { + "epoch": 0.82, + "learning_rate": 9.187878787878788e-05, + "loss": 2.5175, + "theoretical_loss": 3.320576343412344, + "tokens_seen": 3002204160 + }, + { + "epoch": 0.82, + "learning_rate": 9.185858585858586e-05, + "loss": 2.6057, + "theoretical_loss": 3.3205707711300123, + "tokens_seen": 3002269696 + }, + { + "epoch": 0.82, + "learning_rate": 9.183838383838384e-05, + "loss": 2.6782, + "theoretical_loss": 3.3205651990033727, + "tokens_seen": 3002335232 + }, + { + "epoch": 0.82, + "learning_rate": 9.181818181818182e-05, + "loss": 2.5273, + "theoretical_loss": 3.3205596270324174, + "tokens_seen": 3002400768 + }, + { + "epoch": 0.82, + "learning_rate": 9.17979797979798e-05, + "loss": 2.6646, + "theoretical_loss": 3.3205540552171398, + "tokens_seen": 3002466304 + }, + { + "epoch": 0.82, + "learning_rate": 9.177777777777778e-05, + "loss": 2.8542, + "theoretical_loss": 3.3205484835575305, + "tokens_seen": 3002531840 + }, + { + "epoch": 0.82, + "learning_rate": 9.175757575757577e-05, + "loss": 2.6612, + "theoretical_loss": 3.320542912053583, + "tokens_seen": 3002597376 + }, + { + "epoch": 0.82, + "learning_rate": 9.173737373737375e-05, + "loss": 2.3449, + "theoretical_loss": 3.3205373407052887, + "tokens_seen": 3002662912 + }, + { + "epoch": 0.82, + "learning_rate": 9.171717171717171e-05, + "loss": 2.5562, + "theoretical_loss": 3.3205317695126406, + "tokens_seen": 3002728448 + }, + { + "epoch": 0.82, + "learning_rate": 9.169696969696969e-05, + "loss": 2.4667, + "theoretical_loss": 3.3205261984756307, + "tokens_seen": 3002793984 + }, + { + "epoch": 0.82, + "learning_rate": 9.167676767676768e-05, + "loss": 2.5774, + "theoretical_loss": 3.320520627594251, + "tokens_seen": 3002859520 + }, + { + "epoch": 0.82, + "learning_rate": 9.165656565656566e-05, + "loss": 2.6063, + "theoretical_loss": 3.3205150568684942, + "tokens_seen": 3002925056 + }, + { + "epoch": 0.82, + "learning_rate": 9.163636363636364e-05, + "loss": 2.4844, + "theoretical_loss": 3.3205094862983517, + "tokens_seen": 3002990592 + }, + { + "epoch": 0.82, + "learning_rate": 9.161616161616161e-05, + "loss": 2.5878, + "theoretical_loss": 3.3205039158838168, + "tokens_seen": 3003056128 + }, + { + "epoch": 0.82, + "learning_rate": 9.15959595959596e-05, + "loss": 2.7641, + "theoretical_loss": 3.320498345624881, + "tokens_seen": 3003121664 + }, + { + "epoch": 0.82, + "learning_rate": 9.157575757575758e-05, + "loss": 2.7896, + "theoretical_loss": 3.320492775521537, + "tokens_seen": 3003187200 + }, + { + "epoch": 0.82, + "learning_rate": 9.155555555555555e-05, + "loss": 2.5685, + "theoretical_loss": 3.320487205573777, + "tokens_seen": 3003252736 + }, + { + "epoch": 0.82, + "learning_rate": 9.153535353535354e-05, + "loss": 2.5357, + "theoretical_loss": 3.3204816357815927, + "tokens_seen": 3003318272 + }, + { + "epoch": 0.82, + "learning_rate": 9.151515151515152e-05, + "loss": 2.396, + "theoretical_loss": 3.3204760661449777, + "tokens_seen": 3003383808 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1689746, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.332847833633423, + "objective/train/theoretical_loss": 3.3204704966639227, + "objective/train/tokens_used": 1362308576, + "theoretical_loss": 3.3204704966639227, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 9.149494949494949e-05, + "loss": 2.4467, + "theoretical_loss": 3.3204704966639227, + "tokens_seen": 3003449344 + }, + { + "epoch": 0.82, + "learning_rate": 9.147474747474747e-05, + "loss": 2.7776, + "theoretical_loss": 3.3204649273384206, + "tokens_seen": 3003514880 + }, + { + "epoch": 0.82, + "learning_rate": 9.145454545454546e-05, + "loss": 2.3714, + "theoretical_loss": 3.3204593581684643, + "tokens_seen": 3003580416 + }, + { + "epoch": 0.82, + "learning_rate": 9.143434343434344e-05, + "loss": 2.5925, + "theoretical_loss": 3.320453789154045, + "tokens_seen": 3003645952 + }, + { + "epoch": 0.82, + "learning_rate": 9.141414141414143e-05, + "loss": 2.6137, + "theoretical_loss": 3.3204482202951557, + "tokens_seen": 3003711488 + }, + { + "epoch": 0.82, + "learning_rate": 9.13939393939394e-05, + "loss": 2.5091, + "theoretical_loss": 3.320442651591788, + "tokens_seen": 3003777024 + }, + { + "epoch": 0.82, + "learning_rate": 9.137373737373737e-05, + "loss": 2.493, + "theoretical_loss": 3.320437083043935, + "tokens_seen": 3003842560 + }, + { + "epoch": 0.82, + "learning_rate": 9.135353535353535e-05, + "loss": 2.451, + "theoretical_loss": 3.320431514651588, + "tokens_seen": 3003908096 + }, + { + "epoch": 0.82, + "learning_rate": 9.133333333333334e-05, + "loss": 2.5836, + "theoretical_loss": 3.3204259464147405, + "tokens_seen": 3003973632 + }, + { + "epoch": 0.82, + "learning_rate": 9.131313131313132e-05, + "loss": 2.6143, + "theoretical_loss": 3.3204203783333837, + "tokens_seen": 3004039168 + }, + { + "epoch": 0.82, + "learning_rate": 9.129292929292929e-05, + "loss": 2.6531, + "theoretical_loss": 3.3204148104075104, + "tokens_seen": 3004104704 + }, + { + "epoch": 0.82, + "learning_rate": 9.127272727272727e-05, + "loss": 2.5735, + "theoretical_loss": 3.3204092426371123, + "tokens_seen": 3004170240 + }, + { + "epoch": 0.82, + "learning_rate": 9.125252525252526e-05, + "loss": 2.4653, + "theoretical_loss": 3.3204036750221824, + "tokens_seen": 3004235776 + }, + { + "epoch": 0.82, + "learning_rate": 9.123232323232324e-05, + "loss": 2.8518, + "theoretical_loss": 3.3203981075627125, + "tokens_seen": 3004301312 + }, + { + "epoch": 0.82, + "learning_rate": 9.121212121212121e-05, + "loss": 2.7606, + "theoretical_loss": 3.320392540258695, + "tokens_seen": 3004366848 + }, + { + "epoch": 0.82, + "learning_rate": 9.119191919191918e-05, + "loss": 2.5803, + "theoretical_loss": 3.3203869731101223, + "tokens_seen": 3004432384 + }, + { + "epoch": 0.82, + "learning_rate": 9.117171717171717e-05, + "loss": 2.5416, + "theoretical_loss": 3.3203814061169865, + "tokens_seen": 3004497920 + }, + { + "epoch": 0.82, + "learning_rate": 9.115151515151515e-05, + "loss": 2.7201, + "theoretical_loss": 3.3203758392792797, + "tokens_seen": 3004563456 + }, + { + "epoch": 0.82, + "learning_rate": 9.113131313131313e-05, + "loss": 2.5302, + "theoretical_loss": 3.3203702725969944, + "tokens_seen": 3004628992 + }, + { + "epoch": 0.82, + "learning_rate": 9.111111111111112e-05, + "loss": 2.4148, + "theoretical_loss": 3.320364706070123, + "tokens_seen": 3004694528 + }, + { + "epoch": 0.82, + "learning_rate": 9.10909090909091e-05, + "loss": 2.4671, + "theoretical_loss": 3.3203591396986574, + "tokens_seen": 3004760064 + }, + { + "epoch": 0.82, + "learning_rate": 9.107070707070707e-05, + "loss": 2.7085, + "theoretical_loss": 3.32035357348259, + "tokens_seen": 3004825600 + }, + { + "epoch": 0.82, + "learning_rate": 9.105050505050506e-05, + "loss": 2.4697, + "theoretical_loss": 3.320348007421914, + "tokens_seen": 3004891136 + }, + { + "epoch": 0.82, + "learning_rate": 9.103030303030303e-05, + "loss": 2.666, + "theoretical_loss": 3.32034244151662, + "tokens_seen": 3004956672 + }, + { + "epoch": 0.82, + "learning_rate": 9.101010101010101e-05, + "loss": 2.683, + "theoretical_loss": 3.320336875766701, + "tokens_seen": 3005022208 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1690381, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.210015296936035, + "objective/train/theoretical_loss": 3.3203313101721497, + "objective/train/tokens_used": 1363946976, + "theoretical_loss": 3.3203313101721497, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.82, + "learning_rate": 9.0989898989899e-05, + "loss": 2.7001, + "theoretical_loss": 3.3203313101721497, + "tokens_seen": 3005087744 + }, + { + "epoch": 0.82, + "learning_rate": 9.096969696969697e-05, + "loss": 2.6068, + "theoretical_loss": 3.3203257447329584, + "tokens_seen": 3005153280 + }, + { + "epoch": 0.82, + "learning_rate": 9.094949494949495e-05, + "loss": 2.5434, + "theoretical_loss": 3.3203201794491184, + "tokens_seen": 3005218816 + }, + { + "epoch": 0.82, + "learning_rate": 9.092929292929293e-05, + "loss": 2.6072, + "theoretical_loss": 3.320314614320623, + "tokens_seen": 3005284352 + }, + { + "epoch": 0.82, + "learning_rate": 9.090909090909092e-05, + "loss": 2.7076, + "theoretical_loss": 3.320309049347464, + "tokens_seen": 3005349888 + }, + { + "epoch": 0.82, + "learning_rate": 9.08888888888889e-05, + "loss": 2.6099, + "theoretical_loss": 3.3203034845296333, + "tokens_seen": 3005415424 + }, + { + "epoch": 0.82, + "learning_rate": 9.086868686868686e-05, + "loss": 2.8039, + "theoretical_loss": 3.3202979198671243, + "tokens_seen": 3005480960 + }, + { + "epoch": 0.82, + "learning_rate": 9.084848484848484e-05, + "loss": 2.6377, + "theoretical_loss": 3.3202923553599284, + "tokens_seen": 3005546496 + }, + { + "epoch": 0.82, + "learning_rate": 9.082828282828283e-05, + "loss": 2.649, + "theoretical_loss": 3.320286791008038, + "tokens_seen": 3005612032 + }, + { + "epoch": 0.82, + "learning_rate": 9.080808080808081e-05, + "loss": 2.6471, + "theoretical_loss": 3.3202812268114457, + "tokens_seen": 3005677568 + }, + { + "epoch": 0.82, + "learning_rate": 9.07878787878788e-05, + "loss": 2.6494, + "theoretical_loss": 3.3202756627701433, + "tokens_seen": 3005743104 + }, + { + "epoch": 0.82, + "learning_rate": 9.076767676767676e-05, + "loss": 2.7447, + "theoretical_loss": 3.3202700988841234, + "tokens_seen": 3005808640 + }, + { + "epoch": 0.82, + "learning_rate": 9.074747474747475e-05, + "loss": 2.6883, + "theoretical_loss": 3.3202645351533784, + "tokens_seen": 3005874176 + }, + { + "epoch": 0.82, + "learning_rate": 9.072727272727273e-05, + "loss": 2.3969, + "theoretical_loss": 3.3202589715779, + "tokens_seen": 3005939712 + }, + { + "epoch": 0.82, + "learning_rate": 9.070707070707072e-05, + "loss": 2.5894, + "theoretical_loss": 3.320253408157681, + "tokens_seen": 3006005248 + }, + { + "epoch": 0.82, + "learning_rate": 9.068686868686869e-05, + "loss": 2.617, + "theoretical_loss": 3.3202478448927137, + "tokens_seen": 3006070784 + }, + { + "epoch": 0.82, + "learning_rate": 9.066666666666667e-05, + "loss": 2.5488, + "theoretical_loss": 3.32024228178299, + "tokens_seen": 3006136320 + }, + { + "epoch": 0.82, + "learning_rate": 9.064646464646464e-05, + "loss": 2.7012, + "theoretical_loss": 3.320236718828503, + "tokens_seen": 3006201856 + }, + { + "epoch": 0.82, + "learning_rate": 9.062626262626262e-05, + "loss": 2.6258, + "theoretical_loss": 3.320231156029244, + "tokens_seen": 3006267392 + }, + { + "epoch": 0.82, + "learning_rate": 9.060606060606061e-05, + "loss": 2.506, + "theoretical_loss": 3.3202255933852056, + "tokens_seen": 3006332928 + }, + { + "epoch": 0.82, + "learning_rate": 9.058585858585859e-05, + "loss": 2.5489, + "theoretical_loss": 3.3202200308963805, + "tokens_seen": 3006398464 + }, + { + "epoch": 0.82, + "learning_rate": 9.056565656565658e-05, + "loss": 2.7013, + "theoretical_loss": 3.3202144685627606, + "tokens_seen": 3006464000 + }, + { + "epoch": 0.82, + "learning_rate": 9.054545454545455e-05, + "loss": 2.5317, + "theoretical_loss": 3.3202089063843383, + "tokens_seen": 3006529536 + }, + { + "epoch": 0.82, + "learning_rate": 9.052525252525253e-05, + "loss": 2.5508, + "theoretical_loss": 3.320203344361105, + "tokens_seen": 3006595072 + }, + { + "epoch": 0.82, + "learning_rate": 9.05050505050505e-05, + "loss": 2.7538, + "theoretical_loss": 3.320197782493055, + "tokens_seen": 3006660608 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1691682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6395201683044434, + "objective/train/theoretical_loss": 3.3201922207801786, + "objective/train/tokens_used": 1365585376, + "theoretical_loss": 3.3201922207801786, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 9.048484848484849e-05, + "loss": 2.4818, + "theoretical_loss": 3.3201922207801786, + "tokens_seen": 3006726144 + }, + { + "epoch": 0.82, + "learning_rate": 9.046464646464647e-05, + "loss": 2.8995, + "theoretical_loss": 3.3201866592224696, + "tokens_seen": 3006791680 + }, + { + "epoch": 0.82, + "learning_rate": 9.044444444444444e-05, + "loss": 2.6597, + "theoretical_loss": 3.320181097819919, + "tokens_seen": 3006857216 + }, + { + "epoch": 0.82, + "learning_rate": 9.042424242424242e-05, + "loss": 2.5626, + "theoretical_loss": 3.32017553657252, + "tokens_seen": 3006922752 + }, + { + "epoch": 0.82, + "learning_rate": 9.040404040404041e-05, + "loss": 2.5558, + "theoretical_loss": 3.3201699754802645, + "tokens_seen": 3006988288 + }, + { + "epoch": 0.82, + "learning_rate": 9.038383838383839e-05, + "loss": 2.565, + "theoretical_loss": 3.320164414543145, + "tokens_seen": 3007053824 + }, + { + "epoch": 0.82, + "learning_rate": 9.036363636363638e-05, + "loss": 2.7779, + "theoretical_loss": 3.3201588537611535, + "tokens_seen": 3007119360 + }, + { + "epoch": 0.82, + "learning_rate": 9.034343434343433e-05, + "loss": 2.6528, + "theoretical_loss": 3.320153293134282, + "tokens_seen": 3007184896 + }, + { + "epoch": 0.82, + "learning_rate": 9.032323232323232e-05, + "loss": 2.7138, + "theoretical_loss": 3.320147732662524, + "tokens_seen": 3007250432 + }, + { + "epoch": 0.82, + "learning_rate": 9.03030303030303e-05, + "loss": 2.7969, + "theoretical_loss": 3.3201421723458706, + "tokens_seen": 3007315968 + }, + { + "epoch": 0.82, + "learning_rate": 9.028282828282828e-05, + "loss": 2.6161, + "theoretical_loss": 3.3201366121843146, + "tokens_seen": 3007381504 + }, + { + "epoch": 0.82, + "learning_rate": 9.026262626262627e-05, + "loss": 2.5899, + "theoretical_loss": 3.320131052177848, + "tokens_seen": 3007447040 + }, + { + "epoch": 0.82, + "learning_rate": 9.024242424242425e-05, + "loss": 2.8212, + "theoretical_loss": 3.320125492326464, + "tokens_seen": 3007512576 + }, + { + "epoch": 0.82, + "learning_rate": 9.022222222222222e-05, + "loss": 2.3951, + "theoretical_loss": 3.3201199326301536, + "tokens_seen": 3007578112 + }, + { + "epoch": 0.82, + "learning_rate": 9.02020202020202e-05, + "loss": 2.6661, + "theoretical_loss": 3.3201143730889098, + "tokens_seen": 3007643648 + }, + { + "epoch": 0.82, + "learning_rate": 9.018181818181819e-05, + "loss": 2.7333, + "theoretical_loss": 3.320108813702725, + "tokens_seen": 3007709184 + }, + { + "epoch": 0.82, + "learning_rate": 9.016161616161616e-05, + "loss": 2.532, + "theoretical_loss": 3.320103254471591, + "tokens_seen": 3007774720 + }, + { + "epoch": 0.82, + "learning_rate": 9.014141414141415e-05, + "loss": 2.7355, + "theoretical_loss": 3.320097695395501, + "tokens_seen": 3007840256 + }, + { + "epoch": 0.82, + "learning_rate": 9.012121212121212e-05, + "loss": 2.4412, + "theoretical_loss": 3.320092136474446, + "tokens_seen": 3007905792 + }, + { + "epoch": 0.82, + "learning_rate": 9.01010101010101e-05, + "loss": 2.698, + "theoretical_loss": 3.320086577708419, + "tokens_seen": 3007971328 + }, + { + "epoch": 0.82, + "learning_rate": 9.008080808080808e-05, + "loss": 2.7331, + "theoretical_loss": 3.3200810190974126, + "tokens_seen": 3008036864 + }, + { + "epoch": 0.82, + "learning_rate": 9.006060606060607e-05, + "loss": 2.5875, + "theoretical_loss": 3.320075460641419, + "tokens_seen": 3008102400 + }, + { + "epoch": 0.82, + "learning_rate": 9.004040404040405e-05, + "loss": 2.3991, + "theoretical_loss": 3.32006990234043, + "tokens_seen": 3008167936 + }, + { + "epoch": 0.82, + "learning_rate": 9.002020202020202e-05, + "loss": 2.7041, + "theoretical_loss": 3.320064344194438, + "tokens_seen": 3008233472 + }, + { + "epoch": 0.82, + "learning_rate": 8.999999999999999e-05, + "loss": 2.6143, + "theoretical_loss": 3.320058786203436, + "tokens_seen": 3008299008 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1692449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5855281352996826, + "objective/train/theoretical_loss": 3.3200532283674153, + "objective/train/tokens_used": 1367223776, + "theoretical_loss": 3.3200532283674153, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.82, + "learning_rate": 8.997979797979798e-05, + "loss": 2.6435, + "theoretical_loss": 3.3200532283674153, + "tokens_seen": 3008364544 + }, + { + "epoch": 0.82, + "learning_rate": 8.995959595959596e-05, + "loss": 2.4907, + "theoretical_loss": 3.320047670686369, + "tokens_seen": 3008430080 + }, + { + "epoch": 0.82, + "learning_rate": 8.993939393939394e-05, + "loss": 2.7612, + "theoretical_loss": 3.320042113160289, + "tokens_seen": 3008495616 + }, + { + "epoch": 0.82, + "learning_rate": 8.991919191919191e-05, + "loss": 2.6732, + "theoretical_loss": 3.3200365557891676, + "tokens_seen": 3008561152 + }, + { + "epoch": 0.82, + "learning_rate": 8.98989898989899e-05, + "loss": 2.52, + "theoretical_loss": 3.320030998572997, + "tokens_seen": 3008626688 + }, + { + "epoch": 0.82, + "learning_rate": 8.987878787878788e-05, + "loss": 2.6745, + "theoretical_loss": 3.3200254415117705, + "tokens_seen": 3008692224 + }, + { + "epoch": 0.82, + "learning_rate": 8.985858585858587e-05, + "loss": 2.6081, + "theoretical_loss": 3.320019884605479, + "tokens_seen": 3008757760 + }, + { + "epoch": 0.82, + "learning_rate": 8.983838383838385e-05, + "loss": 2.6519, + "theoretical_loss": 3.3200143278541154, + "tokens_seen": 3008823296 + }, + { + "epoch": 0.82, + "learning_rate": 8.981818181818182e-05, + "loss": 2.6314, + "theoretical_loss": 3.3200087712576725, + "tokens_seen": 3008888832 + }, + { + "epoch": 0.82, + "learning_rate": 8.979797979797979e-05, + "loss": 2.7378, + "theoretical_loss": 3.3200032148161416, + "tokens_seen": 3008954368 + }, + { + "epoch": 0.82, + "learning_rate": 8.977777777777778e-05, + "loss": 2.6911, + "theoretical_loss": 3.319997658529516, + "tokens_seen": 3009019904 + }, + { + "epoch": 0.82, + "learning_rate": 8.975757575757576e-05, + "loss": 2.4451, + "theoretical_loss": 3.319992102397787, + "tokens_seen": 3009085440 + }, + { + "epoch": 0.82, + "learning_rate": 8.973737373737374e-05, + "loss": 2.6188, + "theoretical_loss": 3.3199865464209477, + "tokens_seen": 3009150976 + }, + { + "epoch": 0.82, + "learning_rate": 8.971717171717173e-05, + "loss": 2.6513, + "theoretical_loss": 3.3199809905989905, + "tokens_seen": 3009216512 + }, + { + "epoch": 0.82, + "learning_rate": 8.96969696969697e-05, + "loss": 2.4069, + "theoretical_loss": 3.319975434931907, + "tokens_seen": 3009282048 + }, + { + "epoch": 0.82, + "learning_rate": 8.967676767676768e-05, + "loss": 2.8865, + "theoretical_loss": 3.3199698794196904, + "tokens_seen": 3009347584 + }, + { + "epoch": 0.82, + "learning_rate": 8.965656565656565e-05, + "loss": 2.743, + "theoretical_loss": 3.3199643240623318, + "tokens_seen": 3009413120 + }, + { + "epoch": 0.82, + "learning_rate": 8.963636363636364e-05, + "loss": 2.6932, + "theoretical_loss": 3.3199587688598244, + "tokens_seen": 3009478656 + }, + { + "epoch": 0.82, + "learning_rate": 8.961616161616162e-05, + "loss": 2.7001, + "theoretical_loss": 3.3199532138121604, + "tokens_seen": 3009544192 + }, + { + "epoch": 0.82, + "learning_rate": 8.959595959595959e-05, + "loss": 2.6071, + "theoretical_loss": 3.3199476589193324, + "tokens_seen": 3009609728 + }, + { + "epoch": 0.82, + "learning_rate": 8.957575757575757e-05, + "loss": 2.4898, + "theoretical_loss": 3.3199421041813317, + "tokens_seen": 3009675264 + }, + { + "epoch": 0.82, + "learning_rate": 8.955555555555556e-05, + "loss": 2.6556, + "theoretical_loss": 3.3199365495981517, + "tokens_seen": 3009740800 + }, + { + "epoch": 0.82, + "learning_rate": 8.953535353535354e-05, + "loss": 2.5102, + "theoretical_loss": 3.319930995169784, + "tokens_seen": 3009806336 + }, + { + "epoch": 0.82, + "learning_rate": 8.951515151515153e-05, + "loss": 2.7162, + "theoretical_loss": 3.3199254408962213, + "tokens_seen": 3009871872 + }, + { + "epoch": 0.82, + "learning_rate": 8.94949494949495e-05, + "loss": 2.4096, + "theoretical_loss": 3.319919886777456, + "tokens_seen": 3009937408 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1693425, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.781566858291626, + "objective/train/theoretical_loss": 3.3199143328134797, + "objective/train/tokens_used": 1368862176, + "theoretical_loss": 3.3199143328134797, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 8.947474747474747e-05, + "loss": 2.5641, + "theoretical_loss": 3.3199143328134797, + "tokens_seen": 3010002944 + }, + { + "epoch": 0.82, + "learning_rate": 8.945454545454545e-05, + "loss": 2.6837, + "theoretical_loss": 3.319908779004286, + "tokens_seen": 3010068480 + }, + { + "epoch": 0.82, + "learning_rate": 8.943434343434343e-05, + "loss": 2.5555, + "theoretical_loss": 3.319903225349866, + "tokens_seen": 3010134016 + }, + { + "epoch": 0.82, + "learning_rate": 8.941414141414142e-05, + "loss": 2.6547, + "theoretical_loss": 3.3198976718502125, + "tokens_seen": 3010199552 + }, + { + "epoch": 0.82, + "learning_rate": 8.93939393939394e-05, + "loss": 2.3656, + "theoretical_loss": 3.319892118505318, + "tokens_seen": 3010265088 + }, + { + "epoch": 0.82, + "learning_rate": 8.937373737373737e-05, + "loss": 2.7814, + "theoretical_loss": 3.319886565315174, + "tokens_seen": 3010330624 + }, + { + "epoch": 0.82, + "learning_rate": 8.935353535353536e-05, + "loss": 2.6155, + "theoretical_loss": 3.3198810122797737, + "tokens_seen": 3010396160 + }, + { + "epoch": 0.82, + "learning_rate": 8.933333333333334e-05, + "loss": 2.6776, + "theoretical_loss": 3.3198754593991096, + "tokens_seen": 3010461696 + }, + { + "epoch": 0.82, + "learning_rate": 8.931313131313132e-05, + "loss": 2.6029, + "theoretical_loss": 3.319869906673173, + "tokens_seen": 3010527232 + }, + { + "epoch": 0.82, + "learning_rate": 8.92929292929293e-05, + "loss": 2.6562, + "theoretical_loss": 3.319864354101957, + "tokens_seen": 3010592768 + }, + { + "epoch": 0.82, + "learning_rate": 8.927272727272727e-05, + "loss": 2.5015, + "theoretical_loss": 3.3198588016854536, + "tokens_seen": 3010658304 + }, + { + "epoch": 0.82, + "learning_rate": 8.925252525252525e-05, + "loss": 2.9045, + "theoretical_loss": 3.3198532494236552, + "tokens_seen": 3010723840 + }, + { + "epoch": 0.82, + "learning_rate": 8.923232323232323e-05, + "loss": 2.6659, + "theoretical_loss": 3.3198476973165545, + "tokens_seen": 3010789376 + }, + { + "epoch": 0.82, + "learning_rate": 8.921212121212122e-05, + "loss": 2.6566, + "theoretical_loss": 3.3198421453641433, + "tokens_seen": 3010854912 + }, + { + "epoch": 0.82, + "learning_rate": 8.91919191919192e-05, + "loss": 2.3997, + "theoretical_loss": 3.319836593566414, + "tokens_seen": 3010920448 + }, + { + "epoch": 0.82, + "learning_rate": 8.917171717171717e-05, + "loss": 2.3178, + "theoretical_loss": 3.3198310419233588, + "tokens_seen": 3010985984 + }, + { + "epoch": 0.82, + "learning_rate": 8.915151515151516e-05, + "loss": 2.7118, + "theoretical_loss": 3.3198254904349707, + "tokens_seen": 3011051520 + }, + { + "epoch": 0.82, + "learning_rate": 8.913131313131313e-05, + "loss": 2.5981, + "theoretical_loss": 3.319819939101241, + "tokens_seen": 3011117056 + }, + { + "epoch": 0.82, + "learning_rate": 8.911111111111111e-05, + "loss": 2.7197, + "theoretical_loss": 3.3198143879221633, + "tokens_seen": 3011182592 + }, + { + "epoch": 0.82, + "learning_rate": 8.90909090909091e-05, + "loss": 2.7977, + "theoretical_loss": 3.319808836897729, + "tokens_seen": 3011248128 + }, + { + "epoch": 0.82, + "learning_rate": 8.907070707070706e-05, + "loss": 2.4746, + "theoretical_loss": 3.3198032860279305, + "tokens_seen": 3011313664 + }, + { + "epoch": 0.82, + "learning_rate": 8.905050505050505e-05, + "loss": 2.5963, + "theoretical_loss": 3.31979773531276, + "tokens_seen": 3011379200 + }, + { + "epoch": 0.82, + "learning_rate": 8.903030303030303e-05, + "loss": 2.6583, + "theoretical_loss": 3.3197921847522105, + "tokens_seen": 3011444736 + }, + { + "epoch": 0.82, + "learning_rate": 8.901010101010102e-05, + "loss": 2.7818, + "theoretical_loss": 3.319786634346274, + "tokens_seen": 3011510272 + }, + { + "epoch": 0.82, + "learning_rate": 8.8989898989899e-05, + "loss": 2.3946, + "theoretical_loss": 3.3197810840949424, + "tokens_seen": 3011575808 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1694138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.954774856567383, + "objective/train/theoretical_loss": 3.3197755339982082, + "objective/train/tokens_used": 1370500576, + "theoretical_loss": 3.3197755339982082, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.82, + "learning_rate": 8.896969696969698e-05, + "loss": 2.628, + "theoretical_loss": 3.3197755339982082, + "tokens_seen": 3011641344 + }, + { + "epoch": 0.82, + "learning_rate": 8.894949494949494e-05, + "loss": 2.6018, + "theoretical_loss": 3.319769984056064, + "tokens_seen": 3011706880 + }, + { + "epoch": 0.82, + "learning_rate": 8.892929292929293e-05, + "loss": 2.4582, + "theoretical_loss": 3.3197644342685027, + "tokens_seen": 3011772416 + }, + { + "epoch": 0.82, + "learning_rate": 8.890909090909091e-05, + "loss": 2.7987, + "theoretical_loss": 3.3197588846355157, + "tokens_seen": 3011837952 + }, + { + "epoch": 0.82, + "learning_rate": 8.888888888888889e-05, + "loss": 2.7735, + "theoretical_loss": 3.3197533351570954, + "tokens_seen": 3011903488 + }, + { + "epoch": 0.82, + "learning_rate": 8.886868686868688e-05, + "loss": 2.6206, + "theoretical_loss": 3.3197477858332345, + "tokens_seen": 3011969024 + }, + { + "epoch": 0.82, + "learning_rate": 8.884848484848485e-05, + "loss": 2.8019, + "theoretical_loss": 3.3197422366639247, + "tokens_seen": 3012034560 + }, + { + "epoch": 0.82, + "learning_rate": 8.882828282828283e-05, + "loss": 2.6226, + "theoretical_loss": 3.319736687649159, + "tokens_seen": 3012100096 + }, + { + "epoch": 0.82, + "learning_rate": 8.880808080808082e-05, + "loss": 2.4918, + "theoretical_loss": 3.31973113878893, + "tokens_seen": 3012165632 + }, + { + "epoch": 0.82, + "learning_rate": 8.878787878787879e-05, + "loss": 2.8713, + "theoretical_loss": 3.3197255900832294, + "tokens_seen": 3012231168 + }, + { + "epoch": 0.82, + "learning_rate": 8.876767676767677e-05, + "loss": 2.6114, + "theoretical_loss": 3.3197200415320496, + "tokens_seen": 3012296704 + }, + { + "epoch": 0.82, + "learning_rate": 8.874747474747474e-05, + "loss": 2.5979, + "theoretical_loss": 3.319714493135383, + "tokens_seen": 3012362240 + }, + { + "epoch": 0.82, + "learning_rate": 8.872727272727272e-05, + "loss": 2.5755, + "theoretical_loss": 3.319708944893222, + "tokens_seen": 3012427776 + }, + { + "epoch": 0.82, + "learning_rate": 8.870707070707071e-05, + "loss": 2.552, + "theoretical_loss": 3.3197033968055587, + "tokens_seen": 3012493312 + }, + { + "epoch": 0.82, + "learning_rate": 8.868686868686869e-05, + "loss": 2.7328, + "theoretical_loss": 3.3196978488723863, + "tokens_seen": 3012558848 + }, + { + "epoch": 0.82, + "learning_rate": 8.866666666666668e-05, + "loss": 2.5555, + "theoretical_loss": 3.3196923010936956, + "tokens_seen": 3012624384 + }, + { + "epoch": 0.82, + "learning_rate": 8.864646464646465e-05, + "loss": 2.6711, + "theoretical_loss": 3.3196867534694805, + "tokens_seen": 3012689920 + }, + { + "epoch": 0.82, + "learning_rate": 8.862626262626263e-05, + "loss": 2.5557, + "theoretical_loss": 3.3196812059997325, + "tokens_seen": 3012755456 + }, + { + "epoch": 0.82, + "learning_rate": 8.86060606060606e-05, + "loss": 2.493, + "theoretical_loss": 3.319675658684444, + "tokens_seen": 3012820992 + }, + { + "epoch": 0.82, + "learning_rate": 8.858585858585858e-05, + "loss": 2.6951, + "theoretical_loss": 3.3196701115236076, + "tokens_seen": 3012886528 + }, + { + "epoch": 0.82, + "learning_rate": 8.856565656565657e-05, + "loss": 2.4655, + "theoretical_loss": 3.3196645645172156, + "tokens_seen": 3012952064 + }, + { + "epoch": 0.82, + "learning_rate": 8.854545454545455e-05, + "loss": 2.6858, + "theoretical_loss": 3.3196590176652596, + "tokens_seen": 3013017600 + }, + { + "epoch": 0.82, + "learning_rate": 8.852525252525252e-05, + "loss": 2.7773, + "theoretical_loss": 3.319653470967733, + "tokens_seen": 3013083136 + }, + { + "epoch": 0.82, + "learning_rate": 8.850505050505051e-05, + "loss": 2.7118, + "theoretical_loss": 3.3196479244246277, + "tokens_seen": 3013148672 + }, + { + "epoch": 0.82, + "learning_rate": 8.848484848484849e-05, + "loss": 2.4726, + "theoretical_loss": 3.319642378035936, + "tokens_seen": 3013214208 + }, + { + "epoch": 0.82, + "objective/train/docs_used": 1695115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.574808359146118, + "objective/train/theoretical_loss": 3.3196368318016507, + "objective/train/tokens_used": 1372138976, + "theoretical_loss": 3.3196368318016507, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.82, + "learning_rate": 8.846464646464648e-05, + "loss": 2.7252, + "theoretical_loss": 3.3196368318016507, + "tokens_seen": 3013279744 + }, + { + "epoch": 0.83, + "learning_rate": 8.844444444444446e-05, + "loss": 2.5074, + "theoretical_loss": 3.319631285721763, + "tokens_seen": 3013345280 + }, + { + "epoch": 0.83, + "learning_rate": 8.842424242424242e-05, + "loss": 2.6227, + "theoretical_loss": 3.3196257397962663, + "tokens_seen": 3013410816 + }, + { + "epoch": 0.83, + "learning_rate": 8.84040404040404e-05, + "loss": 2.5959, + "theoretical_loss": 3.319620194025153, + "tokens_seen": 3013476352 + }, + { + "epoch": 0.83, + "learning_rate": 8.838383838383838e-05, + "loss": 2.7269, + "theoretical_loss": 3.3196146484084146, + "tokens_seen": 3013541888 + }, + { + "epoch": 0.83, + "learning_rate": 8.836363636363637e-05, + "loss": 2.5042, + "theoretical_loss": 3.319609102946044, + "tokens_seen": 3013607424 + }, + { + "epoch": 0.83, + "learning_rate": 8.834343434343435e-05, + "loss": 2.7836, + "theoretical_loss": 3.3196035576380334, + "tokens_seen": 3013672960 + }, + { + "epoch": 0.83, + "learning_rate": 8.832323232323232e-05, + "loss": 2.7875, + "theoretical_loss": 3.3195980124843754, + "tokens_seen": 3013738496 + }, + { + "epoch": 0.83, + "learning_rate": 8.83030303030303e-05, + "loss": 2.6196, + "theoretical_loss": 3.319592467485062, + "tokens_seen": 3013804032 + }, + { + "epoch": 0.83, + "learning_rate": 8.828282828282829e-05, + "loss": 2.7387, + "theoretical_loss": 3.319586922640086, + "tokens_seen": 3013869568 + }, + { + "epoch": 0.83, + "learning_rate": 8.826262626262626e-05, + "loss": 2.7032, + "theoretical_loss": 3.319581377949439, + "tokens_seen": 3013935104 + }, + { + "epoch": 0.83, + "learning_rate": 8.824242424242424e-05, + "loss": 2.7236, + "theoretical_loss": 3.319575833413114, + "tokens_seen": 3014000640 + }, + { + "epoch": 0.83, + "learning_rate": 8.822222222222221e-05, + "loss": 2.7746, + "theoretical_loss": 3.319570289031103, + "tokens_seen": 3014066176 + }, + { + "epoch": 0.83, + "learning_rate": 8.82020202020202e-05, + "loss": 2.6424, + "theoretical_loss": 3.319564744803399, + "tokens_seen": 3014131712 + }, + { + "epoch": 0.83, + "learning_rate": 8.818181818181818e-05, + "loss": 2.5125, + "theoretical_loss": 3.3195592007299934, + "tokens_seen": 3014197248 + }, + { + "epoch": 0.83, + "learning_rate": 8.816161616161617e-05, + "loss": 2.8322, + "theoretical_loss": 3.319553656810879, + "tokens_seen": 3014262784 + }, + { + "epoch": 0.83, + "learning_rate": 8.814141414141415e-05, + "loss": 2.8665, + "theoretical_loss": 3.3195481130460482, + "tokens_seen": 3014328320 + }, + { + "epoch": 0.83, + "learning_rate": 8.812121212121213e-05, + "loss": 2.714, + "theoretical_loss": 3.3195425694354936, + "tokens_seen": 3014393856 + }, + { + "epoch": 0.83, + "learning_rate": 8.81010101010101e-05, + "loss": 2.6602, + "theoretical_loss": 3.3195370259792067, + "tokens_seen": 3014459392 + }, + { + "epoch": 0.83, + "learning_rate": 8.808080808080808e-05, + "loss": 2.5629, + "theoretical_loss": 3.319531482677181, + "tokens_seen": 3014524928 + }, + { + "epoch": 0.83, + "learning_rate": 8.806060606060606e-05, + "loss": 2.6041, + "theoretical_loss": 3.319525939529408, + "tokens_seen": 3014590464 + }, + { + "epoch": 0.83, + "learning_rate": 8.804040404040404e-05, + "loss": 2.5479, + "theoretical_loss": 3.3195203965358804, + "tokens_seen": 3014656000 + }, + { + "epoch": 0.83, + "learning_rate": 8.802020202020203e-05, + "loss": 2.6351, + "theoretical_loss": 3.3195148536965906, + "tokens_seen": 3014721536 + }, + { + "epoch": 0.83, + "learning_rate": 8.8e-05, + "loss": 2.4776, + "theoretical_loss": 3.3195093110115304, + "tokens_seen": 3014787072 + }, + { + "epoch": 0.83, + "learning_rate": 8.797979797979798e-05, + "loss": 2.4084, + "theoretical_loss": 3.319503768480693, + "tokens_seen": 3014852608 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1695676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670842409133911, + "objective/train/theoretical_loss": 3.31949822610407, + "objective/train/tokens_used": 1373777376, + "theoretical_loss": 3.31949822610407, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.83, + "learning_rate": 8.795959595959597e-05, + "loss": 2.7017, + "theoretical_loss": 3.31949822610407, + "tokens_seen": 3014918144 + }, + { + "epoch": 0.83, + "learning_rate": 8.793939393939395e-05, + "loss": 2.449, + "theoretical_loss": 3.3194926838816543, + "tokens_seen": 3014983680 + }, + { + "epoch": 0.83, + "learning_rate": 8.791919191919192e-05, + "loss": 2.4385, + "theoretical_loss": 3.319487141813438, + "tokens_seen": 3015049216 + }, + { + "epoch": 0.83, + "learning_rate": 8.789898989898989e-05, + "loss": 2.5621, + "theoretical_loss": 3.3194815998994134, + "tokens_seen": 3015114752 + }, + { + "epoch": 0.83, + "learning_rate": 8.787878787878787e-05, + "loss": 2.5066, + "theoretical_loss": 3.319476058139573, + "tokens_seen": 3015180288 + }, + { + "epoch": 0.83, + "learning_rate": 8.785858585858586e-05, + "loss": 2.819, + "theoretical_loss": 3.3194705165339093, + "tokens_seen": 3015245824 + }, + { + "epoch": 0.83, + "learning_rate": 8.783838383838384e-05, + "loss": 2.6909, + "theoretical_loss": 3.319464975082415, + "tokens_seen": 3015311360 + }, + { + "epoch": 0.83, + "learning_rate": 8.781818181818183e-05, + "loss": 2.3987, + "theoretical_loss": 3.319459433785081, + "tokens_seen": 3015376896 + }, + { + "epoch": 0.83, + "learning_rate": 8.77979797979798e-05, + "loss": 2.4557, + "theoretical_loss": 3.319453892641901, + "tokens_seen": 3015442432 + }, + { + "epoch": 0.83, + "learning_rate": 8.777777777777778e-05, + "loss": 2.5505, + "theoretical_loss": 3.319448351652867, + "tokens_seen": 3015507968 + }, + { + "epoch": 0.83, + "learning_rate": 8.775757575757576e-05, + "loss": 2.3179, + "theoretical_loss": 3.3194428108179714, + "tokens_seen": 3015573504 + }, + { + "epoch": 0.83, + "learning_rate": 8.773737373737373e-05, + "loss": 2.5443, + "theoretical_loss": 3.319437270137206, + "tokens_seen": 3015639040 + }, + { + "epoch": 0.83, + "learning_rate": 8.771717171717172e-05, + "loss": 2.4431, + "theoretical_loss": 3.319431729610564, + "tokens_seen": 3015704576 + }, + { + "epoch": 0.83, + "learning_rate": 8.76969696969697e-05, + "loss": 2.6297, + "theoretical_loss": 3.3194261892380377, + "tokens_seen": 3015770112 + }, + { + "epoch": 0.83, + "learning_rate": 8.767676767676767e-05, + "loss": 2.6339, + "theoretical_loss": 3.319420649019619, + "tokens_seen": 3015835648 + }, + { + "epoch": 0.83, + "learning_rate": 8.765656565656566e-05, + "loss": 2.7149, + "theoretical_loss": 3.3194151089553006, + "tokens_seen": 3015901184 + }, + { + "epoch": 0.83, + "learning_rate": 8.763636363636364e-05, + "loss": 2.5414, + "theoretical_loss": 3.3194095690450744, + "tokens_seen": 3015966720 + }, + { + "epoch": 0.83, + "learning_rate": 8.761616161616163e-05, + "loss": 2.5336, + "theoretical_loss": 3.3194040292889335, + "tokens_seen": 3016032256 + }, + { + "epoch": 0.83, + "learning_rate": 8.759595959595961e-05, + "loss": 2.6721, + "theoretical_loss": 3.3193984896868693, + "tokens_seen": 3016097792 + }, + { + "epoch": 0.83, + "learning_rate": 8.757575757575757e-05, + "loss": 2.7151, + "theoretical_loss": 3.3193929502388753, + "tokens_seen": 3016163328 + }, + { + "epoch": 0.83, + "learning_rate": 8.755555555555555e-05, + "loss": 2.5325, + "theoretical_loss": 3.319387410944943, + "tokens_seen": 3016228864 + }, + { + "epoch": 0.83, + "learning_rate": 8.753535353535353e-05, + "loss": 2.5562, + "theoretical_loss": 3.319381871805065, + "tokens_seen": 3016294400 + }, + { + "epoch": 0.83, + "learning_rate": 8.751515151515152e-05, + "loss": 2.3467, + "theoretical_loss": 3.3193763328192336, + "tokens_seen": 3016359936 + }, + { + "epoch": 0.83, + "learning_rate": 8.74949494949495e-05, + "loss": 2.1696, + "theoretical_loss": 3.3193707939874417, + "tokens_seen": 3016425472 + }, + { + "epoch": 0.83, + "learning_rate": 8.747474747474747e-05, + "loss": 2.7295, + "theoretical_loss": 3.319365255309681, + "tokens_seen": 3016491008 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1696888, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7353992462158203, + "objective/train/theoretical_loss": 3.3193597167859443, + "objective/train/tokens_used": 1375415776, + "theoretical_loss": 3.3193597167859443, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 8.745454545454546e-05, + "loss": 2.7637, + "theoretical_loss": 3.3193597167859443, + "tokens_seen": 3016556544 + }, + { + "epoch": 0.83, + "learning_rate": 8.743434343434344e-05, + "loss": 2.39, + "theoretical_loss": 3.3193541784162237, + "tokens_seen": 3016622080 + }, + { + "epoch": 0.83, + "learning_rate": 8.741414141414142e-05, + "loss": 2.4656, + "theoretical_loss": 3.3193486402005115, + "tokens_seen": 3016687616 + }, + { + "epoch": 0.83, + "learning_rate": 8.73939393939394e-05, + "loss": 2.6515, + "theoretical_loss": 3.3193431021388005, + "tokens_seen": 3016753152 + }, + { + "epoch": 0.83, + "learning_rate": 8.737373737373736e-05, + "loss": 2.6111, + "theoretical_loss": 3.3193375642310827, + "tokens_seen": 3016818688 + }, + { + "epoch": 0.83, + "learning_rate": 8.735353535353535e-05, + "loss": 2.5664, + "theoretical_loss": 3.3193320264773507, + "tokens_seen": 3016884224 + }, + { + "epoch": 0.83, + "learning_rate": 8.733333333333333e-05, + "loss": 2.6489, + "theoretical_loss": 3.3193264888775964, + "tokens_seen": 3016949760 + }, + { + "epoch": 0.83, + "learning_rate": 8.731313131313132e-05, + "loss": 2.5056, + "theoretical_loss": 3.3193209514318127, + "tokens_seen": 3017015296 + }, + { + "epoch": 0.83, + "learning_rate": 8.72929292929293e-05, + "loss": 2.7005, + "theoretical_loss": 3.319315414139992, + "tokens_seen": 3017080832 + }, + { + "epoch": 0.83, + "learning_rate": 8.727272727272728e-05, + "loss": 2.6548, + "theoretical_loss": 3.3193098770021265, + "tokens_seen": 3017146368 + }, + { + "epoch": 0.83, + "learning_rate": 8.725252525252526e-05, + "loss": 2.5633, + "theoretical_loss": 3.3193043400182085, + "tokens_seen": 3017211904 + }, + { + "epoch": 0.83, + "learning_rate": 8.723232323232324e-05, + "loss": 2.5466, + "theoretical_loss": 3.3192988031882305, + "tokens_seen": 3017277440 + }, + { + "epoch": 0.83, + "learning_rate": 8.721212121212121e-05, + "loss": 2.6479, + "theoretical_loss": 3.319293266512185, + "tokens_seen": 3017342976 + }, + { + "epoch": 0.83, + "learning_rate": 8.71919191919192e-05, + "loss": 2.5509, + "theoretical_loss": 3.3192877299900636, + "tokens_seen": 3017408512 + }, + { + "epoch": 0.83, + "learning_rate": 8.717171717171718e-05, + "loss": 2.5809, + "theoretical_loss": 3.3192821936218597, + "tokens_seen": 3017474048 + }, + { + "epoch": 0.83, + "learning_rate": 8.715151515151515e-05, + "loss": 2.5118, + "theoretical_loss": 3.319276657407565, + "tokens_seen": 3017539584 + }, + { + "epoch": 0.83, + "learning_rate": 8.713131313131313e-05, + "loss": 2.6888, + "theoretical_loss": 3.3192711213471724, + "tokens_seen": 3017605120 + }, + { + "epoch": 0.83, + "learning_rate": 8.711111111111112e-05, + "loss": 2.6471, + "theoretical_loss": 3.319265585440674, + "tokens_seen": 3017670656 + }, + { + "epoch": 0.83, + "learning_rate": 8.70909090909091e-05, + "loss": 2.6364, + "theoretical_loss": 3.319260049688062, + "tokens_seen": 3017736192 + }, + { + "epoch": 0.83, + "learning_rate": 8.707070707070708e-05, + "loss": 2.9157, + "theoretical_loss": 3.3192545140893293, + "tokens_seen": 3017801728 + }, + { + "epoch": 0.83, + "learning_rate": 8.705050505050504e-05, + "loss": 2.7804, + "theoretical_loss": 3.3192489786444677, + "tokens_seen": 3017867264 + }, + { + "epoch": 0.83, + "learning_rate": 8.703030303030302e-05, + "loss": 2.633, + "theoretical_loss": 3.3192434433534697, + "tokens_seen": 3017932800 + }, + { + "epoch": 0.83, + "learning_rate": 8.701010101010101e-05, + "loss": 2.7452, + "theoretical_loss": 3.3192379082163277, + "tokens_seen": 3017998336 + }, + { + "epoch": 0.83, + "learning_rate": 8.698989898989899e-05, + "loss": 2.6836, + "theoretical_loss": 3.3192323732330347, + "tokens_seen": 3018063872 + }, + { + "epoch": 0.83, + "learning_rate": 8.696969696969698e-05, + "loss": 2.5202, + "theoretical_loss": 3.319226838403582, + "tokens_seen": 3018129408 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1697654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.883934736251831, + "objective/train/theoretical_loss": 3.319221303727963, + "objective/train/tokens_used": 1377054176, + "theoretical_loss": 3.319221303727963, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.83, + "learning_rate": 8.694949494949495e-05, + "loss": 2.6914, + "theoretical_loss": 3.319221303727963, + "tokens_seen": 3018194944 + }, + { + "epoch": 0.83, + "learning_rate": 8.692929292929293e-05, + "loss": 2.848, + "theoretical_loss": 3.3192157692061697, + "tokens_seen": 3018260480 + }, + { + "epoch": 0.83, + "learning_rate": 8.690909090909091e-05, + "loss": 2.5718, + "theoretical_loss": 3.319210234838194, + "tokens_seen": 3018326016 + }, + { + "epoch": 0.83, + "learning_rate": 8.68888888888889e-05, + "loss": 2.4217, + "theoretical_loss": 3.319204700624029, + "tokens_seen": 3018391552 + }, + { + "epoch": 0.83, + "learning_rate": 8.686868686868687e-05, + "loss": 2.7179, + "theoretical_loss": 3.3191991665636666, + "tokens_seen": 3018457088 + }, + { + "epoch": 0.83, + "learning_rate": 8.684848484848485e-05, + "loss": 2.6952, + "theoretical_loss": 3.3191936326571, + "tokens_seen": 3018522624 + }, + { + "epoch": 0.83, + "learning_rate": 8.682828282828282e-05, + "loss": 2.4499, + "theoretical_loss": 3.3191880989043203, + "tokens_seen": 3018588160 + }, + { + "epoch": 0.83, + "learning_rate": 8.680808080808081e-05, + "loss": 2.5262, + "theoretical_loss": 3.319182565305321, + "tokens_seen": 3018653696 + }, + { + "epoch": 0.83, + "learning_rate": 8.678787878787879e-05, + "loss": 2.4967, + "theoretical_loss": 3.319177031860094, + "tokens_seen": 3018719232 + }, + { + "epoch": 0.83, + "learning_rate": 8.676767676767678e-05, + "loss": 2.621, + "theoretical_loss": 3.319171498568631, + "tokens_seen": 3018784768 + }, + { + "epoch": 0.83, + "learning_rate": 8.674747474747476e-05, + "loss": 2.8268, + "theoretical_loss": 3.3191659654309262, + "tokens_seen": 3018850304 + }, + { + "epoch": 0.83, + "learning_rate": 8.672727272727273e-05, + "loss": 2.7592, + "theoretical_loss": 3.3191604324469703, + "tokens_seen": 3018915840 + }, + { + "epoch": 0.83, + "learning_rate": 8.67070707070707e-05, + "loss": 2.5885, + "theoretical_loss": 3.3191548996167564, + "tokens_seen": 3018981376 + }, + { + "epoch": 0.83, + "learning_rate": 8.668686868686868e-05, + "loss": 2.5215, + "theoretical_loss": 3.3191493669402767, + "tokens_seen": 3019046912 + }, + { + "epoch": 0.83, + "learning_rate": 8.666666666666667e-05, + "loss": 2.5908, + "theoretical_loss": 3.319143834417524, + "tokens_seen": 3019112448 + }, + { + "epoch": 0.83, + "learning_rate": 8.664646464646465e-05, + "loss": 2.5964, + "theoretical_loss": 3.3191383020484904, + "tokens_seen": 3019177984 + }, + { + "epoch": 0.83, + "learning_rate": 8.662626262626262e-05, + "loss": 2.6369, + "theoretical_loss": 3.319132769833168, + "tokens_seen": 3019243520 + }, + { + "epoch": 0.83, + "learning_rate": 8.66060606060606e-05, + "loss": 2.5183, + "theoretical_loss": 3.3191272377715495, + "tokens_seen": 3019309056 + }, + { + "epoch": 0.83, + "learning_rate": 8.658585858585859e-05, + "loss": 2.6548, + "theoretical_loss": 3.319121705863627, + "tokens_seen": 3019374592 + }, + { + "epoch": 0.83, + "learning_rate": 8.656565656565657e-05, + "loss": 2.5607, + "theoretical_loss": 3.3191161741093937, + "tokens_seen": 3019440128 + }, + { + "epoch": 0.83, + "learning_rate": 8.654545454545456e-05, + "loss": 2.5404, + "theoretical_loss": 3.3191106425088415, + "tokens_seen": 3019505664 + }, + { + "epoch": 0.83, + "learning_rate": 8.652525252525252e-05, + "loss": 2.7038, + "theoretical_loss": 3.3191051110619627, + "tokens_seen": 3019571200 + }, + { + "epoch": 0.83, + "learning_rate": 8.65050505050505e-05, + "loss": 2.5086, + "theoretical_loss": 3.319099579768749, + "tokens_seen": 3019636736 + }, + { + "epoch": 0.83, + "learning_rate": 8.648484848484848e-05, + "loss": 2.573, + "theoretical_loss": 3.319094048629194, + "tokens_seen": 3019702272 + }, + { + "epoch": 0.83, + "learning_rate": 8.646464646464647e-05, + "loss": 2.5491, + "theoretical_loss": 3.31908851764329, + "tokens_seen": 3019767808 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1699020, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5956108570098877, + "objective/train/theoretical_loss": 3.3190829868110288, + "objective/train/tokens_used": 1378692576, + "theoretical_loss": 3.3190829868110288, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 8.644444444444445e-05, + "loss": 2.7137, + "theoretical_loss": 3.3190829868110288, + "tokens_seen": 3019833344 + }, + { + "epoch": 0.83, + "learning_rate": 8.642424242424242e-05, + "loss": 2.511, + "theoretical_loss": 3.319077456132403, + "tokens_seen": 3019898880 + }, + { + "epoch": 0.83, + "learning_rate": 8.64040404040404e-05, + "loss": 2.671, + "theoretical_loss": 3.319071925607405, + "tokens_seen": 3019964416 + }, + { + "epoch": 0.83, + "learning_rate": 8.638383838383839e-05, + "loss": 2.7004, + "theoretical_loss": 3.3190663952360273, + "tokens_seen": 3020029952 + }, + { + "epoch": 0.83, + "learning_rate": 8.636363636363636e-05, + "loss": 2.6183, + "theoretical_loss": 3.319060865018262, + "tokens_seen": 3020095488 + }, + { + "epoch": 0.83, + "learning_rate": 8.634343434343434e-05, + "loss": 2.7478, + "theoretical_loss": 3.319055334954102, + "tokens_seen": 3020161024 + }, + { + "epoch": 0.83, + "learning_rate": 8.632323232323233e-05, + "loss": 2.6936, + "theoretical_loss": 3.319049805043539, + "tokens_seen": 3020226560 + }, + { + "epoch": 0.83, + "learning_rate": 8.63030303030303e-05, + "loss": 2.4452, + "theoretical_loss": 3.3190442752865663, + "tokens_seen": 3020292096 + }, + { + "epoch": 0.83, + "learning_rate": 8.628282828282828e-05, + "loss": 2.625, + "theoretical_loss": 3.3190387456831756, + "tokens_seen": 3020357632 + }, + { + "epoch": 0.83, + "learning_rate": 8.626262626262627e-05, + "loss": 2.7238, + "theoretical_loss": 3.31903321623336, + "tokens_seen": 3020423168 + }, + { + "epoch": 0.83, + "learning_rate": 8.624242424242425e-05, + "loss": 2.7366, + "theoretical_loss": 3.3190276869371105, + "tokens_seen": 3020488704 + }, + { + "epoch": 0.83, + "learning_rate": 8.622222222222223e-05, + "loss": 2.5917, + "theoretical_loss": 3.319022157794421, + "tokens_seen": 3020554240 + }, + { + "epoch": 0.83, + "learning_rate": 8.62020202020202e-05, + "loss": 2.8545, + "theoretical_loss": 3.3190166288052834, + "tokens_seen": 3020619776 + }, + { + "epoch": 0.83, + "learning_rate": 8.618181818181817e-05, + "loss": 2.7792, + "theoretical_loss": 3.31901109996969, + "tokens_seen": 3020685312 + }, + { + "epoch": 0.83, + "learning_rate": 8.616161616161616e-05, + "loss": 2.753, + "theoretical_loss": 3.3190055712876334, + "tokens_seen": 3020750848 + }, + { + "epoch": 0.83, + "learning_rate": 8.614141414141414e-05, + "loss": 2.5821, + "theoretical_loss": 3.3190000427591055, + "tokens_seen": 3020816384 + }, + { + "epoch": 0.83, + "learning_rate": 8.612121212121213e-05, + "loss": 2.5282, + "theoretical_loss": 3.3189945143840993, + "tokens_seen": 3020881920 + }, + { + "epoch": 0.83, + "learning_rate": 8.61010101010101e-05, + "loss": 2.6359, + "theoretical_loss": 3.3189889861626067, + "tokens_seen": 3020947456 + }, + { + "epoch": 0.83, + "learning_rate": 8.608080808080808e-05, + "loss": 2.8469, + "theoretical_loss": 3.3189834580946207, + "tokens_seen": 3021012992 + }, + { + "epoch": 0.83, + "learning_rate": 8.606060606060606e-05, + "loss": 2.5124, + "theoretical_loss": 3.318977930180133, + "tokens_seen": 3021078528 + }, + { + "epoch": 0.83, + "learning_rate": 8.604040404040405e-05, + "loss": 2.2699, + "theoretical_loss": 3.3189724024191367, + "tokens_seen": 3021144064 + }, + { + "epoch": 0.83, + "learning_rate": 8.602020202020203e-05, + "loss": 2.7632, + "theoretical_loss": 3.318966874811624, + "tokens_seen": 3021209600 + }, + { + "epoch": 0.83, + "learning_rate": 8.599999999999999e-05, + "loss": 2.8448, + "theoretical_loss": 3.318961347357587, + "tokens_seen": 3021275136 + }, + { + "epoch": 0.83, + "learning_rate": 8.597979797979797e-05, + "loss": 2.6279, + "theoretical_loss": 3.3189558200570186, + "tokens_seen": 3021340672 + }, + { + "epoch": 0.83, + "learning_rate": 8.595959595959596e-05, + "loss": 2.4841, + "theoretical_loss": 3.3189502929099106, + "tokens_seen": 3021406208 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1699637, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.413766860961914, + "objective/train/theoretical_loss": 3.318944765916256, + "objective/train/tokens_used": 1380330976, + "theoretical_loss": 3.318944765916256, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.83, + "learning_rate": 8.593939393939394e-05, + "loss": 2.4864, + "theoretical_loss": 3.318944765916256, + "tokens_seen": 3021471744 + }, + { + "epoch": 0.83, + "learning_rate": 8.591919191919193e-05, + "loss": 2.5361, + "theoretical_loss": 3.3189392390760464, + "tokens_seen": 3021537280 + }, + { + "epoch": 0.83, + "learning_rate": 8.589898989898991e-05, + "loss": 2.6096, + "theoretical_loss": 3.318933712389275, + "tokens_seen": 3021602816 + }, + { + "epoch": 0.83, + "learning_rate": 8.587878787878788e-05, + "loss": 2.4621, + "theoretical_loss": 3.3189281858559343, + "tokens_seen": 3021668352 + }, + { + "epoch": 0.83, + "learning_rate": 8.585858585858586e-05, + "loss": 2.502, + "theoretical_loss": 3.318922659476016, + "tokens_seen": 3021733888 + }, + { + "epoch": 0.83, + "learning_rate": 8.583838383838383e-05, + "loss": 2.4575, + "theoretical_loss": 3.3189171332495135, + "tokens_seen": 3021799424 + }, + { + "epoch": 0.83, + "learning_rate": 8.581818181818182e-05, + "loss": 2.6158, + "theoretical_loss": 3.3189116071764184, + "tokens_seen": 3021864960 + }, + { + "epoch": 0.83, + "learning_rate": 8.57979797979798e-05, + "loss": 2.6634, + "theoretical_loss": 3.318906081256723, + "tokens_seen": 3021930496 + }, + { + "epoch": 0.83, + "learning_rate": 8.577777777777777e-05, + "loss": 2.681, + "theoretical_loss": 3.31890055549042, + "tokens_seen": 3021996032 + }, + { + "epoch": 0.83, + "learning_rate": 8.575757575757576e-05, + "loss": 2.5658, + "theoretical_loss": 3.318895029877502, + "tokens_seen": 3022061568 + }, + { + "epoch": 0.83, + "learning_rate": 8.573737373737374e-05, + "loss": 2.522, + "theoretical_loss": 3.3188895044179616, + "tokens_seen": 3022127104 + }, + { + "epoch": 0.83, + "learning_rate": 8.571717171717172e-05, + "loss": 2.6955, + "theoretical_loss": 3.3188839791117903, + "tokens_seen": 3022192640 + }, + { + "epoch": 0.83, + "learning_rate": 8.569696969696971e-05, + "loss": 2.7069, + "theoretical_loss": 3.3188784539589817, + "tokens_seen": 3022258176 + }, + { + "epoch": 0.83, + "learning_rate": 8.567676767676768e-05, + "loss": 2.6153, + "theoretical_loss": 3.3188729289595273, + "tokens_seen": 3022323712 + }, + { + "epoch": 0.83, + "learning_rate": 8.565656565656565e-05, + "loss": 2.4774, + "theoretical_loss": 3.31886740411342, + "tokens_seen": 3022389248 + }, + { + "epoch": 0.83, + "learning_rate": 8.563636363636363e-05, + "loss": 2.6175, + "theoretical_loss": 3.318861879420652, + "tokens_seen": 3022454784 + }, + { + "epoch": 0.83, + "learning_rate": 8.561616161616162e-05, + "loss": 2.7685, + "theoretical_loss": 3.3188563548812158, + "tokens_seen": 3022520320 + }, + { + "epoch": 0.83, + "learning_rate": 8.55959595959596e-05, + "loss": 2.8493, + "theoretical_loss": 3.3188508304951037, + "tokens_seen": 3022585856 + }, + { + "epoch": 0.83, + "learning_rate": 8.557575757575757e-05, + "loss": 2.7809, + "theoretical_loss": 3.3188453062623084, + "tokens_seen": 3022651392 + }, + { + "epoch": 0.83, + "learning_rate": 8.555555555555556e-05, + "loss": 2.6795, + "theoretical_loss": 3.318839782182822, + "tokens_seen": 3022716928 + }, + { + "epoch": 0.83, + "learning_rate": 8.553535353535354e-05, + "loss": 2.2772, + "theoretical_loss": 3.3188342582566372, + "tokens_seen": 3022782464 + }, + { + "epoch": 0.83, + "learning_rate": 8.551515151515152e-05, + "loss": 2.5257, + "theoretical_loss": 3.318828734483746, + "tokens_seen": 3022848000 + }, + { + "epoch": 0.83, + "learning_rate": 8.54949494949495e-05, + "loss": 2.5851, + "theoretical_loss": 3.3188232108641413, + "tokens_seen": 3022913536 + }, + { + "epoch": 0.83, + "learning_rate": 8.547474747474748e-05, + "loss": 2.6957, + "theoretical_loss": 3.3188176873978152, + "tokens_seen": 3022979072 + }, + { + "epoch": 0.83, + "learning_rate": 8.545454545454545e-05, + "loss": 2.7445, + "theoretical_loss": 3.3188121640847608, + "tokens_seen": 3023044608 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1700711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63458514213562, + "objective/train/theoretical_loss": 3.3188066409249695, + "objective/train/tokens_used": 1381969376, + "theoretical_loss": 3.3188066409249695, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 8.543434343434343e-05, + "loss": 2.5916, + "theoretical_loss": 3.3188066409249695, + "tokens_seen": 3023110144 + }, + { + "epoch": 0.83, + "learning_rate": 8.541414141414142e-05, + "loss": 2.5556, + "theoretical_loss": 3.3188011179184342, + "tokens_seen": 3023175680 + }, + { + "epoch": 0.83, + "learning_rate": 8.53939393939394e-05, + "loss": 2.7329, + "theoretical_loss": 3.3187955950651475, + "tokens_seen": 3023241216 + }, + { + "epoch": 0.83, + "learning_rate": 8.537373737373738e-05, + "loss": 2.7724, + "theoretical_loss": 3.3187900723651014, + "tokens_seen": 3023306752 + }, + { + "epoch": 0.83, + "learning_rate": 8.535353535353535e-05, + "loss": 2.9677, + "theoretical_loss": 3.318784549818289, + "tokens_seen": 3023372288 + }, + { + "epoch": 0.83, + "learning_rate": 8.533333333333334e-05, + "loss": 2.4528, + "theoretical_loss": 3.318779027424702, + "tokens_seen": 3023437824 + }, + { + "epoch": 0.83, + "learning_rate": 8.531313131313131e-05, + "loss": 2.6669, + "theoretical_loss": 3.3187735051843332, + "tokens_seen": 3023503360 + }, + { + "epoch": 0.83, + "learning_rate": 8.529292929292929e-05, + "loss": 2.8167, + "theoretical_loss": 3.318767983097175, + "tokens_seen": 3023568896 + }, + { + "epoch": 0.83, + "learning_rate": 8.527272727272728e-05, + "loss": 2.4321, + "theoretical_loss": 3.31876246116322, + "tokens_seen": 3023634432 + }, + { + "epoch": 0.83, + "learning_rate": 8.525252525252525e-05, + "loss": 2.5491, + "theoretical_loss": 3.3187569393824603, + "tokens_seen": 3023699968 + }, + { + "epoch": 0.83, + "learning_rate": 8.523232323232323e-05, + "loss": 2.5753, + "theoretical_loss": 3.3187514177548882, + "tokens_seen": 3023765504 + }, + { + "epoch": 0.83, + "learning_rate": 8.521212121212122e-05, + "loss": 2.8227, + "theoretical_loss": 3.3187458962804968, + "tokens_seen": 3023831040 + }, + { + "epoch": 0.83, + "learning_rate": 8.51919191919192e-05, + "loss": 2.5716, + "theoretical_loss": 3.318740374959278, + "tokens_seen": 3023896576 + }, + { + "epoch": 0.83, + "learning_rate": 8.517171717171718e-05, + "loss": 2.6056, + "theoretical_loss": 3.318734853791224, + "tokens_seen": 3023962112 + }, + { + "epoch": 0.83, + "learning_rate": 8.515151515151514e-05, + "loss": 2.5715, + "theoretical_loss": 3.318729332776328, + "tokens_seen": 3024027648 + }, + { + "epoch": 0.83, + "learning_rate": 8.513131313131312e-05, + "loss": 2.5449, + "theoretical_loss": 3.318723811914582, + "tokens_seen": 3024093184 + }, + { + "epoch": 0.83, + "learning_rate": 8.511111111111111e-05, + "loss": 2.4991, + "theoretical_loss": 3.318718291205978, + "tokens_seen": 3024158720 + }, + { + "epoch": 0.83, + "learning_rate": 8.509090909090909e-05, + "loss": 2.5834, + "theoretical_loss": 3.3187127706505093, + "tokens_seen": 3024224256 + }, + { + "epoch": 0.83, + "learning_rate": 8.507070707070708e-05, + "loss": 2.6086, + "theoretical_loss": 3.318707250248168, + "tokens_seen": 3024289792 + }, + { + "epoch": 0.83, + "learning_rate": 8.505050505050506e-05, + "loss": 2.6224, + "theoretical_loss": 3.318701729998946, + "tokens_seen": 3024355328 + }, + { + "epoch": 0.83, + "learning_rate": 8.503030303030303e-05, + "loss": 2.7057, + "theoretical_loss": 3.3186962099028365, + "tokens_seen": 3024420864 + }, + { + "epoch": 0.83, + "learning_rate": 8.501010101010101e-05, + "loss": 2.4774, + "theoretical_loss": 3.3186906899598316, + "tokens_seen": 3024486400 + }, + { + "epoch": 0.83, + "learning_rate": 8.4989898989899e-05, + "loss": 2.6043, + "theoretical_loss": 3.3186851701699234, + "tokens_seen": 3024551936 + }, + { + "epoch": 0.83, + "learning_rate": 8.496969696969697e-05, + "loss": 2.4965, + "theoretical_loss": 3.318679650533105, + "tokens_seen": 3024617472 + }, + { + "epoch": 0.83, + "learning_rate": 8.494949494949495e-05, + "loss": 2.5179, + "theoretical_loss": 3.3186741310493684, + "tokens_seen": 3024683008 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1701440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.186544179916382, + "objective/train/theoretical_loss": 3.3186686117187065, + "objective/train/tokens_used": 1383607776, + "theoretical_loss": 3.3186686117187065, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.83, + "learning_rate": 8.492929292929292e-05, + "loss": 2.2874, + "theoretical_loss": 3.3186686117187065, + "tokens_seen": 3024748544 + }, + { + "epoch": 0.83, + "learning_rate": 8.490909090909091e-05, + "loss": 2.3675, + "theoretical_loss": 3.318663092541111, + "tokens_seen": 3024814080 + }, + { + "epoch": 0.83, + "learning_rate": 8.488888888888889e-05, + "loss": 2.773, + "theoretical_loss": 3.318657573516575, + "tokens_seen": 3024879616 + }, + { + "epoch": 0.83, + "learning_rate": 8.486868686868687e-05, + "loss": 2.7608, + "theoretical_loss": 3.318652054645091, + "tokens_seen": 3024945152 + }, + { + "epoch": 0.83, + "learning_rate": 8.484848484848486e-05, + "loss": 2.4817, + "theoretical_loss": 3.3186465359266504, + "tokens_seen": 3025010688 + }, + { + "epoch": 0.83, + "learning_rate": 8.482828282828283e-05, + "loss": 2.4771, + "theoretical_loss": 3.3186410173612466, + "tokens_seen": 3025076224 + }, + { + "epoch": 0.83, + "learning_rate": 8.480808080808081e-05, + "loss": 2.6178, + "theoretical_loss": 3.318635498948872, + "tokens_seen": 3025141760 + }, + { + "epoch": 0.83, + "learning_rate": 8.478787878787878e-05, + "loss": 2.64, + "theoretical_loss": 3.3186299806895185, + "tokens_seen": 3025207296 + }, + { + "epoch": 0.83, + "learning_rate": 8.476767676767677e-05, + "loss": 2.621, + "theoretical_loss": 3.3186244625831796, + "tokens_seen": 3025272832 + }, + { + "epoch": 0.83, + "learning_rate": 8.474747474747475e-05, + "loss": 2.4837, + "theoretical_loss": 3.318618944629846, + "tokens_seen": 3025338368 + }, + { + "epoch": 0.83, + "learning_rate": 8.472727272727272e-05, + "loss": 2.4917, + "theoretical_loss": 3.318613426829512, + "tokens_seen": 3025403904 + }, + { + "epoch": 0.83, + "learning_rate": 8.47070707070707e-05, + "loss": 2.6533, + "theoretical_loss": 3.318607909182169, + "tokens_seen": 3025469440 + }, + { + "epoch": 0.83, + "learning_rate": 8.468686868686869e-05, + "loss": 2.6522, + "theoretical_loss": 3.3186023916878096, + "tokens_seen": 3025534976 + }, + { + "epoch": 0.83, + "learning_rate": 8.466666666666667e-05, + "loss": 2.5398, + "theoretical_loss": 3.3185968743464263, + "tokens_seen": 3025600512 + }, + { + "epoch": 0.83, + "learning_rate": 8.464646464646466e-05, + "loss": 2.509, + "theoretical_loss": 3.3185913571580117, + "tokens_seen": 3025666048 + }, + { + "epoch": 0.83, + "learning_rate": 8.462626262626263e-05, + "loss": 2.6845, + "theoretical_loss": 3.318585840122558, + "tokens_seen": 3025731584 + }, + { + "epoch": 0.83, + "learning_rate": 8.46060606060606e-05, + "loss": 2.3774, + "theoretical_loss": 3.3185803232400577, + "tokens_seen": 3025797120 + }, + { + "epoch": 0.83, + "learning_rate": 8.458585858585858e-05, + "loss": 2.6245, + "theoretical_loss": 3.318574806510503, + "tokens_seen": 3025862656 + }, + { + "epoch": 0.83, + "learning_rate": 8.456565656565657e-05, + "loss": 2.681, + "theoretical_loss": 3.318569289933887, + "tokens_seen": 3025928192 + }, + { + "epoch": 0.83, + "learning_rate": 8.454545454545455e-05, + "loss": 2.596, + "theoretical_loss": 3.318563773510202, + "tokens_seen": 3025993728 + }, + { + "epoch": 0.83, + "learning_rate": 8.452525252525253e-05, + "loss": 2.8099, + "theoretical_loss": 3.31855825723944, + "tokens_seen": 3026059264 + }, + { + "epoch": 0.83, + "learning_rate": 8.45050505050505e-05, + "loss": 2.6033, + "theoretical_loss": 3.3185527411215934, + "tokens_seen": 3026124800 + }, + { + "epoch": 0.83, + "learning_rate": 8.448484848484849e-05, + "loss": 2.5939, + "theoretical_loss": 3.3185472251566552, + "tokens_seen": 3026190336 + }, + { + "epoch": 0.83, + "learning_rate": 8.446464646464647e-05, + "loss": 2.3916, + "theoretical_loss": 3.3185417093446175, + "tokens_seen": 3026255872 + }, + { + "epoch": 0.83, + "learning_rate": 8.444444444444444e-05, + "loss": 2.5476, + "theoretical_loss": 3.318536193685473, + "tokens_seen": 3026321408 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1702703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3933892250061035, + "objective/train/theoretical_loss": 3.3185306781792137, + "objective/train/tokens_used": 1385246176, + "theoretical_loss": 3.3185306781792137, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 8.442424242424243e-05, + "loss": 2.5266, + "theoretical_loss": 3.3185306781792137, + "tokens_seen": 3026386944 + }, + { + "epoch": 0.83, + "learning_rate": 8.44040404040404e-05, + "loss": 2.5144, + "theoretical_loss": 3.3185251628258325, + "tokens_seen": 3026452480 + }, + { + "epoch": 0.83, + "learning_rate": 8.438383838383838e-05, + "loss": 2.5478, + "theoretical_loss": 3.3185196476253216, + "tokens_seen": 3026518016 + }, + { + "epoch": 0.83, + "learning_rate": 8.436363636363637e-05, + "loss": 2.6449, + "theoretical_loss": 3.3185141325776737, + "tokens_seen": 3026583552 + }, + { + "epoch": 0.83, + "learning_rate": 8.434343434343435e-05, + "loss": 2.5168, + "theoretical_loss": 3.318508617682881, + "tokens_seen": 3026649088 + }, + { + "epoch": 0.83, + "learning_rate": 8.432323232323233e-05, + "loss": 2.6638, + "theoretical_loss": 3.318503102940936, + "tokens_seen": 3026714624 + }, + { + "epoch": 0.83, + "learning_rate": 8.43030303030303e-05, + "loss": 2.6438, + "theoretical_loss": 3.318497588351831, + "tokens_seen": 3026780160 + }, + { + "epoch": 0.83, + "learning_rate": 8.428282828282827e-05, + "loss": 2.5901, + "theoretical_loss": 3.318492073915559, + "tokens_seen": 3026845696 + }, + { + "epoch": 0.83, + "learning_rate": 8.426262626262626e-05, + "loss": 2.5537, + "theoretical_loss": 3.3184865596321123, + "tokens_seen": 3026911232 + }, + { + "epoch": 0.83, + "learning_rate": 8.424242424242424e-05, + "loss": 2.5831, + "theoretical_loss": 3.318481045501483, + "tokens_seen": 3026976768 + }, + { + "epoch": 0.83, + "learning_rate": 8.422222222222223e-05, + "loss": 2.4588, + "theoretical_loss": 3.3184755315236636, + "tokens_seen": 3027042304 + }, + { + "epoch": 0.83, + "learning_rate": 8.420202020202021e-05, + "loss": 2.3858, + "theoretical_loss": 3.3184700176986466, + "tokens_seen": 3027107840 + }, + { + "epoch": 0.83, + "learning_rate": 8.418181818181818e-05, + "loss": 2.7398, + "theoretical_loss": 3.3184645040264247, + "tokens_seen": 3027173376 + }, + { + "epoch": 0.83, + "learning_rate": 8.416161616161616e-05, + "loss": 2.5601, + "theoretical_loss": 3.31845899050699, + "tokens_seen": 3027238912 + }, + { + "epoch": 0.83, + "learning_rate": 8.414141414141415e-05, + "loss": 2.5592, + "theoretical_loss": 3.318453477140335, + "tokens_seen": 3027304448 + }, + { + "epoch": 0.83, + "learning_rate": 8.412121212121213e-05, + "loss": 2.3448, + "theoretical_loss": 3.318447963926453, + "tokens_seen": 3027369984 + }, + { + "epoch": 0.83, + "learning_rate": 8.41010101010101e-05, + "loss": 2.6943, + "theoretical_loss": 3.3184424508653354, + "tokens_seen": 3027435520 + }, + { + "epoch": 0.83, + "learning_rate": 8.408080808080807e-05, + "loss": 2.4965, + "theoretical_loss": 3.318436937956975, + "tokens_seen": 3027501056 + }, + { + "epoch": 0.83, + "learning_rate": 8.406060606060606e-05, + "loss": 2.4004, + "theoretical_loss": 3.3184314252013642, + "tokens_seen": 3027566592 + }, + { + "epoch": 0.83, + "learning_rate": 8.404040404040404e-05, + "loss": 2.6204, + "theoretical_loss": 3.318425912598496, + "tokens_seen": 3027632128 + }, + { + "epoch": 0.83, + "learning_rate": 8.402020202020202e-05, + "loss": 2.6509, + "theoretical_loss": 3.3184204001483617, + "tokens_seen": 3027697664 + }, + { + "epoch": 0.83, + "learning_rate": 8.400000000000001e-05, + "loss": 2.7037, + "theoretical_loss": 3.318414887850955, + "tokens_seen": 3027763200 + }, + { + "epoch": 0.83, + "learning_rate": 8.397979797979798e-05, + "loss": 2.7858, + "theoretical_loss": 3.318409375706268, + "tokens_seen": 3027828736 + }, + { + "epoch": 0.83, + "learning_rate": 8.395959595959596e-05, + "loss": 2.4572, + "theoretical_loss": 3.318403863714292, + "tokens_seen": 3027894272 + }, + { + "epoch": 0.83, + "learning_rate": 8.393939393939393e-05, + "loss": 2.774, + "theoretical_loss": 3.3183983518750217, + "tokens_seen": 3027959808 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1703090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.705465793609619, + "objective/train/theoretical_loss": 3.318392840188448, + "objective/train/tokens_used": 1386884576, + "theoretical_loss": 3.318392840188448, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.83, + "learning_rate": 8.391919191919192e-05, + "loss": 2.7257, + "theoretical_loss": 3.318392840188448, + "tokens_seen": 3028025344 + }, + { + "epoch": 0.83, + "learning_rate": 8.38989898989899e-05, + "loss": 2.5078, + "theoretical_loss": 3.3183873286545635, + "tokens_seen": 3028090880 + }, + { + "epoch": 0.83, + "learning_rate": 8.387878787878787e-05, + "loss": 2.362, + "theoretical_loss": 3.3183818172733606, + "tokens_seen": 3028156416 + }, + { + "epoch": 0.83, + "learning_rate": 8.385858585858586e-05, + "loss": 2.8107, + "theoretical_loss": 3.3183763060448324, + "tokens_seen": 3028221952 + }, + { + "epoch": 0.83, + "learning_rate": 8.383838383838384e-05, + "loss": 2.4942, + "theoretical_loss": 3.318370794968971, + "tokens_seen": 3028287488 + }, + { + "epoch": 0.83, + "learning_rate": 8.381818181818182e-05, + "loss": 2.7673, + "theoretical_loss": 3.3183652840457687, + "tokens_seen": 3028353024 + }, + { + "epoch": 0.83, + "learning_rate": 8.379797979797981e-05, + "loss": 2.7459, + "theoretical_loss": 3.318359773275218, + "tokens_seen": 3028418560 + }, + { + "epoch": 0.83, + "learning_rate": 8.377777777777779e-05, + "loss": 2.6376, + "theoretical_loss": 3.318354262657312, + "tokens_seen": 3028484096 + }, + { + "epoch": 0.83, + "learning_rate": 8.375757575757575e-05, + "loss": 2.6196, + "theoretical_loss": 3.3183487521920423, + "tokens_seen": 3028549632 + }, + { + "epoch": 0.83, + "learning_rate": 8.373737373737373e-05, + "loss": 2.4206, + "theoretical_loss": 3.318343241879402, + "tokens_seen": 3028615168 + }, + { + "epoch": 0.83, + "learning_rate": 8.371717171717172e-05, + "loss": 2.594, + "theoretical_loss": 3.318337731719383, + "tokens_seen": 3028680704 + }, + { + "epoch": 0.83, + "learning_rate": 8.36969696969697e-05, + "loss": 2.6484, + "theoretical_loss": 3.3183322217119784, + "tokens_seen": 3028746240 + }, + { + "epoch": 0.83, + "learning_rate": 8.367676767676768e-05, + "loss": 2.545, + "theoretical_loss": 3.31832671185718, + "tokens_seen": 3028811776 + }, + { + "epoch": 0.83, + "learning_rate": 8.365656565656565e-05, + "loss": 2.3922, + "theoretical_loss": 3.318321202154981, + "tokens_seen": 3028877312 + }, + { + "epoch": 0.83, + "learning_rate": 8.363636363636364e-05, + "loss": 2.5416, + "theoretical_loss": 3.318315692605373, + "tokens_seen": 3028942848 + }, + { + "epoch": 0.83, + "learning_rate": 8.361616161616162e-05, + "loss": 2.6237, + "theoretical_loss": 3.3183101832083497, + "tokens_seen": 3029008384 + }, + { + "epoch": 0.83, + "learning_rate": 8.359595959595961e-05, + "loss": 2.6432, + "theoretical_loss": 3.3183046739639024, + "tokens_seen": 3029073920 + }, + { + "epoch": 0.83, + "learning_rate": 8.357575757575758e-05, + "loss": 2.7266, + "theoretical_loss": 3.318299164872024, + "tokens_seen": 3029139456 + }, + { + "epoch": 0.83, + "learning_rate": 8.355555555555555e-05, + "loss": 2.5167, + "theoretical_loss": 3.318293655932707, + "tokens_seen": 3029204992 + }, + { + "epoch": 0.83, + "learning_rate": 8.353535353535353e-05, + "loss": 2.4784, + "theoretical_loss": 3.318288147145944, + "tokens_seen": 3029270528 + }, + { + "epoch": 0.83, + "learning_rate": 8.351515151515152e-05, + "loss": 2.4873, + "theoretical_loss": 3.318282638511727, + "tokens_seen": 3029336064 + }, + { + "epoch": 0.83, + "learning_rate": 8.34949494949495e-05, + "loss": 2.3333, + "theoretical_loss": 3.318277130030049, + "tokens_seen": 3029401600 + }, + { + "epoch": 0.83, + "learning_rate": 8.347474747474748e-05, + "loss": 2.558, + "theoretical_loss": 3.3182716217009025, + "tokens_seen": 3029467136 + }, + { + "epoch": 0.83, + "learning_rate": 8.345454545454545e-05, + "loss": 2.3986, + "theoretical_loss": 3.3182661135242797, + "tokens_seen": 3029532672 + }, + { + "epoch": 0.83, + "learning_rate": 8.343434343434344e-05, + "loss": 2.7775, + "theoretical_loss": 3.318260605500173, + "tokens_seen": 3029598208 + }, + { + "epoch": 0.83, + "objective/train/docs_used": 1704525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6479332447052, + "objective/train/theoretical_loss": 3.318255097628575, + "objective/train/tokens_used": 1388522976, + "theoretical_loss": 3.318255097628575, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.83, + "learning_rate": 8.341414141414141e-05, + "loss": 2.526, + "theoretical_loss": 3.318255097628575, + "tokens_seen": 3029663744 + }, + { + "epoch": 0.84, + "learning_rate": 8.339393939393939e-05, + "loss": 2.5772, + "theoretical_loss": 3.3182495899094784, + "tokens_seen": 3029729280 + }, + { + "epoch": 0.84, + "learning_rate": 8.337373737373738e-05, + "loss": 2.4118, + "theoretical_loss": 3.318244082342875, + "tokens_seen": 3029794816 + }, + { + "epoch": 0.84, + "learning_rate": 8.335353535353536e-05, + "loss": 2.5129, + "theoretical_loss": 3.3182385749287584, + "tokens_seen": 3029860352 + }, + { + "epoch": 0.84, + "learning_rate": 8.333333333333333e-05, + "loss": 2.4419, + "theoretical_loss": 3.31823306766712, + "tokens_seen": 3029925888 + }, + { + "epoch": 0.84, + "learning_rate": 8.331313131313131e-05, + "loss": 2.6318, + "theoretical_loss": 3.318227560557953, + "tokens_seen": 3029991424 + }, + { + "epoch": 0.84, + "learning_rate": 8.32929292929293e-05, + "loss": 2.5961, + "theoretical_loss": 3.318222053601249, + "tokens_seen": 3030056960 + }, + { + "epoch": 0.84, + "learning_rate": 8.327272727272728e-05, + "loss": 2.5536, + "theoretical_loss": 3.318216546797002, + "tokens_seen": 3030122496 + }, + { + "epoch": 0.84, + "learning_rate": 8.325252525252527e-05, + "loss": 2.5424, + "theoretical_loss": 3.3182110401452034, + "tokens_seen": 3030188032 + }, + { + "epoch": 0.84, + "learning_rate": 8.323232323232322e-05, + "loss": 2.5043, + "theoretical_loss": 3.318205533645845, + "tokens_seen": 3030253568 + }, + { + "epoch": 0.84, + "learning_rate": 8.321212121212121e-05, + "loss": 2.5164, + "theoretical_loss": 3.318200027298921, + "tokens_seen": 3030319104 + }, + { + "epoch": 0.84, + "learning_rate": 8.319191919191919e-05, + "loss": 2.6034, + "theoretical_loss": 3.318194521104423, + "tokens_seen": 3030384640 + }, + { + "epoch": 0.84, + "learning_rate": 8.317171717171718e-05, + "loss": 2.5615, + "theoretical_loss": 3.3181890150623428, + "tokens_seen": 3030450176 + }, + { + "epoch": 0.84, + "learning_rate": 8.315151515151516e-05, + "loss": 2.5625, + "theoretical_loss": 3.318183509172674, + "tokens_seen": 3030515712 + }, + { + "epoch": 0.84, + "learning_rate": 8.313131313131313e-05, + "loss": 2.6098, + "theoretical_loss": 3.318178003435409, + "tokens_seen": 3030581248 + }, + { + "epoch": 0.84, + "learning_rate": 8.311111111111111e-05, + "loss": 2.3908, + "theoretical_loss": 3.318172497850539, + "tokens_seen": 3030646784 + }, + { + "epoch": 0.84, + "learning_rate": 8.30909090909091e-05, + "loss": 2.2666, + "theoretical_loss": 3.3181669924180586, + "tokens_seen": 3030712320 + }, + { + "epoch": 0.84, + "learning_rate": 8.307070707070707e-05, + "loss": 2.5381, + "theoretical_loss": 3.3181614871379583, + "tokens_seen": 3030777856 + }, + { + "epoch": 0.84, + "learning_rate": 8.305050505050505e-05, + "loss": 2.4594, + "theoretical_loss": 3.3181559820102318, + "tokens_seen": 3030843392 + }, + { + "epoch": 0.84, + "learning_rate": 8.303030303030302e-05, + "loss": 2.5203, + "theoretical_loss": 3.3181504770348713, + "tokens_seen": 3030908928 + }, + { + "epoch": 0.84, + "learning_rate": 8.3010101010101e-05, + "loss": 2.7167, + "theoretical_loss": 3.318144972211869, + "tokens_seen": 3030974464 + }, + { + "epoch": 0.84, + "learning_rate": 8.298989898989899e-05, + "loss": 2.3916, + "theoretical_loss": 3.3181394675412177, + "tokens_seen": 3031040000 + }, + { + "epoch": 0.84, + "learning_rate": 8.296969696969697e-05, + "loss": 2.6935, + "theoretical_loss": 3.3181339630229094, + "tokens_seen": 3031105536 + }, + { + "epoch": 0.84, + "learning_rate": 8.294949494949496e-05, + "loss": 2.297, + "theoretical_loss": 3.3181284586569375, + "tokens_seen": 3031171072 + }, + { + "epoch": 0.84, + "learning_rate": 8.292929292929294e-05, + "loss": 2.5581, + "theoretical_loss": 3.3181229544432935, + "tokens_seen": 3031236608 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1705064, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.584144353866577, + "objective/train/theoretical_loss": 3.3181174503819704, + "objective/train/tokens_used": 1390161376, + "theoretical_loss": 3.3181174503819704, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.84, + "learning_rate": 8.290909090909091e-05, + "loss": 2.4238, + "theoretical_loss": 3.3181174503819704, + "tokens_seen": 3031302144 + }, + { + "epoch": 0.84, + "learning_rate": 8.288888888888888e-05, + "loss": 2.5201, + "theoretical_loss": 3.318111946472961, + "tokens_seen": 3031367680 + }, + { + "epoch": 0.84, + "learning_rate": 8.286868686868687e-05, + "loss": 2.6043, + "theoretical_loss": 3.3181064427162568, + "tokens_seen": 3031433216 + }, + { + "epoch": 0.84, + "learning_rate": 8.284848484848485e-05, + "loss": 2.5671, + "theoretical_loss": 3.3181009391118517, + "tokens_seen": 3031498752 + }, + { + "epoch": 0.84, + "learning_rate": 8.282828282828283e-05, + "loss": 2.5303, + "theoretical_loss": 3.318095435659737, + "tokens_seen": 3031564288 + }, + { + "epoch": 0.84, + "learning_rate": 8.28080808080808e-05, + "loss": 2.8076, + "theoretical_loss": 3.3180899323599053, + "tokens_seen": 3031629824 + }, + { + "epoch": 0.84, + "learning_rate": 8.278787878787879e-05, + "loss": 2.6933, + "theoretical_loss": 3.31808442921235, + "tokens_seen": 3031695360 + }, + { + "epoch": 0.84, + "learning_rate": 8.276767676767677e-05, + "loss": 2.4936, + "theoretical_loss": 3.318078926217063, + "tokens_seen": 3031760896 + }, + { + "epoch": 0.84, + "learning_rate": 8.274747474747476e-05, + "loss": 2.4298, + "theoretical_loss": 3.3180734233740363, + "tokens_seen": 3031826432 + }, + { + "epoch": 0.84, + "learning_rate": 8.272727272727274e-05, + "loss": 2.4845, + "theoretical_loss": 3.318067920683263, + "tokens_seen": 3031891968 + }, + { + "epoch": 0.84, + "learning_rate": 8.27070707070707e-05, + "loss": 2.7212, + "theoretical_loss": 3.3180624181447356, + "tokens_seen": 3031957504 + }, + { + "epoch": 0.84, + "learning_rate": 8.268686868686868e-05, + "loss": 2.4578, + "theoretical_loss": 3.3180569157584463, + "tokens_seen": 3032023040 + }, + { + "epoch": 0.84, + "learning_rate": 8.266666666666667e-05, + "loss": 2.5099, + "theoretical_loss": 3.318051413524388, + "tokens_seen": 3032088576 + }, + { + "epoch": 0.84, + "learning_rate": 8.264646464646465e-05, + "loss": 2.4264, + "theoretical_loss": 3.318045911442553, + "tokens_seen": 3032154112 + }, + { + "epoch": 0.84, + "learning_rate": 8.262626262626263e-05, + "loss": 2.5427, + "theoretical_loss": 3.3180404095129337, + "tokens_seen": 3032219648 + }, + { + "epoch": 0.84, + "learning_rate": 8.26060606060606e-05, + "loss": 2.4366, + "theoretical_loss": 3.3180349077355227, + "tokens_seen": 3032285184 + }, + { + "epoch": 0.84, + "learning_rate": 8.258585858585859e-05, + "loss": 2.4114, + "theoretical_loss": 3.3180294061103126, + "tokens_seen": 3032350720 + }, + { + "epoch": 0.84, + "learning_rate": 8.256565656565657e-05, + "loss": 2.3447, + "theoretical_loss": 3.3180239046372955, + "tokens_seen": 3032416256 + }, + { + "epoch": 0.84, + "learning_rate": 8.254545454545454e-05, + "loss": 2.5249, + "theoretical_loss": 3.318018403316464, + "tokens_seen": 3032481792 + }, + { + "epoch": 0.84, + "learning_rate": 8.252525252525253e-05, + "loss": 2.5557, + "theoretical_loss": 3.3180129021478115, + "tokens_seen": 3032547328 + }, + { + "epoch": 0.84, + "learning_rate": 8.250505050505051e-05, + "loss": 2.322, + "theoretical_loss": 3.318007401131329, + "tokens_seen": 3032612864 + }, + { + "epoch": 0.84, + "learning_rate": 8.248484848484848e-05, + "loss": 2.6336, + "theoretical_loss": 3.3180019002670105, + "tokens_seen": 3032678400 + }, + { + "epoch": 0.84, + "learning_rate": 8.246464646464646e-05, + "loss": 2.3663, + "theoretical_loss": 3.3179963995548474, + "tokens_seen": 3032743936 + }, + { + "epoch": 0.84, + "learning_rate": 8.244444444444445e-05, + "loss": 2.6781, + "theoretical_loss": 3.3179908989948323, + "tokens_seen": 3032809472 + }, + { + "epoch": 0.84, + "learning_rate": 8.242424242424243e-05, + "loss": 2.7119, + "theoretical_loss": 3.3179853985869587, + "tokens_seen": 3032875008 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1706237, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.271461248397827, + "objective/train/theoretical_loss": 3.317979898331218, + "objective/train/tokens_used": 1391799776, + "theoretical_loss": 3.317979898331218, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 8.240404040404042e-05, + "loss": 2.678, + "theoretical_loss": 3.317979898331218, + "tokens_seen": 3032940544 + }, + { + "epoch": 0.84, + "learning_rate": 8.238383838383839e-05, + "loss": 2.5511, + "theoretical_loss": 3.3179743982276033, + "tokens_seen": 3033006080 + }, + { + "epoch": 0.84, + "learning_rate": 8.236363636363636e-05, + "loss": 2.3092, + "theoretical_loss": 3.3179688982761064, + "tokens_seen": 3033071616 + }, + { + "epoch": 0.84, + "learning_rate": 8.234343434343434e-05, + "loss": 2.5649, + "theoretical_loss": 3.3179633984767207, + "tokens_seen": 3033137152 + }, + { + "epoch": 0.84, + "learning_rate": 8.232323232323233e-05, + "loss": 2.5283, + "theoretical_loss": 3.3179578988294383, + "tokens_seen": 3033202688 + }, + { + "epoch": 0.84, + "learning_rate": 8.230303030303031e-05, + "loss": 2.7465, + "theoretical_loss": 3.3179523993342515, + "tokens_seen": 3033268224 + }, + { + "epoch": 0.84, + "learning_rate": 8.228282828282828e-05, + "loss": 2.6406, + "theoretical_loss": 3.317946899991153, + "tokens_seen": 3033333760 + }, + { + "epoch": 0.84, + "learning_rate": 8.226262626262626e-05, + "loss": 2.6922, + "theoretical_loss": 3.317941400800136, + "tokens_seen": 3033399296 + }, + { + "epoch": 0.84, + "learning_rate": 8.224242424242425e-05, + "loss": 2.7303, + "theoretical_loss": 3.317935901761192, + "tokens_seen": 3033464832 + }, + { + "epoch": 0.84, + "learning_rate": 8.222222222222223e-05, + "loss": 2.5076, + "theoretical_loss": 3.3179304028743135, + "tokens_seen": 3033530368 + }, + { + "epoch": 0.84, + "learning_rate": 8.22020202020202e-05, + "loss": 2.697, + "theoretical_loss": 3.3179249041394936, + "tokens_seen": 3033595904 + }, + { + "epoch": 0.84, + "learning_rate": 8.218181818181817e-05, + "loss": 2.5992, + "theoretical_loss": 3.317919405556725, + "tokens_seen": 3033661440 + }, + { + "epoch": 0.84, + "learning_rate": 8.216161616161616e-05, + "loss": 2.5924, + "theoretical_loss": 3.317913907125999, + "tokens_seen": 3033726976 + }, + { + "epoch": 0.84, + "learning_rate": 8.214141414141414e-05, + "loss": 2.5201, + "theoretical_loss": 3.31790840884731, + "tokens_seen": 3033792512 + }, + { + "epoch": 0.84, + "learning_rate": 8.212121212121212e-05, + "loss": 2.5242, + "theoretical_loss": 3.3179029107206484, + "tokens_seen": 3033858048 + }, + { + "epoch": 0.84, + "learning_rate": 8.210101010101011e-05, + "loss": 2.8671, + "theoretical_loss": 3.317897412746008, + "tokens_seen": 3033923584 + }, + { + "epoch": 0.84, + "learning_rate": 8.208080808080809e-05, + "loss": 2.6112, + "theoretical_loss": 3.317891914923381, + "tokens_seen": 3033989120 + }, + { + "epoch": 0.84, + "learning_rate": 8.206060606060606e-05, + "loss": 2.5165, + "theoretical_loss": 3.31788641725276, + "tokens_seen": 3034054656 + }, + { + "epoch": 0.84, + "learning_rate": 8.204040404040405e-05, + "loss": 2.4512, + "theoretical_loss": 3.3178809197341375, + "tokens_seen": 3034120192 + }, + { + "epoch": 0.84, + "learning_rate": 8.202020202020202e-05, + "loss": 2.4212, + "theoretical_loss": 3.317875422367506, + "tokens_seen": 3034185728 + }, + { + "epoch": 0.84, + "learning_rate": 8.2e-05, + "loss": 2.7839, + "theoretical_loss": 3.3178699251528583, + "tokens_seen": 3034251264 + }, + { + "epoch": 0.84, + "learning_rate": 8.197979797979798e-05, + "loss": 2.5307, + "theoretical_loss": 3.3178644280901866, + "tokens_seen": 3034316800 + }, + { + "epoch": 0.84, + "learning_rate": 8.195959595959596e-05, + "loss": 2.511, + "theoretical_loss": 3.3178589311794826, + "tokens_seen": 3034382336 + }, + { + "epoch": 0.84, + "learning_rate": 8.193939393939394e-05, + "loss": 2.9113, + "theoretical_loss": 3.3178534344207407, + "tokens_seen": 3034447872 + }, + { + "epoch": 0.84, + "learning_rate": 8.191919191919192e-05, + "loss": 2.7979, + "theoretical_loss": 3.3178479378139514, + "tokens_seen": 3034513408 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1706768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.531189441680908, + "objective/train/theoretical_loss": 3.317842441359109, + "objective/train/tokens_used": 1393438176, + "theoretical_loss": 3.317842441359109, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.84, + "learning_rate": 8.189898989898991e-05, + "loss": 2.6867, + "theoretical_loss": 3.317842441359109, + "tokens_seen": 3034578944 + }, + { + "epoch": 0.84, + "learning_rate": 8.187878787878789e-05, + "loss": 2.5384, + "theoretical_loss": 3.3178369450562046, + "tokens_seen": 3034644480 + }, + { + "epoch": 0.84, + "learning_rate": 8.185858585858585e-05, + "loss": 2.4298, + "theoretical_loss": 3.3178314489052316, + "tokens_seen": 3034710016 + }, + { + "epoch": 0.84, + "learning_rate": 8.183838383838383e-05, + "loss": 2.6789, + "theoretical_loss": 3.3178259529061824, + "tokens_seen": 3034775552 + }, + { + "epoch": 0.84, + "learning_rate": 8.181818181818182e-05, + "loss": 2.7035, + "theoretical_loss": 3.317820457059049, + "tokens_seen": 3034841088 + }, + { + "epoch": 0.84, + "learning_rate": 8.17979797979798e-05, + "loss": 2.5605, + "theoretical_loss": 3.3178149613638244, + "tokens_seen": 3034906624 + }, + { + "epoch": 0.84, + "learning_rate": 8.177777777777778e-05, + "loss": 2.6074, + "theoretical_loss": 3.3178094658205013, + "tokens_seen": 3034972160 + }, + { + "epoch": 0.84, + "learning_rate": 8.175757575757575e-05, + "loss": 2.756, + "theoretical_loss": 3.3178039704290714, + "tokens_seen": 3035037696 + }, + { + "epoch": 0.84, + "learning_rate": 8.173737373737374e-05, + "loss": 2.6546, + "theoretical_loss": 3.3177984751895284, + "tokens_seen": 3035103232 + }, + { + "epoch": 0.84, + "learning_rate": 8.171717171717172e-05, + "loss": 2.4029, + "theoretical_loss": 3.3177929801018635, + "tokens_seen": 3035168768 + }, + { + "epoch": 0.84, + "learning_rate": 8.16969696969697e-05, + "loss": 2.7972, + "theoretical_loss": 3.31778748516607, + "tokens_seen": 3035234304 + }, + { + "epoch": 0.84, + "learning_rate": 8.167676767676768e-05, + "loss": 2.6235, + "theoretical_loss": 3.317781990382141, + "tokens_seen": 3035299840 + }, + { + "epoch": 0.84, + "learning_rate": 8.165656565656566e-05, + "loss": 2.6238, + "theoretical_loss": 3.3177764957500675, + "tokens_seen": 3035365376 + }, + { + "epoch": 0.84, + "learning_rate": 8.163636363636363e-05, + "loss": 2.4335, + "theoretical_loss": 3.3177710012698434, + "tokens_seen": 3035430912 + }, + { + "epoch": 0.84, + "learning_rate": 8.161616161616161e-05, + "loss": 2.6599, + "theoretical_loss": 3.31776550694146, + "tokens_seen": 3035496448 + }, + { + "epoch": 0.84, + "learning_rate": 8.15959595959596e-05, + "loss": 2.8747, + "theoretical_loss": 3.317760012764911, + "tokens_seen": 3035561984 + }, + { + "epoch": 0.84, + "learning_rate": 8.157575757575758e-05, + "loss": 2.6931, + "theoretical_loss": 3.3177545187401885, + "tokens_seen": 3035627520 + }, + { + "epoch": 0.84, + "learning_rate": 8.155555555555557e-05, + "loss": 2.6029, + "theoretical_loss": 3.317749024867285, + "tokens_seen": 3035693056 + }, + { + "epoch": 0.84, + "learning_rate": 8.153535353535354e-05, + "loss": 2.534, + "theoretical_loss": 3.317743531146193, + "tokens_seen": 3035758592 + }, + { + "epoch": 0.84, + "learning_rate": 8.151515151515152e-05, + "loss": 2.7106, + "theoretical_loss": 3.317738037576905, + "tokens_seen": 3035824128 + }, + { + "epoch": 0.84, + "learning_rate": 8.149494949494949e-05, + "loss": 2.7623, + "theoretical_loss": 3.3177325441594134, + "tokens_seen": 3035889664 + }, + { + "epoch": 0.84, + "learning_rate": 8.147474747474748e-05, + "loss": 2.6049, + "theoretical_loss": 3.317727050893711, + "tokens_seen": 3035955200 + }, + { + "epoch": 0.84, + "learning_rate": 8.145454545454546e-05, + "loss": 2.7854, + "theoretical_loss": 3.31772155777979, + "tokens_seen": 3036020736 + }, + { + "epoch": 0.84, + "learning_rate": 8.143434343434343e-05, + "loss": 2.5349, + "theoretical_loss": 3.317716064817643, + "tokens_seen": 3036086272 + }, + { + "epoch": 0.84, + "learning_rate": 8.141414141414141e-05, + "loss": 2.4364, + "theoretical_loss": 3.3177105720072633, + "tokens_seen": 3036151808 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1707993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.661835193634033, + "objective/train/theoretical_loss": 3.3177050793486424, + "objective/train/tokens_used": 1395076576, + "theoretical_loss": 3.3177050793486424, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 8.13939393939394e-05, + "loss": 2.6449, + "theoretical_loss": 3.3177050793486424, + "tokens_seen": 3036217344 + }, + { + "epoch": 0.84, + "learning_rate": 8.137373737373738e-05, + "loss": 2.4973, + "theoretical_loss": 3.317699586841773, + "tokens_seen": 3036282880 + }, + { + "epoch": 0.84, + "learning_rate": 8.135353535353537e-05, + "loss": 2.5195, + "theoretical_loss": 3.317694094486648, + "tokens_seen": 3036348416 + }, + { + "epoch": 0.84, + "learning_rate": 8.133333333333332e-05, + "loss": 2.5571, + "theoretical_loss": 3.31768860228326, + "tokens_seen": 3036413952 + }, + { + "epoch": 0.84, + "learning_rate": 8.13131313131313e-05, + "loss": 2.5733, + "theoretical_loss": 3.3176831102316013, + "tokens_seen": 3036479488 + }, + { + "epoch": 0.84, + "learning_rate": 8.129292929292929e-05, + "loss": 2.6334, + "theoretical_loss": 3.3176776183316647, + "tokens_seen": 3036545024 + }, + { + "epoch": 0.84, + "learning_rate": 8.127272727272727e-05, + "loss": 2.5757, + "theoretical_loss": 3.3176721265834423, + "tokens_seen": 3036610560 + }, + { + "epoch": 0.84, + "learning_rate": 8.125252525252526e-05, + "loss": 2.615, + "theoretical_loss": 3.3176666349869266, + "tokens_seen": 3036676096 + }, + { + "epoch": 0.84, + "learning_rate": 8.123232323232324e-05, + "loss": 2.5938, + "theoretical_loss": 3.3176611435421104, + "tokens_seen": 3036741632 + }, + { + "epoch": 0.84, + "learning_rate": 8.121212121212121e-05, + "loss": 2.6904, + "theoretical_loss": 3.317655652248986, + "tokens_seen": 3036807168 + }, + { + "epoch": 0.84, + "learning_rate": 8.11919191919192e-05, + "loss": 2.5967, + "theoretical_loss": 3.317650161107547, + "tokens_seen": 3036872704 + }, + { + "epoch": 0.84, + "learning_rate": 8.117171717171718e-05, + "loss": 2.6133, + "theoretical_loss": 3.3176446701177844, + "tokens_seen": 3036938240 + }, + { + "epoch": 0.84, + "learning_rate": 8.115151515151515e-05, + "loss": 2.6941, + "theoretical_loss": 3.3176391792796913, + "tokens_seen": 3037003776 + }, + { + "epoch": 0.84, + "learning_rate": 8.113131313131314e-05, + "loss": 2.6196, + "theoretical_loss": 3.3176336885932605, + "tokens_seen": 3037069312 + }, + { + "epoch": 0.84, + "learning_rate": 8.11111111111111e-05, + "loss": 2.5724, + "theoretical_loss": 3.3176281980584847, + "tokens_seen": 3037134848 + }, + { + "epoch": 0.84, + "learning_rate": 8.109090909090909e-05, + "loss": 2.5532, + "theoretical_loss": 3.3176227076753557, + "tokens_seen": 3037200384 + }, + { + "epoch": 0.84, + "learning_rate": 8.107070707070707e-05, + "loss": 2.4817, + "theoretical_loss": 3.3176172174438667, + "tokens_seen": 3037265920 + }, + { + "epoch": 0.84, + "learning_rate": 8.105050505050506e-05, + "loss": 2.6669, + "theoretical_loss": 3.31761172736401, + "tokens_seen": 3037331456 + }, + { + "epoch": 0.84, + "learning_rate": 8.103030303030304e-05, + "loss": 2.6822, + "theoretical_loss": 3.317606237435778, + "tokens_seen": 3037396992 + }, + { + "epoch": 0.84, + "learning_rate": 8.101010101010101e-05, + "loss": 2.5076, + "theoretical_loss": 3.3176007476591636, + "tokens_seen": 3037462528 + }, + { + "epoch": 0.84, + "learning_rate": 8.098989898989898e-05, + "loss": 2.4951, + "theoretical_loss": 3.3175952580341592, + "tokens_seen": 3037528064 + }, + { + "epoch": 0.84, + "learning_rate": 8.096969696969697e-05, + "loss": 2.5275, + "theoretical_loss": 3.317589768560757, + "tokens_seen": 3037593600 + }, + { + "epoch": 0.84, + "learning_rate": 8.094949494949495e-05, + "loss": 2.4476, + "theoretical_loss": 3.3175842792389503, + "tokens_seen": 3037659136 + }, + { + "epoch": 0.84, + "learning_rate": 8.092929292929293e-05, + "loss": 2.3959, + "theoretical_loss": 3.3175787900687306, + "tokens_seen": 3037724672 + }, + { + "epoch": 0.84, + "learning_rate": 8.09090909090909e-05, + "loss": 2.5584, + "theoretical_loss": 3.317573301050091, + "tokens_seen": 3037790208 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1708712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.502825975418091, + "objective/train/theoretical_loss": 3.3175678121830243, + "objective/train/tokens_used": 1396714976, + "theoretical_loss": 3.3175678121830243, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.84, + "learning_rate": 8.088888888888889e-05, + "loss": 2.6009, + "theoretical_loss": 3.3175678121830243, + "tokens_seen": 3037855744 + }, + { + "epoch": 0.84, + "learning_rate": 8.086868686868687e-05, + "loss": 2.445, + "theoretical_loss": 3.3175623234675227, + "tokens_seen": 3037921280 + }, + { + "epoch": 0.84, + "learning_rate": 8.084848484848486e-05, + "loss": 2.688, + "theoretical_loss": 3.317556834903579, + "tokens_seen": 3037986816 + }, + { + "epoch": 0.84, + "learning_rate": 8.082828282828284e-05, + "loss": 2.6468, + "theoretical_loss": 3.3175513464911854, + "tokens_seen": 3038052352 + }, + { + "epoch": 0.84, + "learning_rate": 8.080808080808081e-05, + "loss": 2.4911, + "theoretical_loss": 3.3175458582303348, + "tokens_seen": 3038117888 + }, + { + "epoch": 0.84, + "learning_rate": 8.078787878787878e-05, + "loss": 2.6676, + "theoretical_loss": 3.3175403701210193, + "tokens_seen": 3038183424 + }, + { + "epoch": 0.84, + "learning_rate": 8.076767676767677e-05, + "loss": 2.6016, + "theoretical_loss": 3.317534882163232, + "tokens_seen": 3038248960 + }, + { + "epoch": 0.84, + "learning_rate": 8.074747474747475e-05, + "loss": 2.5812, + "theoretical_loss": 3.317529394356965, + "tokens_seen": 3038314496 + }, + { + "epoch": 0.84, + "learning_rate": 8.072727272727273e-05, + "loss": 2.8527, + "theoretical_loss": 3.317523906702211, + "tokens_seen": 3038380032 + }, + { + "epoch": 0.84, + "learning_rate": 8.070707070707072e-05, + "loss": 2.4579, + "theoretical_loss": 3.3175184191989624, + "tokens_seen": 3038445568 + }, + { + "epoch": 0.84, + "learning_rate": 8.068686868686869e-05, + "loss": 2.8028, + "theoretical_loss": 3.3175129318472125, + "tokens_seen": 3038511104 + }, + { + "epoch": 0.84, + "learning_rate": 8.066666666666667e-05, + "loss": 2.6027, + "theoretical_loss": 3.3175074446469526, + "tokens_seen": 3038576640 + }, + { + "epoch": 0.84, + "learning_rate": 8.064646464646464e-05, + "loss": 2.3729, + "theoretical_loss": 3.3175019575981763, + "tokens_seen": 3038642176 + }, + { + "epoch": 0.84, + "learning_rate": 8.062626262626263e-05, + "loss": 2.5786, + "theoretical_loss": 3.317496470700876, + "tokens_seen": 3038707712 + }, + { + "epoch": 0.84, + "learning_rate": 8.060606060606061e-05, + "loss": 2.7852, + "theoretical_loss": 3.3174909839550435, + "tokens_seen": 3038773248 + }, + { + "epoch": 0.84, + "learning_rate": 8.058585858585858e-05, + "loss": 2.3666, + "theoretical_loss": 3.317485497360672, + "tokens_seen": 3038838784 + }, + { + "epoch": 0.84, + "learning_rate": 8.056565656565656e-05, + "loss": 2.6361, + "theoretical_loss": 3.317480010917754, + "tokens_seen": 3038904320 + }, + { + "epoch": 0.84, + "learning_rate": 8.054545454545455e-05, + "loss": 2.73, + "theoretical_loss": 3.3174745246262822, + "tokens_seen": 3038969856 + }, + { + "epoch": 0.84, + "learning_rate": 8.052525252525253e-05, + "loss": 2.5212, + "theoretical_loss": 3.3174690384862484, + "tokens_seen": 3039035392 + }, + { + "epoch": 0.84, + "learning_rate": 8.050505050505052e-05, + "loss": 2.6876, + "theoretical_loss": 3.317463552497646, + "tokens_seen": 3039100928 + }, + { + "epoch": 0.84, + "learning_rate": 8.048484848484849e-05, + "loss": 2.6463, + "theoretical_loss": 3.3174580666604676, + "tokens_seen": 3039166464 + }, + { + "epoch": 0.84, + "learning_rate": 8.046464646464646e-05, + "loss": 2.5282, + "theoretical_loss": 3.317452580974705, + "tokens_seen": 3039232000 + }, + { + "epoch": 0.84, + "learning_rate": 8.044444444444444e-05, + "loss": 2.5404, + "theoretical_loss": 3.317447095440351, + "tokens_seen": 3039297536 + }, + { + "epoch": 0.84, + "learning_rate": 8.042424242424242e-05, + "loss": 2.4653, + "theoretical_loss": 3.3174416100573985, + "tokens_seen": 3039363072 + }, + { + "epoch": 0.84, + "learning_rate": 8.040404040404041e-05, + "loss": 2.7005, + "theoretical_loss": 3.31743612482584, + "tokens_seen": 3039428608 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1709972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3593599796295166, + "objective/train/theoretical_loss": 3.317430639745668, + "objective/train/tokens_used": 1398353376, + "theoretical_loss": 3.317430639745668, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 8.038383838383839e-05, + "loss": 2.7266, + "theoretical_loss": 3.317430639745668, + "tokens_seen": 3039494144 + }, + { + "epoch": 0.84, + "learning_rate": 8.036363636363636e-05, + "loss": 2.6208, + "theoretical_loss": 3.3174251548168745, + "tokens_seen": 3039559680 + }, + { + "epoch": 0.84, + "learning_rate": 8.034343434343435e-05, + "loss": 2.5744, + "theoretical_loss": 3.317419670039453, + "tokens_seen": 3039625216 + }, + { + "epoch": 0.84, + "learning_rate": 8.032323232323233e-05, + "loss": 2.5502, + "theoretical_loss": 3.317414185413395, + "tokens_seen": 3039690752 + }, + { + "epoch": 0.84, + "learning_rate": 8.030303030303031e-05, + "loss": 2.6725, + "theoretical_loss": 3.317408700938694, + "tokens_seen": 3039756288 + }, + { + "epoch": 0.84, + "learning_rate": 8.028282828282829e-05, + "loss": 2.5471, + "theoretical_loss": 3.3174032166153427, + "tokens_seen": 3039821824 + }, + { + "epoch": 0.84, + "learning_rate": 8.026262626262626e-05, + "loss": 2.6118, + "theoretical_loss": 3.317397732443333, + "tokens_seen": 3039887360 + }, + { + "epoch": 0.84, + "learning_rate": 8.024242424242424e-05, + "loss": 2.596, + "theoretical_loss": 3.317392248422657, + "tokens_seen": 3039952896 + }, + { + "epoch": 0.84, + "learning_rate": 8.022222222222222e-05, + "loss": 2.6546, + "theoretical_loss": 3.317386764553308, + "tokens_seen": 3040018432 + }, + { + "epoch": 0.84, + "learning_rate": 8.020202020202021e-05, + "loss": 2.7984, + "theoretical_loss": 3.317381280835279, + "tokens_seen": 3040083968 + }, + { + "epoch": 0.84, + "learning_rate": 8.018181818181819e-05, + "loss": 2.6726, + "theoretical_loss": 3.317375797268562, + "tokens_seen": 3040149504 + }, + { + "epoch": 0.84, + "learning_rate": 8.016161616161616e-05, + "loss": 2.6579, + "theoretical_loss": 3.317370313853149, + "tokens_seen": 3040215040 + }, + { + "epoch": 0.84, + "learning_rate": 8.014141414141415e-05, + "loss": 2.5698, + "theoretical_loss": 3.317364830589034, + "tokens_seen": 3040280576 + }, + { + "epoch": 0.84, + "learning_rate": 8.012121212121212e-05, + "loss": 2.496, + "theoretical_loss": 3.317359347476208, + "tokens_seen": 3040346112 + }, + { + "epoch": 0.84, + "learning_rate": 8.01010101010101e-05, + "loss": 2.6022, + "theoretical_loss": 3.3173538645146645, + "tokens_seen": 3040411648 + }, + { + "epoch": 0.84, + "learning_rate": 8.008080808080808e-05, + "loss": 2.6004, + "theoretical_loss": 3.317348381704396, + "tokens_seen": 3040477184 + }, + { + "epoch": 0.84, + "learning_rate": 8.006060606060605e-05, + "loss": 2.5477, + "theoretical_loss": 3.3173428990453946, + "tokens_seen": 3040542720 + }, + { + "epoch": 0.84, + "learning_rate": 8.004040404040404e-05, + "loss": 2.6182, + "theoretical_loss": 3.3173374165376535, + "tokens_seen": 3040608256 + }, + { + "epoch": 0.84, + "learning_rate": 8.002020202020202e-05, + "loss": 2.6867, + "theoretical_loss": 3.3173319341811647, + "tokens_seen": 3040673792 + }, + { + "epoch": 0.84, + "learning_rate": 8e-05, + "loss": 2.625, + "theoretical_loss": 3.317326451975921, + "tokens_seen": 3040739328 + }, + { + "epoch": 0.84, + "learning_rate": 7.997979797979799e-05, + "loss": 2.6289, + "theoretical_loss": 3.3173209699219153, + "tokens_seen": 3040804864 + }, + { + "epoch": 0.84, + "learning_rate": 7.995959595959596e-05, + "loss": 2.6974, + "theoretical_loss": 3.3173154880191396, + "tokens_seen": 3040870400 + }, + { + "epoch": 0.84, + "learning_rate": 7.993939393939393e-05, + "loss": 2.4188, + "theoretical_loss": 3.317310006267587, + "tokens_seen": 3040935936 + }, + { + "epoch": 0.84, + "learning_rate": 7.991919191919192e-05, + "loss": 2.5501, + "theoretical_loss": 3.3173045246672492, + "tokens_seen": 3041001472 + }, + { + "epoch": 0.84, + "learning_rate": 7.98989898989899e-05, + "loss": 2.6467, + "theoretical_loss": 3.31729904321812, + "tokens_seen": 3041067008 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1710687, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6677358150482178, + "objective/train/theoretical_loss": 3.317293561920191, + "objective/train/tokens_used": 1399991776, + "theoretical_loss": 3.317293561920191, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.84, + "learning_rate": 7.987878787878788e-05, + "loss": 2.5044, + "theoretical_loss": 3.317293561920191, + "tokens_seen": 3041132544 + }, + { + "epoch": 0.84, + "learning_rate": 7.985858585858587e-05, + "loss": 2.8835, + "theoretical_loss": 3.317288080773455, + "tokens_seen": 3041198080 + }, + { + "epoch": 0.84, + "learning_rate": 7.983838383838384e-05, + "loss": 2.6893, + "theoretical_loss": 3.317282599777905, + "tokens_seen": 3041263616 + }, + { + "epoch": 0.84, + "learning_rate": 7.981818181818182e-05, + "loss": 2.6916, + "theoretical_loss": 3.317277118933533, + "tokens_seen": 3041329152 + }, + { + "epoch": 0.84, + "learning_rate": 7.97979797979798e-05, + "loss": 2.6527, + "theoretical_loss": 3.317271638240332, + "tokens_seen": 3041394688 + }, + { + "epoch": 0.84, + "learning_rate": 7.977777777777778e-05, + "loss": 2.6511, + "theoretical_loss": 3.317266157698294, + "tokens_seen": 3041460224 + }, + { + "epoch": 0.84, + "learning_rate": 7.975757575757576e-05, + "loss": 2.3888, + "theoretical_loss": 3.3172606773074125, + "tokens_seen": 3041525760 + }, + { + "epoch": 0.84, + "learning_rate": 7.973737373737373e-05, + "loss": 2.4216, + "theoretical_loss": 3.317255197067679, + "tokens_seen": 3041591296 + }, + { + "epoch": 0.84, + "learning_rate": 7.971717171717171e-05, + "loss": 2.4979, + "theoretical_loss": 3.317249716979087, + "tokens_seen": 3041656832 + }, + { + "epoch": 0.84, + "learning_rate": 7.96969696969697e-05, + "loss": 2.5305, + "theoretical_loss": 3.3172442370416286, + "tokens_seen": 3041722368 + }, + { + "epoch": 0.84, + "learning_rate": 7.967676767676768e-05, + "loss": 2.546, + "theoretical_loss": 3.3172387572552964, + "tokens_seen": 3041787904 + }, + { + "epoch": 0.84, + "learning_rate": 7.965656565656567e-05, + "loss": 2.5425, + "theoretical_loss": 3.3172332776200832, + "tokens_seen": 3041853440 + }, + { + "epoch": 0.84, + "learning_rate": 7.963636363636364e-05, + "loss": 2.4163, + "theoretical_loss": 3.3172277981359812, + "tokens_seen": 3041918976 + }, + { + "epoch": 0.84, + "learning_rate": 7.961616161616162e-05, + "loss": 2.6414, + "theoretical_loss": 3.3172223188029832, + "tokens_seen": 3041984512 + }, + { + "epoch": 0.84, + "learning_rate": 7.959595959595959e-05, + "loss": 2.3899, + "theoretical_loss": 3.317216839621082, + "tokens_seen": 3042050048 + }, + { + "epoch": 0.84, + "learning_rate": 7.957575757575757e-05, + "loss": 2.5308, + "theoretical_loss": 3.31721136059027, + "tokens_seen": 3042115584 + }, + { + "epoch": 0.84, + "learning_rate": 7.955555555555556e-05, + "loss": 2.4359, + "theoretical_loss": 3.317205881710539, + "tokens_seen": 3042181120 + }, + { + "epoch": 0.84, + "learning_rate": 7.953535353535353e-05, + "loss": 2.4597, + "theoretical_loss": 3.317200402981883, + "tokens_seen": 3042246656 + }, + { + "epoch": 0.84, + "learning_rate": 7.951515151515151e-05, + "loss": 2.4629, + "theoretical_loss": 3.3171949244042938, + "tokens_seen": 3042312192 + }, + { + "epoch": 0.84, + "learning_rate": 7.94949494949495e-05, + "loss": 2.6572, + "theoretical_loss": 3.317189445977764, + "tokens_seen": 3042377728 + }, + { + "epoch": 0.84, + "learning_rate": 7.947474747474748e-05, + "loss": 2.4836, + "theoretical_loss": 3.3171839677022863, + "tokens_seen": 3042443264 + }, + { + "epoch": 0.84, + "learning_rate": 7.945454545454547e-05, + "loss": 2.5328, + "theoretical_loss": 3.317178489577853, + "tokens_seen": 3042508800 + }, + { + "epoch": 0.84, + "learning_rate": 7.943434343434345e-05, + "loss": 2.5177, + "theoretical_loss": 3.3171730116044573, + "tokens_seen": 3042574336 + }, + { + "epoch": 0.84, + "learning_rate": 7.94141414141414e-05, + "loss": 2.5054, + "theoretical_loss": 3.317167533782091, + "tokens_seen": 3042639872 + }, + { + "epoch": 0.84, + "learning_rate": 7.939393939393939e-05, + "loss": 2.6175, + "theoretical_loss": 3.3171620561107473, + "tokens_seen": 3042705408 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1712243, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4307212829589844, + "objective/train/theoretical_loss": 3.3171565785904185, + "objective/train/tokens_used": 1401630176, + "theoretical_loss": 3.3171565785904185, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 7.937373737373737e-05, + "loss": 2.5112, + "theoretical_loss": 3.3171565785904185, + "tokens_seen": 3042770944 + }, + { + "epoch": 0.84, + "learning_rate": 7.935353535353536e-05, + "loss": 2.6723, + "theoretical_loss": 3.3171511012210972, + "tokens_seen": 3042836480 + }, + { + "epoch": 0.84, + "learning_rate": 7.933333333333334e-05, + "loss": 2.7671, + "theoretical_loss": 3.317145624002776, + "tokens_seen": 3042902016 + }, + { + "epoch": 0.84, + "learning_rate": 7.931313131313131e-05, + "loss": 2.3663, + "theoretical_loss": 3.317140146935448, + "tokens_seen": 3042967552 + }, + { + "epoch": 0.84, + "learning_rate": 7.92929292929293e-05, + "loss": 2.6252, + "theoretical_loss": 3.317134670019105, + "tokens_seen": 3043033088 + }, + { + "epoch": 0.84, + "learning_rate": 7.927272727272728e-05, + "loss": 2.477, + "theoretical_loss": 3.3171291932537397, + "tokens_seen": 3043098624 + }, + { + "epoch": 0.84, + "learning_rate": 7.925252525252525e-05, + "loss": 2.7008, + "theoretical_loss": 3.317123716639345, + "tokens_seen": 3043164160 + }, + { + "epoch": 0.84, + "learning_rate": 7.923232323232323e-05, + "loss": 2.6423, + "theoretical_loss": 3.317118240175913, + "tokens_seen": 3043229696 + }, + { + "epoch": 0.84, + "learning_rate": 7.92121212121212e-05, + "loss": 2.5266, + "theoretical_loss": 3.3171127638634372, + "tokens_seen": 3043295232 + }, + { + "epoch": 0.84, + "learning_rate": 7.919191919191919e-05, + "loss": 2.5244, + "theoretical_loss": 3.3171072877019094, + "tokens_seen": 3043360768 + }, + { + "epoch": 0.84, + "learning_rate": 7.917171717171717e-05, + "loss": 2.6004, + "theoretical_loss": 3.3171018116913222, + "tokens_seen": 3043426304 + }, + { + "epoch": 0.84, + "learning_rate": 7.915151515151516e-05, + "loss": 2.4859, + "theoretical_loss": 3.317096335831669, + "tokens_seen": 3043491840 + }, + { + "epoch": 0.84, + "learning_rate": 7.913131313131314e-05, + "loss": 2.5797, + "theoretical_loss": 3.317090860122941, + "tokens_seen": 3043557376 + }, + { + "epoch": 0.84, + "learning_rate": 7.911111111111111e-05, + "loss": 2.6738, + "theoretical_loss": 3.3170853845651322, + "tokens_seen": 3043622912 + }, + { + "epoch": 0.84, + "learning_rate": 7.90909090909091e-05, + "loss": 2.6305, + "theoretical_loss": 3.3170799091582346, + "tokens_seen": 3043688448 + }, + { + "epoch": 0.84, + "learning_rate": 7.907070707070707e-05, + "loss": 2.5813, + "theoretical_loss": 3.3170744339022407, + "tokens_seen": 3043753984 + }, + { + "epoch": 0.84, + "learning_rate": 7.905050505050505e-05, + "loss": 2.3729, + "theoretical_loss": 3.317068958797143, + "tokens_seen": 3043819520 + }, + { + "epoch": 0.84, + "learning_rate": 7.903030303030303e-05, + "loss": 2.4609, + "theoretical_loss": 3.3170634838429347, + "tokens_seen": 3043885056 + }, + { + "epoch": 0.84, + "learning_rate": 7.901010101010102e-05, + "loss": 2.6216, + "theoretical_loss": 3.3170580090396076, + "tokens_seen": 3043950592 + }, + { + "epoch": 0.84, + "learning_rate": 7.898989898989899e-05, + "loss": 2.6049, + "theoretical_loss": 3.3170525343871544, + "tokens_seen": 3044016128 + }, + { + "epoch": 0.84, + "learning_rate": 7.896969696969697e-05, + "loss": 2.5976, + "theoretical_loss": 3.317047059885568, + "tokens_seen": 3044081664 + }, + { + "epoch": 0.84, + "learning_rate": 7.894949494949496e-05, + "loss": 2.6331, + "theoretical_loss": 3.3170415855348416, + "tokens_seen": 3044147200 + }, + { + "epoch": 0.84, + "learning_rate": 7.892929292929294e-05, + "loss": 2.7247, + "theoretical_loss": 3.317036111334967, + "tokens_seen": 3044212736 + }, + { + "epoch": 0.84, + "learning_rate": 7.890909090909091e-05, + "loss": 2.5416, + "theoretical_loss": 3.3170306372859364, + "tokens_seen": 3044278272 + }, + { + "epoch": 0.84, + "learning_rate": 7.888888888888888e-05, + "loss": 2.2535, + "theoretical_loss": 3.317025163387743, + "tokens_seen": 3044343808 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1712674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6885592937469482, + "objective/train/theoretical_loss": 3.3170196896403796, + "objective/train/tokens_used": 1403268576, + "theoretical_loss": 3.3170196896403796, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.84, + "learning_rate": 7.886868686868686e-05, + "loss": 2.4871, + "theoretical_loss": 3.3170196896403796, + "tokens_seen": 3044409344 + }, + { + "epoch": 0.84, + "learning_rate": 7.884848484848485e-05, + "loss": 2.695, + "theoretical_loss": 3.3170142160438383, + "tokens_seen": 3044474880 + }, + { + "epoch": 0.84, + "learning_rate": 7.882828282828283e-05, + "loss": 2.5348, + "theoretical_loss": 3.3170087425981123, + "tokens_seen": 3044540416 + }, + { + "epoch": 0.84, + "learning_rate": 7.880808080808082e-05, + "loss": 2.5854, + "theoretical_loss": 3.3170032693031937, + "tokens_seen": 3044605952 + }, + { + "epoch": 0.84, + "learning_rate": 7.878787878787879e-05, + "loss": 2.5615, + "theoretical_loss": 3.316997796159075, + "tokens_seen": 3044671488 + }, + { + "epoch": 0.84, + "learning_rate": 7.876767676767677e-05, + "loss": 2.5694, + "theoretical_loss": 3.3169923231657488, + "tokens_seen": 3044737024 + }, + { + "epoch": 0.84, + "learning_rate": 7.874747474747475e-05, + "loss": 2.3058, + "theoretical_loss": 3.3169868503232083, + "tokens_seen": 3044802560 + }, + { + "epoch": 0.84, + "learning_rate": 7.872727272727273e-05, + "loss": 2.2625, + "theoretical_loss": 3.3169813776314454, + "tokens_seen": 3044868096 + }, + { + "epoch": 0.84, + "learning_rate": 7.870707070707071e-05, + "loss": 2.5038, + "theoretical_loss": 3.3169759050904535, + "tokens_seen": 3044933632 + }, + { + "epoch": 0.84, + "learning_rate": 7.868686868686868e-05, + "loss": 2.6075, + "theoretical_loss": 3.3169704327002245, + "tokens_seen": 3044999168 + }, + { + "epoch": 0.84, + "learning_rate": 7.866666666666666e-05, + "loss": 2.4083, + "theoretical_loss": 3.316964960460751, + "tokens_seen": 3045064704 + }, + { + "epoch": 0.84, + "learning_rate": 7.864646464646465e-05, + "loss": 2.3636, + "theoretical_loss": 3.316959488372026, + "tokens_seen": 3045130240 + }, + { + "epoch": 0.84, + "learning_rate": 7.862626262626263e-05, + "loss": 2.6577, + "theoretical_loss": 3.3169540164340416, + "tokens_seen": 3045195776 + }, + { + "epoch": 0.84, + "learning_rate": 7.860606060606062e-05, + "loss": 2.5692, + "theoretical_loss": 3.3169485446467912, + "tokens_seen": 3045261312 + }, + { + "epoch": 0.84, + "learning_rate": 7.85858585858586e-05, + "loss": 2.6091, + "theoretical_loss": 3.316943073010267, + "tokens_seen": 3045326848 + }, + { + "epoch": 0.84, + "learning_rate": 7.856565656565656e-05, + "loss": 2.5998, + "theoretical_loss": 3.316937601524461, + "tokens_seen": 3045392384 + }, + { + "epoch": 0.84, + "learning_rate": 7.854545454545454e-05, + "loss": 2.5405, + "theoretical_loss": 3.316932130189367, + "tokens_seen": 3045457920 + }, + { + "epoch": 0.84, + "learning_rate": 7.852525252525252e-05, + "loss": 2.6427, + "theoretical_loss": 3.3169266590049764, + "tokens_seen": 3045523456 + }, + { + "epoch": 0.84, + "learning_rate": 7.850505050505051e-05, + "loss": 2.6488, + "theoretical_loss": 3.3169211879712828, + "tokens_seen": 3045588992 + }, + { + "epoch": 0.84, + "learning_rate": 7.848484848484849e-05, + "loss": 2.4076, + "theoretical_loss": 3.316915717088278, + "tokens_seen": 3045654528 + }, + { + "epoch": 0.84, + "learning_rate": 7.846464646464646e-05, + "loss": 2.5012, + "theoretical_loss": 3.316910246355955, + "tokens_seen": 3045720064 + }, + { + "epoch": 0.84, + "learning_rate": 7.844444444444445e-05, + "loss": 2.5974, + "theoretical_loss": 3.316904775774306, + "tokens_seen": 3045785600 + }, + { + "epoch": 0.84, + "learning_rate": 7.842424242424243e-05, + "loss": 2.6746, + "theoretical_loss": 3.316899305343325, + "tokens_seen": 3045851136 + }, + { + "epoch": 0.84, + "learning_rate": 7.840404040404041e-05, + "loss": 2.6381, + "theoretical_loss": 3.316893835063003, + "tokens_seen": 3045916672 + }, + { + "epoch": 0.84, + "learning_rate": 7.838383838383838e-05, + "loss": 2.5885, + "theoretical_loss": 3.316888364933333, + "tokens_seen": 3045982208 + }, + { + "epoch": 0.84, + "objective/train/docs_used": 1713719, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8562474250793457, + "objective/train/theoretical_loss": 3.3168828949543085, + "objective/train/tokens_used": 1404906976, + "theoretical_loss": 3.3168828949543085, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.84, + "learning_rate": 7.836363636363636e-05, + "loss": 2.6227, + "theoretical_loss": 3.3168828949543085, + "tokens_seen": 3046047744 + }, + { + "epoch": 0.85, + "learning_rate": 7.834343434343434e-05, + "loss": 2.4828, + "theoretical_loss": 3.316877425125921, + "tokens_seen": 3046113280 + }, + { + "epoch": 0.85, + "learning_rate": 7.832323232323232e-05, + "loss": 2.7639, + "theoretical_loss": 3.316871955448163, + "tokens_seen": 3046178816 + }, + { + "epoch": 0.85, + "learning_rate": 7.830303030303031e-05, + "loss": 2.6728, + "theoretical_loss": 3.3168664859210284, + "tokens_seen": 3046244352 + }, + { + "epoch": 0.85, + "learning_rate": 7.828282828282829e-05, + "loss": 2.4912, + "theoretical_loss": 3.316861016544509, + "tokens_seen": 3046309888 + }, + { + "epoch": 0.85, + "learning_rate": 7.826262626262626e-05, + "loss": 2.6212, + "theoretical_loss": 3.316855547318597, + "tokens_seen": 3046375424 + }, + { + "epoch": 0.85, + "learning_rate": 7.824242424242425e-05, + "loss": 2.7065, + "theoretical_loss": 3.316850078243286, + "tokens_seen": 3046440960 + }, + { + "epoch": 0.85, + "learning_rate": 7.822222222222222e-05, + "loss": 2.245, + "theoretical_loss": 3.3168446093185677, + "tokens_seen": 3046506496 + }, + { + "epoch": 0.85, + "learning_rate": 7.82020202020202e-05, + "loss": 2.5397, + "theoretical_loss": 3.3168391405444355, + "tokens_seen": 3046572032 + }, + { + "epoch": 0.85, + "learning_rate": 7.818181818181818e-05, + "loss": 2.6465, + "theoretical_loss": 3.316833671920881, + "tokens_seen": 3046637568 + }, + { + "epoch": 0.85, + "learning_rate": 7.816161616161617e-05, + "loss": 2.5302, + "theoretical_loss": 3.3168282034478977, + "tokens_seen": 3046703104 + }, + { + "epoch": 0.85, + "learning_rate": 7.814141414141414e-05, + "loss": 2.5019, + "theoretical_loss": 3.3168227351254784, + "tokens_seen": 3046768640 + }, + { + "epoch": 0.85, + "learning_rate": 7.812121212121212e-05, + "loss": 2.6816, + "theoretical_loss": 3.316817266953615, + "tokens_seen": 3046834176 + }, + { + "epoch": 0.85, + "learning_rate": 7.81010101010101e-05, + "loss": 2.3738, + "theoretical_loss": 3.3168117989323003, + "tokens_seen": 3046899712 + }, + { + "epoch": 0.85, + "learning_rate": 7.808080808080809e-05, + "loss": 2.6031, + "theoretical_loss": 3.316806331061527, + "tokens_seen": 3046965248 + }, + { + "epoch": 0.85, + "learning_rate": 7.806060606060607e-05, + "loss": 2.5138, + "theoretical_loss": 3.316800863341288, + "tokens_seen": 3047030784 + }, + { + "epoch": 0.85, + "learning_rate": 7.804040404040403e-05, + "loss": 2.568, + "theoretical_loss": 3.316795395771575, + "tokens_seen": 3047096320 + }, + { + "epoch": 0.85, + "learning_rate": 7.802020202020201e-05, + "loss": 2.4294, + "theoretical_loss": 3.3167899283523816, + "tokens_seen": 3047161856 + }, + { + "epoch": 0.85, + "learning_rate": 7.8e-05, + "loss": 2.7392, + "theoretical_loss": 3.3167844610837003, + "tokens_seen": 3047227392 + }, + { + "epoch": 0.85, + "learning_rate": 7.797979797979798e-05, + "loss": 2.6279, + "theoretical_loss": 3.3167789939655234, + "tokens_seen": 3047292928 + }, + { + "epoch": 0.85, + "learning_rate": 7.795959595959597e-05, + "loss": 2.6142, + "theoretical_loss": 3.3167735269978436, + "tokens_seen": 3047358464 + }, + { + "epoch": 0.85, + "learning_rate": 7.793939393939394e-05, + "loss": 2.7275, + "theoretical_loss": 3.3167680601806535, + "tokens_seen": 3047424000 + }, + { + "epoch": 0.85, + "learning_rate": 7.791919191919192e-05, + "loss": 2.6817, + "theoretical_loss": 3.3167625935139458, + "tokens_seen": 3047489536 + }, + { + "epoch": 0.85, + "learning_rate": 7.78989898989899e-05, + "loss": 2.4681, + "theoretical_loss": 3.3167571269977127, + "tokens_seen": 3047555072 + }, + { + "epoch": 0.85, + "learning_rate": 7.787878787878789e-05, + "loss": 2.6988, + "theoretical_loss": 3.316751660631948, + "tokens_seen": 3047620608 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1714459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.585123062133789, + "objective/train/theoretical_loss": 3.316746194416643, + "objective/train/tokens_used": 1406545376, + "theoretical_loss": 3.316746194416643, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.85, + "learning_rate": 7.785858585858586e-05, + "loss": 2.4881, + "theoretical_loss": 3.316746194416643, + "tokens_seen": 3047686144 + }, + { + "epoch": 0.85, + "learning_rate": 7.783838383838383e-05, + "loss": 2.438, + "theoretical_loss": 3.3167407283517907, + "tokens_seen": 3047751680 + }, + { + "epoch": 0.85, + "learning_rate": 7.781818181818181e-05, + "loss": 2.5747, + "theoretical_loss": 3.3167352624373843, + "tokens_seen": 3047817216 + }, + { + "epoch": 0.85, + "learning_rate": 7.77979797979798e-05, + "loss": 2.507, + "theoretical_loss": 3.3167297966734157, + "tokens_seen": 3047882752 + }, + { + "epoch": 0.85, + "learning_rate": 7.777777777777778e-05, + "loss": 2.6198, + "theoretical_loss": 3.316724331059878, + "tokens_seen": 3047948288 + }, + { + "epoch": 0.85, + "learning_rate": 7.775757575757577e-05, + "loss": 2.6891, + "theoretical_loss": 3.316718865596763, + "tokens_seen": 3048013824 + }, + { + "epoch": 0.85, + "learning_rate": 7.773737373737375e-05, + "loss": 2.6413, + "theoretical_loss": 3.3167134002840646, + "tokens_seen": 3048079360 + }, + { + "epoch": 0.85, + "learning_rate": 7.771717171717172e-05, + "loss": 2.5933, + "theoretical_loss": 3.3167079351217748, + "tokens_seen": 3048144896 + }, + { + "epoch": 0.85, + "learning_rate": 7.769696969696969e-05, + "loss": 2.7642, + "theoretical_loss": 3.3167024701098864, + "tokens_seen": 3048210432 + }, + { + "epoch": 0.85, + "learning_rate": 7.767676767676767e-05, + "loss": 2.7618, + "theoretical_loss": 3.3166970052483915, + "tokens_seen": 3048275968 + }, + { + "epoch": 0.85, + "learning_rate": 7.765656565656566e-05, + "loss": 2.4432, + "theoretical_loss": 3.316691540537283, + "tokens_seen": 3048341504 + }, + { + "epoch": 0.85, + "learning_rate": 7.763636363636364e-05, + "loss": 2.7258, + "theoretical_loss": 3.3166860759765537, + "tokens_seen": 3048407040 + }, + { + "epoch": 0.85, + "learning_rate": 7.761616161616161e-05, + "loss": 2.5846, + "theoretical_loss": 3.3166806115661966, + "tokens_seen": 3048472576 + }, + { + "epoch": 0.85, + "learning_rate": 7.75959595959596e-05, + "loss": 2.3856, + "theoretical_loss": 3.3166751473062033, + "tokens_seen": 3048538112 + }, + { + "epoch": 0.85, + "learning_rate": 7.757575757575758e-05, + "loss": 2.7791, + "theoretical_loss": 3.3166696831965674, + "tokens_seen": 3048603648 + }, + { + "epoch": 0.85, + "learning_rate": 7.755555555555556e-05, + "loss": 2.5487, + "theoretical_loss": 3.316664219237281, + "tokens_seen": 3048669184 + }, + { + "epoch": 0.85, + "learning_rate": 7.753535353535355e-05, + "loss": 2.3189, + "theoretical_loss": 3.3166587554283367, + "tokens_seen": 3048734720 + }, + { + "epoch": 0.85, + "learning_rate": 7.75151515151515e-05, + "loss": 2.4926, + "theoretical_loss": 3.316653291769727, + "tokens_seen": 3048800256 + }, + { + "epoch": 0.85, + "learning_rate": 7.749494949494949e-05, + "loss": 2.6131, + "theoretical_loss": 3.3166478282614458, + "tokens_seen": 3048865792 + }, + { + "epoch": 0.85, + "learning_rate": 7.747474747474747e-05, + "loss": 2.5705, + "theoretical_loss": 3.3166423649034837, + "tokens_seen": 3048931328 + }, + { + "epoch": 0.85, + "learning_rate": 7.745454545454546e-05, + "loss": 2.7533, + "theoretical_loss": 3.3166369016958352, + "tokens_seen": 3048996864 + }, + { + "epoch": 0.85, + "learning_rate": 7.743434343434344e-05, + "loss": 2.5682, + "theoretical_loss": 3.316631438638492, + "tokens_seen": 3049062400 + }, + { + "epoch": 0.85, + "learning_rate": 7.741414141414141e-05, + "loss": 2.3521, + "theoretical_loss": 3.316625975731446, + "tokens_seen": 3049127936 + }, + { + "epoch": 0.85, + "learning_rate": 7.73939393939394e-05, + "loss": 2.5125, + "theoretical_loss": 3.3166205129746915, + "tokens_seen": 3049193472 + }, + { + "epoch": 0.85, + "learning_rate": 7.737373737373738e-05, + "loss": 2.3643, + "theoretical_loss": 3.3166150503682204, + "tokens_seen": 3049259008 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1714997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.555889368057251, + "objective/train/theoretical_loss": 3.316609587912025, + "objective/train/tokens_used": 1408183776, + "theoretical_loss": 3.316609587912025, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 7.735353535353535e-05, + "loss": 2.6416, + "theoretical_loss": 3.316609587912025, + "tokens_seen": 3049324544 + }, + { + "epoch": 0.85, + "learning_rate": 7.733333333333333e-05, + "loss": 2.6156, + "theoretical_loss": 3.3166041256060983, + "tokens_seen": 3049390080 + }, + { + "epoch": 0.85, + "learning_rate": 7.731313131313132e-05, + "loss": 2.5316, + "theoretical_loss": 3.3165986634504327, + "tokens_seen": 3049455616 + }, + { + "epoch": 0.85, + "learning_rate": 7.729292929292929e-05, + "loss": 2.7375, + "theoretical_loss": 3.316593201445021, + "tokens_seen": 3049521152 + }, + { + "epoch": 0.85, + "learning_rate": 7.727272727272727e-05, + "loss": 2.69, + "theoretical_loss": 3.3165877395898558, + "tokens_seen": 3049586688 + }, + { + "epoch": 0.85, + "learning_rate": 7.725252525252526e-05, + "loss": 2.7007, + "theoretical_loss": 3.3165822778849297, + "tokens_seen": 3049652224 + }, + { + "epoch": 0.85, + "learning_rate": 7.723232323232324e-05, + "loss": 2.5537, + "theoretical_loss": 3.316576816330236, + "tokens_seen": 3049717760 + }, + { + "epoch": 0.85, + "learning_rate": 7.721212121212122e-05, + "loss": 2.5952, + "theoretical_loss": 3.316571354925766, + "tokens_seen": 3049783296 + }, + { + "epoch": 0.85, + "learning_rate": 7.71919191919192e-05, + "loss": 2.608, + "theoretical_loss": 3.3165658936715134, + "tokens_seen": 3049848832 + }, + { + "epoch": 0.85, + "learning_rate": 7.717171717171716e-05, + "loss": 2.5413, + "theoretical_loss": 3.3165604325674702, + "tokens_seen": 3049914368 + }, + { + "epoch": 0.85, + "learning_rate": 7.715151515151515e-05, + "loss": 2.6186, + "theoretical_loss": 3.3165549716136296, + "tokens_seen": 3049979904 + }, + { + "epoch": 0.85, + "learning_rate": 7.713131313131313e-05, + "loss": 2.7027, + "theoretical_loss": 3.316549510809984, + "tokens_seen": 3050045440 + }, + { + "epoch": 0.85, + "learning_rate": 7.711111111111112e-05, + "loss": 2.2632, + "theoretical_loss": 3.316544050156526, + "tokens_seen": 3050110976 + }, + { + "epoch": 0.85, + "learning_rate": 7.709090909090909e-05, + "loss": 2.5839, + "theoretical_loss": 3.3165385896532484, + "tokens_seen": 3050176512 + }, + { + "epoch": 0.85, + "learning_rate": 7.707070707070707e-05, + "loss": 2.73, + "theoretical_loss": 3.3165331293001437, + "tokens_seen": 3050242048 + }, + { + "epoch": 0.85, + "learning_rate": 7.705050505050506e-05, + "loss": 2.4916, + "theoretical_loss": 3.316527669097204, + "tokens_seen": 3050307584 + }, + { + "epoch": 0.85, + "learning_rate": 7.703030303030304e-05, + "loss": 2.6856, + "theoretical_loss": 3.3165222090444235, + "tokens_seen": 3050373120 + }, + { + "epoch": 0.85, + "learning_rate": 7.701010101010102e-05, + "loss": 2.4588, + "theoretical_loss": 3.316516749141793, + "tokens_seen": 3050438656 + }, + { + "epoch": 0.85, + "learning_rate": 7.698989898989898e-05, + "loss": 2.6509, + "theoretical_loss": 3.3165112893893065, + "tokens_seen": 3050504192 + }, + { + "epoch": 0.85, + "learning_rate": 7.696969696969696e-05, + "loss": 2.4906, + "theoretical_loss": 3.316505829786956, + "tokens_seen": 3050569728 + }, + { + "epoch": 0.85, + "learning_rate": 7.694949494949495e-05, + "loss": 2.297, + "theoretical_loss": 3.316500370334734, + "tokens_seen": 3050635264 + }, + { + "epoch": 0.85, + "learning_rate": 7.692929292929293e-05, + "loss": 2.7798, + "theoretical_loss": 3.316494911032634, + "tokens_seen": 3050700800 + }, + { + "epoch": 0.85, + "learning_rate": 7.690909090909092e-05, + "loss": 2.5731, + "theoretical_loss": 3.3164894518806474, + "tokens_seen": 3050766336 + }, + { + "epoch": 0.85, + "learning_rate": 7.68888888888889e-05, + "loss": 2.442, + "theoretical_loss": 3.316483992878768, + "tokens_seen": 3050831872 + }, + { + "epoch": 0.85, + "learning_rate": 7.686868686868687e-05, + "loss": 2.3898, + "theoretical_loss": 3.316478534026988, + "tokens_seen": 3050897408 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1716070, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6597750186920166, + "objective/train/theoretical_loss": 3.3164730753253, + "objective/train/tokens_used": 1409822176, + "theoretical_loss": 3.3164730753253, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.85, + "learning_rate": 7.684848484848485e-05, + "loss": 2.7366, + "theoretical_loss": 3.3164730753253, + "tokens_seen": 3050962944 + }, + { + "epoch": 0.85, + "learning_rate": 7.682828282828282e-05, + "loss": 2.7513, + "theoretical_loss": 3.3164676167736964, + "tokens_seen": 3051028480 + }, + { + "epoch": 0.85, + "learning_rate": 7.680808080808081e-05, + "loss": 2.5273, + "theoretical_loss": 3.3164621583721705, + "tokens_seen": 3051094016 + }, + { + "epoch": 0.85, + "learning_rate": 7.678787878787879e-05, + "loss": 2.4961, + "theoretical_loss": 3.3164567001207144, + "tokens_seen": 3051159552 + }, + { + "epoch": 0.85, + "learning_rate": 7.676767676767676e-05, + "loss": 2.6035, + "theoretical_loss": 3.316451242019321, + "tokens_seen": 3051225088 + }, + { + "epoch": 0.85, + "learning_rate": 7.674747474747475e-05, + "loss": 2.6301, + "theoretical_loss": 3.316445784067983, + "tokens_seen": 3051290624 + }, + { + "epoch": 0.85, + "learning_rate": 7.672727272727273e-05, + "loss": 2.8331, + "theoretical_loss": 3.3164403262666924, + "tokens_seen": 3051356160 + }, + { + "epoch": 0.85, + "learning_rate": 7.670707070707071e-05, + "loss": 2.6143, + "theoretical_loss": 3.316434868615443, + "tokens_seen": 3051421696 + }, + { + "epoch": 0.85, + "learning_rate": 7.66868686868687e-05, + "loss": 2.5054, + "theoretical_loss": 3.3164294111142265, + "tokens_seen": 3051487232 + }, + { + "epoch": 0.85, + "learning_rate": 7.666666666666667e-05, + "loss": 2.4664, + "theoretical_loss": 3.316423953763036, + "tokens_seen": 3051552768 + }, + { + "epoch": 0.85, + "learning_rate": 7.664646464646464e-05, + "loss": 2.5388, + "theoretical_loss": 3.3164184965618637, + "tokens_seen": 3051618304 + }, + { + "epoch": 0.85, + "learning_rate": 7.662626262626262e-05, + "loss": 2.418, + "theoretical_loss": 3.316413039510703, + "tokens_seen": 3051683840 + }, + { + "epoch": 0.85, + "learning_rate": 7.660606060606061e-05, + "loss": 2.6398, + "theoretical_loss": 3.3164075826095463, + "tokens_seen": 3051749376 + }, + { + "epoch": 0.85, + "learning_rate": 7.658585858585859e-05, + "loss": 2.605, + "theoretical_loss": 3.3164021258583856, + "tokens_seen": 3051814912 + }, + { + "epoch": 0.85, + "learning_rate": 7.656565656565656e-05, + "loss": 2.546, + "theoretical_loss": 3.316396669257214, + "tokens_seen": 3051880448 + }, + { + "epoch": 0.85, + "learning_rate": 7.654545454545455e-05, + "loss": 2.5694, + "theoretical_loss": 3.3163912128060247, + "tokens_seen": 3051945984 + }, + { + "epoch": 0.85, + "learning_rate": 7.652525252525253e-05, + "loss": 2.7713, + "theoretical_loss": 3.31638575650481, + "tokens_seen": 3052011520 + }, + { + "epoch": 0.85, + "learning_rate": 7.650505050505051e-05, + "loss": 2.5968, + "theoretical_loss": 3.316380300353562, + "tokens_seen": 3052077056 + }, + { + "epoch": 0.85, + "learning_rate": 7.648484848484848e-05, + "loss": 2.4213, + "theoretical_loss": 3.316374844352274, + "tokens_seen": 3052142592 + }, + { + "epoch": 0.85, + "learning_rate": 7.646464646464647e-05, + "loss": 2.3846, + "theoretical_loss": 3.3163693885009384, + "tokens_seen": 3052208128 + }, + { + "epoch": 0.85, + "learning_rate": 7.644444444444444e-05, + "loss": 2.4207, + "theoretical_loss": 3.316363932799548, + "tokens_seen": 3052273664 + }, + { + "epoch": 0.85, + "learning_rate": 7.642424242424242e-05, + "loss": 2.5378, + "theoretical_loss": 3.3163584772480954, + "tokens_seen": 3052339200 + }, + { + "epoch": 0.85, + "learning_rate": 7.64040404040404e-05, + "loss": 2.492, + "theoretical_loss": 3.3163530218465733, + "tokens_seen": 3052404736 + }, + { + "epoch": 0.85, + "learning_rate": 7.638383838383839e-05, + "loss": 2.5584, + "theoretical_loss": 3.316347566594974, + "tokens_seen": 3052470272 + }, + { + "epoch": 0.85, + "learning_rate": 7.636363636363637e-05, + "loss": 2.6884, + "theoretical_loss": 3.3163421114932907, + "tokens_seen": 3052535808 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1717617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.459357738494873, + "objective/train/theoretical_loss": 3.3163366565415155, + "objective/train/tokens_used": 1411460576, + "theoretical_loss": 3.3163366565415155, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 7.634343434343434e-05, + "loss": 2.5289, + "theoretical_loss": 3.3163366565415155, + "tokens_seen": 3052601344 + }, + { + "epoch": 0.85, + "learning_rate": 7.632323232323233e-05, + "loss": 2.5119, + "theoretical_loss": 3.316331201739642, + "tokens_seen": 3052666880 + }, + { + "epoch": 0.85, + "learning_rate": 7.63030303030303e-05, + "loss": 2.6005, + "theoretical_loss": 3.3163257470876615, + "tokens_seen": 3052732416 + }, + { + "epoch": 0.85, + "learning_rate": 7.628282828282828e-05, + "loss": 2.8303, + "theoretical_loss": 3.3163202925855675, + "tokens_seen": 3052797952 + }, + { + "epoch": 0.85, + "learning_rate": 7.626262626262627e-05, + "loss": 2.8315, + "theoretical_loss": 3.316314838233353, + "tokens_seen": 3052863488 + }, + { + "epoch": 0.85, + "learning_rate": 7.624242424242424e-05, + "loss": 2.7, + "theoretical_loss": 3.31630938403101, + "tokens_seen": 3052929024 + }, + { + "epoch": 0.85, + "learning_rate": 7.622222222222222e-05, + "loss": 2.6339, + "theoretical_loss": 3.3163039299785315, + "tokens_seen": 3052994560 + }, + { + "epoch": 0.85, + "learning_rate": 7.62020202020202e-05, + "loss": 2.674, + "theoretical_loss": 3.31629847607591, + "tokens_seen": 3053060096 + }, + { + "epoch": 0.85, + "learning_rate": 7.618181818181819e-05, + "loss": 2.7427, + "theoretical_loss": 3.316293022323138, + "tokens_seen": 3053125632 + }, + { + "epoch": 0.85, + "learning_rate": 7.616161616161617e-05, + "loss": 2.6062, + "theoretical_loss": 3.3162875687202087, + "tokens_seen": 3053191168 + }, + { + "epoch": 0.85, + "learning_rate": 7.614141414141413e-05, + "loss": 2.6628, + "theoretical_loss": 3.3162821152671142, + "tokens_seen": 3053256704 + }, + { + "epoch": 0.85, + "learning_rate": 7.612121212121211e-05, + "loss": 2.5916, + "theoretical_loss": 3.316276661963848, + "tokens_seen": 3053322240 + }, + { + "epoch": 0.85, + "learning_rate": 7.61010101010101e-05, + "loss": 2.3324, + "theoretical_loss": 3.3162712088104014, + "tokens_seen": 3053387776 + }, + { + "epoch": 0.85, + "learning_rate": 7.608080808080808e-05, + "loss": 2.4006, + "theoretical_loss": 3.3162657558067687, + "tokens_seen": 3053453312 + }, + { + "epoch": 0.85, + "learning_rate": 7.606060606060607e-05, + "loss": 2.5159, + "theoretical_loss": 3.316260302952941, + "tokens_seen": 3053518848 + }, + { + "epoch": 0.85, + "learning_rate": 7.604040404040405e-05, + "loss": 2.7204, + "theoretical_loss": 3.316254850248912, + "tokens_seen": 3053584384 + }, + { + "epoch": 0.85, + "learning_rate": 7.602020202020202e-05, + "loss": 2.6254, + "theoretical_loss": 3.3162493976946745, + "tokens_seen": 3053649920 + }, + { + "epoch": 0.85, + "learning_rate": 7.6e-05, + "loss": 2.5924, + "theoretical_loss": 3.3162439452902204, + "tokens_seen": 3053715456 + }, + { + "epoch": 0.85, + "learning_rate": 7.597979797979799e-05, + "loss": 2.5613, + "theoretical_loss": 3.3162384930355424, + "tokens_seen": 3053780992 + }, + { + "epoch": 0.85, + "learning_rate": 7.595959595959596e-05, + "loss": 2.5182, + "theoretical_loss": 3.3162330409306335, + "tokens_seen": 3053846528 + }, + { + "epoch": 0.85, + "learning_rate": 7.593939393939394e-05, + "loss": 2.574, + "theoretical_loss": 3.3162275889754866, + "tokens_seen": 3053912064 + }, + { + "epoch": 0.85, + "learning_rate": 7.591919191919191e-05, + "loss": 2.9004, + "theoretical_loss": 3.316222137170094, + "tokens_seen": 3053977600 + }, + { + "epoch": 0.85, + "learning_rate": 7.58989898989899e-05, + "loss": 2.484, + "theoretical_loss": 3.316216685514449, + "tokens_seen": 3054043136 + }, + { + "epoch": 0.85, + "learning_rate": 7.587878787878788e-05, + "loss": 2.6838, + "theoretical_loss": 3.3162112340085432, + "tokens_seen": 3054108672 + }, + { + "epoch": 0.85, + "learning_rate": 7.585858585858586e-05, + "loss": 2.699, + "theoretical_loss": 3.31620578265237, + "tokens_seen": 3054174208 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1718284, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3081488609313965, + "objective/train/theoretical_loss": 3.316200331445922, + "objective/train/tokens_used": 1413098976, + "theoretical_loss": 3.316200331445922, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.85, + "learning_rate": 7.583838383838385e-05, + "loss": 2.2973, + "theoretical_loss": 3.316200331445922, + "tokens_seen": 3054239744 + }, + { + "epoch": 0.85, + "learning_rate": 7.581818181818182e-05, + "loss": 2.5608, + "theoretical_loss": 3.316194880389192, + "tokens_seen": 3054305280 + }, + { + "epoch": 0.85, + "learning_rate": 7.57979797979798e-05, + "loss": 2.5807, + "theoretical_loss": 3.316189429482172, + "tokens_seen": 3054370816 + }, + { + "epoch": 0.85, + "learning_rate": 7.577777777777777e-05, + "loss": 2.6227, + "theoretical_loss": 3.3161839787248555, + "tokens_seen": 3054436352 + }, + { + "epoch": 0.85, + "learning_rate": 7.575757575757576e-05, + "loss": 2.5293, + "theoretical_loss": 3.3161785281172347, + "tokens_seen": 3054501888 + }, + { + "epoch": 0.85, + "learning_rate": 7.573737373737374e-05, + "loss": 2.5151, + "theoretical_loss": 3.3161730776593026, + "tokens_seen": 3054567424 + }, + { + "epoch": 0.85, + "learning_rate": 7.571717171717171e-05, + "loss": 2.4345, + "theoretical_loss": 3.316167627351051, + "tokens_seen": 3054632960 + }, + { + "epoch": 0.85, + "learning_rate": 7.56969696969697e-05, + "loss": 2.6516, + "theoretical_loss": 3.3161621771924743, + "tokens_seen": 3054698496 + }, + { + "epoch": 0.85, + "learning_rate": 7.567676767676768e-05, + "loss": 2.7234, + "theoretical_loss": 3.3161567271835635, + "tokens_seen": 3054764032 + }, + { + "epoch": 0.85, + "learning_rate": 7.565656565656566e-05, + "loss": 2.7353, + "theoretical_loss": 3.316151277324312, + "tokens_seen": 3054829568 + }, + { + "epoch": 0.85, + "learning_rate": 7.563636363636365e-05, + "loss": 2.6559, + "theoretical_loss": 3.3161458276147124, + "tokens_seen": 3054895104 + }, + { + "epoch": 0.85, + "learning_rate": 7.561616161616162e-05, + "loss": 2.616, + "theoretical_loss": 3.3161403780547576, + "tokens_seen": 3054960640 + }, + { + "epoch": 0.85, + "learning_rate": 7.559595959595959e-05, + "loss": 2.6033, + "theoretical_loss": 3.31613492864444, + "tokens_seen": 3055026176 + }, + { + "epoch": 0.85, + "learning_rate": 7.557575757575757e-05, + "loss": 2.6537, + "theoretical_loss": 3.3161294793837524, + "tokens_seen": 3055091712 + }, + { + "epoch": 0.85, + "learning_rate": 7.555555555555556e-05, + "loss": 2.6929, + "theoretical_loss": 3.316124030272687, + "tokens_seen": 3055157248 + }, + { + "epoch": 0.85, + "learning_rate": 7.553535353535354e-05, + "loss": 2.5371, + "theoretical_loss": 3.3161185813112373, + "tokens_seen": 3055222784 + }, + { + "epoch": 0.85, + "learning_rate": 7.551515151515152e-05, + "loss": 2.5436, + "theoretical_loss": 3.3161131324993955, + "tokens_seen": 3055288320 + }, + { + "epoch": 0.85, + "learning_rate": 7.54949494949495e-05, + "loss": 2.4061, + "theoretical_loss": 3.316107683837154, + "tokens_seen": 3055353856 + }, + { + "epoch": 0.85, + "learning_rate": 7.547474747474748e-05, + "loss": 2.6541, + "theoretical_loss": 3.3161022353245064, + "tokens_seen": 3055419392 + }, + { + "epoch": 0.85, + "learning_rate": 7.545454545454546e-05, + "loss": 2.5245, + "theoretical_loss": 3.316096786961445, + "tokens_seen": 3055484928 + }, + { + "epoch": 0.85, + "learning_rate": 7.543434343434343e-05, + "loss": 2.3983, + "theoretical_loss": 3.316091338747962, + "tokens_seen": 3055550464 + }, + { + "epoch": 0.85, + "learning_rate": 7.541414141414142e-05, + "loss": 2.6158, + "theoretical_loss": 3.3160858906840502, + "tokens_seen": 3055616000 + }, + { + "epoch": 0.85, + "learning_rate": 7.539393939393939e-05, + "loss": 2.4757, + "theoretical_loss": 3.3160804427697026, + "tokens_seen": 3055681536 + }, + { + "epoch": 0.85, + "learning_rate": 7.537373737373737e-05, + "loss": 2.4787, + "theoretical_loss": 3.3160749950049118, + "tokens_seen": 3055747072 + }, + { + "epoch": 0.85, + "learning_rate": 7.535353535353536e-05, + "loss": 2.6943, + "theoretical_loss": 3.3160695473896706, + "tokens_seen": 3055812608 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1719055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.638411283493042, + "objective/train/theoretical_loss": 3.316064099923971, + "objective/train/tokens_used": 1414737376, + "theoretical_loss": 3.316064099923971, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 7.533333333333334e-05, + "loss": 2.5786, + "theoretical_loss": 3.316064099923971, + "tokens_seen": 3055878144 + }, + { + "epoch": 0.85, + "learning_rate": 7.531313131313132e-05, + "loss": 2.4402, + "theoretical_loss": 3.3160586526078073, + "tokens_seen": 3055943680 + }, + { + "epoch": 0.85, + "learning_rate": 7.52929292929293e-05, + "loss": 2.5528, + "theoretical_loss": 3.3160532054411704, + "tokens_seen": 3056009216 + }, + { + "epoch": 0.85, + "learning_rate": 7.527272727272726e-05, + "loss": 2.5512, + "theoretical_loss": 3.316047758424054, + "tokens_seen": 3056074752 + }, + { + "epoch": 0.85, + "learning_rate": 7.525252525252525e-05, + "loss": 2.8455, + "theoretical_loss": 3.3160423115564504, + "tokens_seen": 3056140288 + }, + { + "epoch": 0.85, + "learning_rate": 7.523232323232323e-05, + "loss": 2.6438, + "theoretical_loss": 3.3160368648383525, + "tokens_seen": 3056205824 + }, + { + "epoch": 0.85, + "learning_rate": 7.521212121212122e-05, + "loss": 2.6201, + "theoretical_loss": 3.316031418269753, + "tokens_seen": 3056271360 + }, + { + "epoch": 0.85, + "learning_rate": 7.51919191919192e-05, + "loss": 2.579, + "theoretical_loss": 3.316025971850644, + "tokens_seen": 3056336896 + }, + { + "epoch": 0.85, + "learning_rate": 7.517171717171717e-05, + "loss": 2.5743, + "theoretical_loss": 3.316020525581019, + "tokens_seen": 3056402432 + }, + { + "epoch": 0.85, + "learning_rate": 7.515151515151515e-05, + "loss": 2.7141, + "theoretical_loss": 3.3160150794608705, + "tokens_seen": 3056467968 + }, + { + "epoch": 0.85, + "learning_rate": 7.513131313131314e-05, + "loss": 2.6058, + "theoretical_loss": 3.3160096334901907, + "tokens_seen": 3056533504 + }, + { + "epoch": 0.85, + "learning_rate": 7.511111111111112e-05, + "loss": 2.5457, + "theoretical_loss": 3.3160041876689728, + "tokens_seen": 3056599040 + }, + { + "epoch": 0.85, + "learning_rate": 7.509090909090909e-05, + "loss": 2.5856, + "theoretical_loss": 3.3159987419972095, + "tokens_seen": 3056664576 + }, + { + "epoch": 0.85, + "learning_rate": 7.507070707070706e-05, + "loss": 2.6894, + "theoretical_loss": 3.3159932964748933, + "tokens_seen": 3056730112 + }, + { + "epoch": 0.85, + "learning_rate": 7.505050505050505e-05, + "loss": 2.5454, + "theoretical_loss": 3.315987851102017, + "tokens_seen": 3056795648 + }, + { + "epoch": 0.85, + "learning_rate": 7.503030303030303e-05, + "loss": 2.8089, + "theoretical_loss": 3.3159824058785734, + "tokens_seen": 3056861184 + }, + { + "epoch": 0.85, + "learning_rate": 7.501010101010102e-05, + "loss": 2.8424, + "theoretical_loss": 3.3159769608045546, + "tokens_seen": 3056926720 + }, + { + "epoch": 0.85, + "learning_rate": 7.4989898989899e-05, + "loss": 2.5722, + "theoretical_loss": 3.3159715158799536, + "tokens_seen": 3056992256 + }, + { + "epoch": 0.85, + "learning_rate": 7.496969696969697e-05, + "loss": 2.3683, + "theoretical_loss": 3.3159660711047634, + "tokens_seen": 3057057792 + }, + { + "epoch": 0.85, + "learning_rate": 7.494949494949495e-05, + "loss": 2.453, + "theoretical_loss": 3.3159606264789767, + "tokens_seen": 3057123328 + }, + { + "epoch": 0.85, + "learning_rate": 7.492929292929292e-05, + "loss": 2.6008, + "theoretical_loss": 3.315955182002586, + "tokens_seen": 3057188864 + }, + { + "epoch": 0.85, + "learning_rate": 7.490909090909091e-05, + "loss": 2.8553, + "theoretical_loss": 3.315949737675584, + "tokens_seen": 3057254400 + }, + { + "epoch": 0.85, + "learning_rate": 7.488888888888889e-05, + "loss": 2.5533, + "theoretical_loss": 3.3159442934979637, + "tokens_seen": 3057319936 + }, + { + "epoch": 0.85, + "learning_rate": 7.486868686868686e-05, + "loss": 2.4095, + "theoretical_loss": 3.315938849469717, + "tokens_seen": 3057385472 + }, + { + "epoch": 0.85, + "learning_rate": 7.484848484848485e-05, + "loss": 2.7532, + "theoretical_loss": 3.315933405590837, + "tokens_seen": 3057451008 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1720294, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8228328227996826, + "objective/train/theoretical_loss": 3.315927961861317, + "objective/train/tokens_used": 1416375776, + "theoretical_loss": 3.315927961861317, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.85, + "learning_rate": 7.482828282828283e-05, + "loss": 2.4755, + "theoretical_loss": 3.315927961861317, + "tokens_seen": 3057516544 + }, + { + "epoch": 0.85, + "learning_rate": 7.480808080808081e-05, + "loss": 2.515, + "theoretical_loss": 3.315922518281149, + "tokens_seen": 3057582080 + }, + { + "epoch": 0.85, + "learning_rate": 7.47878787878788e-05, + "loss": 2.5237, + "theoretical_loss": 3.315917074850326, + "tokens_seen": 3057647616 + }, + { + "epoch": 0.85, + "learning_rate": 7.476767676767678e-05, + "loss": 2.4753, + "theoretical_loss": 3.3159116315688406, + "tokens_seen": 3057713152 + }, + { + "epoch": 0.85, + "learning_rate": 7.474747474747474e-05, + "loss": 2.5948, + "theoretical_loss": 3.3159061884366854, + "tokens_seen": 3057778688 + }, + { + "epoch": 0.85, + "learning_rate": 7.472727272727272e-05, + "loss": 2.3931, + "theoretical_loss": 3.3159007454538534, + "tokens_seen": 3057844224 + }, + { + "epoch": 0.85, + "learning_rate": 7.47070707070707e-05, + "loss": 2.59, + "theoretical_loss": 3.315895302620337, + "tokens_seen": 3057909760 + }, + { + "epoch": 0.85, + "learning_rate": 7.468686868686869e-05, + "loss": 2.3251, + "theoretical_loss": 3.315889859936129, + "tokens_seen": 3057975296 + }, + { + "epoch": 0.85, + "learning_rate": 7.466666666666667e-05, + "loss": 2.4345, + "theoretical_loss": 3.3158844174012225, + "tokens_seen": 3058040832 + }, + { + "epoch": 0.85, + "learning_rate": 7.464646464646464e-05, + "loss": 2.5481, + "theoretical_loss": 3.3158789750156092, + "tokens_seen": 3058106368 + }, + { + "epoch": 0.85, + "learning_rate": 7.462626262626263e-05, + "loss": 2.5, + "theoretical_loss": 3.3158735327792828, + "tokens_seen": 3058171904 + }, + { + "epoch": 0.85, + "learning_rate": 7.460606060606061e-05, + "loss": 2.7247, + "theoretical_loss": 3.315868090692236, + "tokens_seen": 3058237440 + }, + { + "epoch": 0.85, + "learning_rate": 7.45858585858586e-05, + "loss": 2.6063, + "theoretical_loss": 3.3158626487544605, + "tokens_seen": 3058302976 + }, + { + "epoch": 0.85, + "learning_rate": 7.456565656565657e-05, + "loss": 2.6965, + "theoretical_loss": 3.31585720696595, + "tokens_seen": 3058368512 + }, + { + "epoch": 0.85, + "learning_rate": 7.454545454545454e-05, + "loss": 2.5468, + "theoretical_loss": 3.315851765326697, + "tokens_seen": 3058434048 + }, + { + "epoch": 0.85, + "learning_rate": 7.452525252525252e-05, + "loss": 2.7189, + "theoretical_loss": 3.315846323836694, + "tokens_seen": 3058499584 + }, + { + "epoch": 0.85, + "learning_rate": 7.45050505050505e-05, + "loss": 2.6505, + "theoretical_loss": 3.3158408824959333, + "tokens_seen": 3058565120 + }, + { + "epoch": 0.85, + "learning_rate": 7.448484848484849e-05, + "loss": 2.5133, + "theoretical_loss": 3.3158354413044084, + "tokens_seen": 3058630656 + }, + { + "epoch": 0.85, + "learning_rate": 7.446464646464647e-05, + "loss": 2.5438, + "theoretical_loss": 3.315830000262112, + "tokens_seen": 3058696192 + }, + { + "epoch": 0.85, + "learning_rate": 7.444444444444444e-05, + "loss": 2.5834, + "theoretical_loss": 3.3158245593690365, + "tokens_seen": 3058761728 + }, + { + "epoch": 0.85, + "learning_rate": 7.442424242424243e-05, + "loss": 2.1585, + "theoretical_loss": 3.315819118625174, + "tokens_seen": 3058827264 + }, + { + "epoch": 0.85, + "learning_rate": 7.44040404040404e-05, + "loss": 2.6343, + "theoretical_loss": 3.3158136780305183, + "tokens_seen": 3058892800 + }, + { + "epoch": 0.85, + "learning_rate": 7.438383838383838e-05, + "loss": 2.5906, + "theoretical_loss": 3.315808237585062, + "tokens_seen": 3058958336 + }, + { + "epoch": 0.85, + "learning_rate": 7.436363636363637e-05, + "loss": 2.453, + "theoretical_loss": 3.315802797288797, + "tokens_seen": 3059023872 + }, + { + "epoch": 0.85, + "learning_rate": 7.434343434343435e-05, + "loss": 2.6592, + "theoretical_loss": 3.3157973571417165, + "tokens_seen": 3059089408 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1721360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6847140789031982, + "objective/train/theoretical_loss": 3.3157919171438133, + "objective/train/tokens_used": 1418014176, + "theoretical_loss": 3.3157919171438133, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 7.432323232323232e-05, + "loss": 2.4518, + "theoretical_loss": 3.3157919171438133, + "tokens_seen": 3059154944 + }, + { + "epoch": 0.85, + "learning_rate": 7.43030303030303e-05, + "loss": 2.4926, + "theoretical_loss": 3.31578647729508, + "tokens_seen": 3059220480 + }, + { + "epoch": 0.85, + "learning_rate": 7.428282828282829e-05, + "loss": 2.5437, + "theoretical_loss": 3.3157810375955092, + "tokens_seen": 3059286016 + }, + { + "epoch": 0.85, + "learning_rate": 7.426262626262627e-05, + "loss": 2.5542, + "theoretical_loss": 3.315775598045094, + "tokens_seen": 3059351552 + }, + { + "epoch": 0.85, + "learning_rate": 7.424242424242426e-05, + "loss": 2.6605, + "theoretical_loss": 3.3157701586438266, + "tokens_seen": 3059417088 + }, + { + "epoch": 0.85, + "learning_rate": 7.422222222222221e-05, + "loss": 2.5716, + "theoretical_loss": 3.3157647193916997, + "tokens_seen": 3059482624 + }, + { + "epoch": 0.85, + "learning_rate": 7.42020202020202e-05, + "loss": 2.4556, + "theoretical_loss": 3.315759280288707, + "tokens_seen": 3059548160 + }, + { + "epoch": 0.85, + "learning_rate": 7.418181818181818e-05, + "loss": 2.4275, + "theoretical_loss": 3.31575384133484, + "tokens_seen": 3059613696 + }, + { + "epoch": 0.85, + "learning_rate": 7.416161616161617e-05, + "loss": 2.5566, + "theoretical_loss": 3.315748402530092, + "tokens_seen": 3059679232 + }, + { + "epoch": 0.85, + "learning_rate": 7.414141414141415e-05, + "loss": 2.5537, + "theoretical_loss": 3.3157429638744556, + "tokens_seen": 3059744768 + }, + { + "epoch": 0.85, + "learning_rate": 7.412121212121212e-05, + "loss": 2.6151, + "theoretical_loss": 3.3157375253679233, + "tokens_seen": 3059810304 + }, + { + "epoch": 0.85, + "learning_rate": 7.41010101010101e-05, + "loss": 2.5147, + "theoretical_loss": 3.3157320870104883, + "tokens_seen": 3059875840 + }, + { + "epoch": 0.85, + "learning_rate": 7.408080808080809e-05, + "loss": 2.5631, + "theoretical_loss": 3.3157266488021433, + "tokens_seen": 3059941376 + }, + { + "epoch": 0.85, + "learning_rate": 7.406060606060606e-05, + "loss": 2.6438, + "theoretical_loss": 3.3157212107428804, + "tokens_seen": 3060006912 + }, + { + "epoch": 0.85, + "learning_rate": 7.404040404040404e-05, + "loss": 2.5856, + "theoretical_loss": 3.315715772832693, + "tokens_seen": 3060072448 + }, + { + "epoch": 0.85, + "learning_rate": 7.402020202020201e-05, + "loss": 2.4744, + "theoretical_loss": 3.3157103350715738, + "tokens_seen": 3060137984 + }, + { + "epoch": 0.85, + "learning_rate": 7.4e-05, + "loss": 2.7697, + "theoretical_loss": 3.315704897459515, + "tokens_seen": 3060203520 + }, + { + "epoch": 0.85, + "learning_rate": 7.397979797979798e-05, + "loss": 2.5886, + "theoretical_loss": 3.3156994599965093, + "tokens_seen": 3060269056 + }, + { + "epoch": 0.85, + "learning_rate": 7.395959595959596e-05, + "loss": 2.5348, + "theoretical_loss": 3.31569402268255, + "tokens_seen": 3060334592 + }, + { + "epoch": 0.85, + "learning_rate": 7.393939393939395e-05, + "loss": 2.5655, + "theoretical_loss": 3.3156885855176297, + "tokens_seen": 3060400128 + }, + { + "epoch": 0.85, + "learning_rate": 7.391919191919193e-05, + "loss": 2.707, + "theoretical_loss": 3.3156831485017406, + "tokens_seen": 3060465664 + }, + { + "epoch": 0.85, + "learning_rate": 7.38989898989899e-05, + "loss": 2.8169, + "theoretical_loss": 3.315677711634876, + "tokens_seen": 3060531200 + }, + { + "epoch": 0.85, + "learning_rate": 7.387878787878787e-05, + "loss": 2.5569, + "theoretical_loss": 3.3156722749170284, + "tokens_seen": 3060596736 + }, + { + "epoch": 0.85, + "learning_rate": 7.385858585858586e-05, + "loss": 2.7047, + "theoretical_loss": 3.31566683834819, + "tokens_seen": 3060662272 + }, + { + "epoch": 0.85, + "learning_rate": 7.383838383838384e-05, + "loss": 2.6003, + "theoretical_loss": 3.315661401928355, + "tokens_seen": 3060727808 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1721958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.84852933883667, + "objective/train/theoretical_loss": 3.315655965657515, + "objective/train/tokens_used": 1419652576, + "theoretical_loss": 3.315655965657515, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.85, + "learning_rate": 7.381818181818182e-05, + "loss": 2.6199, + "theoretical_loss": 3.315655965657515, + "tokens_seen": 3060793344 + }, + { + "epoch": 0.85, + "learning_rate": 7.37979797979798e-05, + "loss": 2.6653, + "theoretical_loss": 3.315650529535662, + "tokens_seen": 3060858880 + }, + { + "epoch": 0.85, + "learning_rate": 7.377777777777778e-05, + "loss": 2.6868, + "theoretical_loss": 3.315645093562791, + "tokens_seen": 3060924416 + }, + { + "epoch": 0.85, + "learning_rate": 7.375757575757576e-05, + "loss": 2.5566, + "theoretical_loss": 3.3156396577388922, + "tokens_seen": 3060989952 + }, + { + "epoch": 0.85, + "learning_rate": 7.373737373737375e-05, + "loss": 2.5121, + "theoretical_loss": 3.31563422206396, + "tokens_seen": 3061055488 + }, + { + "epoch": 0.85, + "learning_rate": 7.371717171717173e-05, + "loss": 2.6837, + "theoretical_loss": 3.3156287865379865, + "tokens_seen": 3061121024 + }, + { + "epoch": 0.85, + "learning_rate": 7.369696969696969e-05, + "loss": 2.8501, + "theoretical_loss": 3.3156233511609647, + "tokens_seen": 3061186560 + }, + { + "epoch": 0.85, + "learning_rate": 7.367676767676767e-05, + "loss": 2.4669, + "theoretical_loss": 3.315617915932887, + "tokens_seen": 3061252096 + }, + { + "epoch": 0.85, + "learning_rate": 7.365656565656566e-05, + "loss": 2.5069, + "theoretical_loss": 3.3156124808537464, + "tokens_seen": 3061317632 + }, + { + "epoch": 0.85, + "learning_rate": 7.363636363636364e-05, + "loss": 2.6653, + "theoretical_loss": 3.3156070459235356, + "tokens_seen": 3061383168 + }, + { + "epoch": 0.85, + "learning_rate": 7.361616161616162e-05, + "loss": 2.4708, + "theoretical_loss": 3.315601611142247, + "tokens_seen": 3061448704 + }, + { + "epoch": 0.85, + "learning_rate": 7.35959595959596e-05, + "loss": 2.6532, + "theoretical_loss": 3.315596176509874, + "tokens_seen": 3061514240 + }, + { + "epoch": 0.85, + "learning_rate": 7.357575757575758e-05, + "loss": 2.8093, + "theoretical_loss": 3.3155907420264086, + "tokens_seen": 3061579776 + }, + { + "epoch": 0.85, + "learning_rate": 7.355555555555556e-05, + "loss": 2.4367, + "theoretical_loss": 3.315585307691844, + "tokens_seen": 3061645312 + }, + { + "epoch": 0.85, + "learning_rate": 7.353535353535353e-05, + "loss": 2.6578, + "theoretical_loss": 3.315579873506173, + "tokens_seen": 3061710848 + }, + { + "epoch": 0.85, + "learning_rate": 7.351515151515152e-05, + "loss": 2.6986, + "theoretical_loss": 3.315574439469388, + "tokens_seen": 3061776384 + }, + { + "epoch": 0.85, + "learning_rate": 7.349494949494949e-05, + "loss": 2.7826, + "theoretical_loss": 3.3155690055814815, + "tokens_seen": 3061841920 + }, + { + "epoch": 0.85, + "learning_rate": 7.347474747474747e-05, + "loss": 2.7415, + "theoretical_loss": 3.315563571842447, + "tokens_seen": 3061907456 + }, + { + "epoch": 0.85, + "learning_rate": 7.345454545454545e-05, + "loss": 2.646, + "theoretical_loss": 3.3155581382522765, + "tokens_seen": 3061972992 + }, + { + "epoch": 0.85, + "learning_rate": 7.343434343434344e-05, + "loss": 2.4602, + "theoretical_loss": 3.3155527048109636, + "tokens_seen": 3062038528 + }, + { + "epoch": 0.85, + "learning_rate": 7.341414141414142e-05, + "loss": 2.4795, + "theoretical_loss": 3.3155472715185, + "tokens_seen": 3062104064 + }, + { + "epoch": 0.85, + "learning_rate": 7.33939393939394e-05, + "loss": 2.6254, + "theoretical_loss": 3.3155418383748794, + "tokens_seen": 3062169600 + }, + { + "epoch": 0.85, + "learning_rate": 7.337373737373738e-05, + "loss": 2.5225, + "theoretical_loss": 3.3155364053800938, + "tokens_seen": 3062235136 + }, + { + "epoch": 0.85, + "learning_rate": 7.335353535353535e-05, + "loss": 2.7076, + "theoretical_loss": 3.315530972534136, + "tokens_seen": 3062300672 + }, + { + "epoch": 0.85, + "learning_rate": 7.333333333333333e-05, + "loss": 2.6511, + "theoretical_loss": 3.3155255398369996, + "tokens_seen": 3062366208 + }, + { + "epoch": 0.85, + "objective/train/docs_used": 1723218, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5025761127471924, + "objective/train/theoretical_loss": 3.315520107288676, + "objective/train/tokens_used": 1421290976, + "theoretical_loss": 3.315520107288676, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.85, + "learning_rate": 7.331313131313132e-05, + "loss": 2.6487, + "theoretical_loss": 3.315520107288676, + "tokens_seen": 3062431744 + }, + { + "epoch": 0.86, + "learning_rate": 7.32929292929293e-05, + "loss": 2.5127, + "theoretical_loss": 3.315514674889159, + "tokens_seen": 3062497280 + }, + { + "epoch": 0.86, + "learning_rate": 7.327272727272727e-05, + "loss": 2.7899, + "theoretical_loss": 3.315509242638441, + "tokens_seen": 3062562816 + }, + { + "epoch": 0.86, + "learning_rate": 7.325252525252525e-05, + "loss": 2.6705, + "theoretical_loss": 3.315503810536515, + "tokens_seen": 3062628352 + }, + { + "epoch": 0.86, + "learning_rate": 7.323232323232324e-05, + "loss": 2.5905, + "theoretical_loss": 3.315498378583373, + "tokens_seen": 3062693888 + }, + { + "epoch": 0.86, + "learning_rate": 7.321212121212122e-05, + "loss": 2.7218, + "theoretical_loss": 3.315492946779008, + "tokens_seen": 3062759424 + }, + { + "epoch": 0.86, + "learning_rate": 7.319191919191919e-05, + "loss": 2.6441, + "theoretical_loss": 3.3154875151234133, + "tokens_seen": 3062824960 + }, + { + "epoch": 0.86, + "learning_rate": 7.317171717171716e-05, + "loss": 2.6762, + "theoretical_loss": 3.3154820836165815, + "tokens_seen": 3062890496 + }, + { + "epoch": 0.86, + "learning_rate": 7.315151515151515e-05, + "loss": 2.5566, + "theoretical_loss": 3.315476652258505, + "tokens_seen": 3062956032 + }, + { + "epoch": 0.86, + "learning_rate": 7.313131313131313e-05, + "loss": 2.5576, + "theoretical_loss": 3.3154712210491764, + "tokens_seen": 3063021568 + }, + { + "epoch": 0.86, + "learning_rate": 7.311111111111111e-05, + "loss": 2.6295, + "theoretical_loss": 3.315465789988589, + "tokens_seen": 3063087104 + }, + { + "epoch": 0.86, + "learning_rate": 7.30909090909091e-05, + "loss": 2.527, + "theoretical_loss": 3.315460359076735, + "tokens_seen": 3063152640 + }, + { + "epoch": 0.86, + "learning_rate": 7.307070707070707e-05, + "loss": 2.8058, + "theoretical_loss": 3.315454928313607, + "tokens_seen": 3063218176 + }, + { + "epoch": 0.86, + "learning_rate": 7.305050505050505e-05, + "loss": 2.4223, + "theoretical_loss": 3.315449497699199, + "tokens_seen": 3063283712 + }, + { + "epoch": 0.86, + "learning_rate": 7.303030303030304e-05, + "loss": 2.7294, + "theoretical_loss": 3.3154440672335026, + "tokens_seen": 3063349248 + }, + { + "epoch": 0.86, + "learning_rate": 7.301010101010101e-05, + "loss": 2.6285, + "theoretical_loss": 3.3154386369165105, + "tokens_seen": 3063414784 + }, + { + "epoch": 0.86, + "learning_rate": 7.298989898989899e-05, + "loss": 2.4642, + "theoretical_loss": 3.3154332067482164, + "tokens_seen": 3063480320 + }, + { + "epoch": 0.86, + "learning_rate": 7.296969696969697e-05, + "loss": 2.5499, + "theoretical_loss": 3.3154277767286118, + "tokens_seen": 3063545856 + }, + { + "epoch": 0.86, + "learning_rate": 7.294949494949495e-05, + "loss": 2.2594, + "theoretical_loss": 3.3154223468576904, + "tokens_seen": 3063611392 + }, + { + "epoch": 0.86, + "learning_rate": 7.292929292929293e-05, + "loss": 2.6492, + "theoretical_loss": 3.3154169171354444, + "tokens_seen": 3063676928 + }, + { + "epoch": 0.86, + "learning_rate": 7.290909090909091e-05, + "loss": 2.6343, + "theoretical_loss": 3.3154114875618674, + "tokens_seen": 3063742464 + }, + { + "epoch": 0.86, + "learning_rate": 7.28888888888889e-05, + "loss": 2.7558, + "theoretical_loss": 3.3154060581369507, + "tokens_seen": 3063808000 + }, + { + "epoch": 0.86, + "learning_rate": 7.286868686868688e-05, + "loss": 2.5384, + "theoretical_loss": 3.3154006288606883, + "tokens_seen": 3063873536 + }, + { + "epoch": 0.86, + "learning_rate": 7.284848484848484e-05, + "loss": 2.5463, + "theoretical_loss": 3.315395199733073, + "tokens_seen": 3063939072 + }, + { + "epoch": 0.86, + "learning_rate": 7.282828282828282e-05, + "loss": 2.5247, + "theoretical_loss": 3.3153897707540962, + "tokens_seen": 3064004608 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1723856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5429117679595947, + "objective/train/theoretical_loss": 3.315384341923752, + "objective/train/tokens_used": 1422929376, + "theoretical_loss": 3.315384341923752, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.86, + "learning_rate": 7.28080808080808e-05, + "loss": 2.4729, + "theoretical_loss": 3.315384341923752, + "tokens_seen": 3064070144 + }, + { + "epoch": 0.86, + "learning_rate": 7.278787878787879e-05, + "loss": 2.6457, + "theoretical_loss": 3.3153789132420326, + "tokens_seen": 3064135680 + }, + { + "epoch": 0.86, + "learning_rate": 7.276767676767677e-05, + "loss": 2.5074, + "theoretical_loss": 3.315373484708931, + "tokens_seen": 3064201216 + }, + { + "epoch": 0.86, + "learning_rate": 7.274747474747474e-05, + "loss": 2.5803, + "theoretical_loss": 3.3153680563244396, + "tokens_seen": 3064266752 + }, + { + "epoch": 0.86, + "learning_rate": 7.272727272727273e-05, + "loss": 2.7852, + "theoretical_loss": 3.3153626280885513, + "tokens_seen": 3064332288 + }, + { + "epoch": 0.86, + "learning_rate": 7.270707070707071e-05, + "loss": 2.5301, + "theoretical_loss": 3.3153572000012588, + "tokens_seen": 3064397824 + }, + { + "epoch": 0.86, + "learning_rate": 7.26868686868687e-05, + "loss": 2.4684, + "theoretical_loss": 3.315351772062555, + "tokens_seen": 3064463360 + }, + { + "epoch": 0.86, + "learning_rate": 7.266666666666667e-05, + "loss": 2.5276, + "theoretical_loss": 3.3153463442724327, + "tokens_seen": 3064528896 + }, + { + "epoch": 0.86, + "learning_rate": 7.264646464646464e-05, + "loss": 2.3983, + "theoretical_loss": 3.315340916630885, + "tokens_seen": 3064594432 + }, + { + "epoch": 0.86, + "learning_rate": 7.262626262626262e-05, + "loss": 2.6415, + "theoretical_loss": 3.3153354891379037, + "tokens_seen": 3064659968 + }, + { + "epoch": 0.86, + "learning_rate": 7.26060606060606e-05, + "loss": 2.1528, + "theoretical_loss": 3.3153300617934818, + "tokens_seen": 3064725504 + }, + { + "epoch": 0.86, + "learning_rate": 7.258585858585859e-05, + "loss": 2.528, + "theoretical_loss": 3.315324634597613, + "tokens_seen": 3064791040 + }, + { + "epoch": 0.86, + "learning_rate": 7.256565656565657e-05, + "loss": 2.7119, + "theoretical_loss": 3.3153192075502886, + "tokens_seen": 3064856576 + }, + { + "epoch": 0.86, + "learning_rate": 7.254545454545456e-05, + "loss": 2.6368, + "theoretical_loss": 3.3153137806515027, + "tokens_seen": 3064922112 + }, + { + "epoch": 0.86, + "learning_rate": 7.252525252525253e-05, + "loss": 2.6861, + "theoretical_loss": 3.315308353901248, + "tokens_seen": 3064987648 + }, + { + "epoch": 0.86, + "learning_rate": 7.25050505050505e-05, + "loss": 2.4701, + "theoretical_loss": 3.315302927299516, + "tokens_seen": 3065053184 + }, + { + "epoch": 0.86, + "learning_rate": 7.248484848484848e-05, + "loss": 2.8615, + "theoretical_loss": 3.3152975008463, + "tokens_seen": 3065118720 + }, + { + "epoch": 0.86, + "learning_rate": 7.246464646464647e-05, + "loss": 2.6372, + "theoretical_loss": 3.3152920745415937, + "tokens_seen": 3065184256 + }, + { + "epoch": 0.86, + "learning_rate": 7.244444444444445e-05, + "loss": 2.547, + "theoretical_loss": 3.3152866483853884, + "tokens_seen": 3065249792 + }, + { + "epoch": 0.86, + "learning_rate": 7.242424242424242e-05, + "loss": 2.4561, + "theoretical_loss": 3.315281222377678, + "tokens_seen": 3065315328 + }, + { + "epoch": 0.86, + "learning_rate": 7.24040404040404e-05, + "loss": 2.5778, + "theoretical_loss": 3.3152757965184554, + "tokens_seen": 3065380864 + }, + { + "epoch": 0.86, + "learning_rate": 7.238383838383839e-05, + "loss": 2.4373, + "theoretical_loss": 3.315270370807712, + "tokens_seen": 3065446400 + }, + { + "epoch": 0.86, + "learning_rate": 7.236363636363637e-05, + "loss": 2.6064, + "theoretical_loss": 3.315264945245442, + "tokens_seen": 3065511936 + }, + { + "epoch": 0.86, + "learning_rate": 7.234343434343436e-05, + "loss": 2.7583, + "theoretical_loss": 3.315259519831637, + "tokens_seen": 3065577472 + }, + { + "epoch": 0.86, + "learning_rate": 7.232323232323231e-05, + "loss": 2.6768, + "theoretical_loss": 3.3152540945662907, + "tokens_seen": 3065643008 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1724430, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6072494983673096, + "objective/train/theoretical_loss": 3.3152486694493954, + "objective/train/tokens_used": 1424567776, + "theoretical_loss": 3.3152486694493954, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 7.23030303030303e-05, + "loss": 2.6755, + "theoretical_loss": 3.3152486694493954, + "tokens_seen": 3065708544 + }, + { + "epoch": 0.86, + "learning_rate": 7.228282828282828e-05, + "loss": 2.7144, + "theoretical_loss": 3.3152432444809437, + "tokens_seen": 3065774080 + }, + { + "epoch": 0.86, + "learning_rate": 7.226262626262626e-05, + "loss": 2.5858, + "theoretical_loss": 3.315237819660929, + "tokens_seen": 3065839616 + }, + { + "epoch": 0.86, + "learning_rate": 7.224242424242425e-05, + "loss": 2.5821, + "theoretical_loss": 3.3152323949893434, + "tokens_seen": 3065905152 + }, + { + "epoch": 0.86, + "learning_rate": 7.222222222222222e-05, + "loss": 2.7025, + "theoretical_loss": 3.3152269704661803, + "tokens_seen": 3065970688 + }, + { + "epoch": 0.86, + "learning_rate": 7.22020202020202e-05, + "loss": 2.3565, + "theoretical_loss": 3.315221546091432, + "tokens_seen": 3066036224 + }, + { + "epoch": 0.86, + "learning_rate": 7.218181818181819e-05, + "loss": 2.6329, + "theoretical_loss": 3.315216121865091, + "tokens_seen": 3066101760 + }, + { + "epoch": 0.86, + "learning_rate": 7.216161616161617e-05, + "loss": 2.7952, + "theoretical_loss": 3.3152106977871503, + "tokens_seen": 3066167296 + }, + { + "epoch": 0.86, + "learning_rate": 7.214141414141414e-05, + "loss": 2.6554, + "theoretical_loss": 3.3152052738576034, + "tokens_seen": 3066232832 + }, + { + "epoch": 0.86, + "learning_rate": 7.212121212121213e-05, + "loss": 2.6568, + "theoretical_loss": 3.3151998500764424, + "tokens_seen": 3066298368 + }, + { + "epoch": 0.86, + "learning_rate": 7.21010101010101e-05, + "loss": 2.787, + "theoretical_loss": 3.31519442644366, + "tokens_seen": 3066363904 + }, + { + "epoch": 0.86, + "learning_rate": 7.208080808080808e-05, + "loss": 2.6514, + "theoretical_loss": 3.315189002959249, + "tokens_seen": 3066429440 + }, + { + "epoch": 0.86, + "learning_rate": 7.206060606060606e-05, + "loss": 2.4714, + "theoretical_loss": 3.3151835796232025, + "tokens_seen": 3066494976 + }, + { + "epoch": 0.86, + "learning_rate": 7.204040404040405e-05, + "loss": 2.4914, + "theoretical_loss": 3.315178156435513, + "tokens_seen": 3066560512 + }, + { + "epoch": 0.86, + "learning_rate": 7.202020202020203e-05, + "loss": 2.7631, + "theoretical_loss": 3.315172733396173, + "tokens_seen": 3066626048 + }, + { + "epoch": 0.86, + "learning_rate": 7.2e-05, + "loss": 2.6681, + "theoretical_loss": 3.315167310505176, + "tokens_seen": 3066691584 + }, + { + "epoch": 0.86, + "learning_rate": 7.197979797979797e-05, + "loss": 2.5457, + "theoretical_loss": 3.3151618877625144, + "tokens_seen": 3066757120 + }, + { + "epoch": 0.86, + "learning_rate": 7.195959595959596e-05, + "loss": 2.5882, + "theoretical_loss": 3.315156465168181, + "tokens_seen": 3066822656 + }, + { + "epoch": 0.86, + "learning_rate": 7.193939393939394e-05, + "loss": 2.5186, + "theoretical_loss": 3.315151042722168, + "tokens_seen": 3066888192 + }, + { + "epoch": 0.86, + "learning_rate": 7.191919191919192e-05, + "loss": 2.4872, + "theoretical_loss": 3.3151456204244694, + "tokens_seen": 3066953728 + }, + { + "epoch": 0.86, + "learning_rate": 7.18989898989899e-05, + "loss": 2.4681, + "theoretical_loss": 3.3151401982750768, + "tokens_seen": 3067019264 + }, + { + "epoch": 0.86, + "learning_rate": 7.187878787878788e-05, + "loss": 2.7059, + "theoretical_loss": 3.3151347762739833, + "tokens_seen": 3067084800 + }, + { + "epoch": 0.86, + "learning_rate": 7.185858585858586e-05, + "loss": 2.4926, + "theoretical_loss": 3.315129354421182, + "tokens_seen": 3067150336 + }, + { + "epoch": 0.86, + "learning_rate": 7.183838383838385e-05, + "loss": 2.4015, + "theoretical_loss": 3.3151239327166655, + "tokens_seen": 3067215872 + }, + { + "epoch": 0.86, + "learning_rate": 7.181818181818183e-05, + "loss": 2.6326, + "theoretical_loss": 3.315118511160427, + "tokens_seen": 3067281408 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1725702, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.825979709625244, + "objective/train/theoretical_loss": 3.315113089752458, + "objective/train/tokens_used": 1426206176, + "theoretical_loss": 3.315113089752458, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.86, + "learning_rate": 7.179797979797979e-05, + "loss": 2.5926, + "theoretical_loss": 3.315113089752458, + "tokens_seen": 3067346944 + }, + { + "epoch": 0.86, + "learning_rate": 7.177777777777777e-05, + "loss": 2.3682, + "theoretical_loss": 3.315107668492753, + "tokens_seen": 3067412480 + }, + { + "epoch": 0.86, + "learning_rate": 7.175757575757576e-05, + "loss": 2.5711, + "theoretical_loss": 3.3151022473813034, + "tokens_seen": 3067478016 + }, + { + "epoch": 0.86, + "learning_rate": 7.173737373737374e-05, + "loss": 2.5619, + "theoretical_loss": 3.3150968264181024, + "tokens_seen": 3067543552 + }, + { + "epoch": 0.86, + "learning_rate": 7.171717171717172e-05, + "loss": 2.7908, + "theoretical_loss": 3.3150914056031433, + "tokens_seen": 3067609088 + }, + { + "epoch": 0.86, + "learning_rate": 7.169696969696971e-05, + "loss": 2.5621, + "theoretical_loss": 3.3150859849364185, + "tokens_seen": 3067674624 + }, + { + "epoch": 0.86, + "learning_rate": 7.167676767676768e-05, + "loss": 2.5345, + "theoretical_loss": 3.31508056441792, + "tokens_seen": 3067740160 + }, + { + "epoch": 0.86, + "learning_rate": 7.165656565656566e-05, + "loss": 2.5004, + "theoretical_loss": 3.3150751440476416, + "tokens_seen": 3067805696 + }, + { + "epoch": 0.86, + "learning_rate": 7.163636363636363e-05, + "loss": 2.5867, + "theoretical_loss": 3.315069723825576, + "tokens_seen": 3067871232 + }, + { + "epoch": 0.86, + "learning_rate": 7.161616161616162e-05, + "loss": 2.4135, + "theoretical_loss": 3.315064303751716, + "tokens_seen": 3067936768 + }, + { + "epoch": 0.86, + "learning_rate": 7.15959595959596e-05, + "loss": 2.4593, + "theoretical_loss": 3.3150588838260537, + "tokens_seen": 3068002304 + }, + { + "epoch": 0.86, + "learning_rate": 7.157575757575757e-05, + "loss": 2.5037, + "theoretical_loss": 3.3150534640485825, + "tokens_seen": 3068067840 + }, + { + "epoch": 0.86, + "learning_rate": 7.155555555555555e-05, + "loss": 2.5962, + "theoretical_loss": 3.315048044419295, + "tokens_seen": 3068133376 + }, + { + "epoch": 0.86, + "learning_rate": 7.153535353535354e-05, + "loss": 2.5872, + "theoretical_loss": 3.315042624938184, + "tokens_seen": 3068198912 + }, + { + "epoch": 0.86, + "learning_rate": 7.151515151515152e-05, + "loss": 2.7011, + "theoretical_loss": 3.315037205605242, + "tokens_seen": 3068264448 + }, + { + "epoch": 0.86, + "learning_rate": 7.14949494949495e-05, + "loss": 2.5392, + "theoretical_loss": 3.315031786420462, + "tokens_seen": 3068329984 + }, + { + "epoch": 0.86, + "learning_rate": 7.147474747474748e-05, + "loss": 2.7072, + "theoretical_loss": 3.315026367383837, + "tokens_seen": 3068395520 + }, + { + "epoch": 0.86, + "learning_rate": 7.145454545454545e-05, + "loss": 2.5418, + "theoretical_loss": 3.31502094849536, + "tokens_seen": 3068461056 + }, + { + "epoch": 0.86, + "learning_rate": 7.143434343434343e-05, + "loss": 2.608, + "theoretical_loss": 3.315015529755023, + "tokens_seen": 3068526592 + }, + { + "epoch": 0.86, + "learning_rate": 7.141414141414141e-05, + "loss": 2.5049, + "theoretical_loss": 3.3150101111628194, + "tokens_seen": 3068592128 + }, + { + "epoch": 0.86, + "learning_rate": 7.13939393939394e-05, + "loss": 2.5725, + "theoretical_loss": 3.315004692718742, + "tokens_seen": 3068657664 + }, + { + "epoch": 0.86, + "learning_rate": 7.137373737373737e-05, + "loss": 2.389, + "theoretical_loss": 3.314999274422783, + "tokens_seen": 3068723200 + }, + { + "epoch": 0.86, + "learning_rate": 7.135353535353535e-05, + "loss": 2.617, + "theoretical_loss": 3.3149938562749357, + "tokens_seen": 3068788736 + }, + { + "epoch": 0.86, + "learning_rate": 7.133333333333334e-05, + "loss": 2.633, + "theoretical_loss": 3.3149884382751926, + "tokens_seen": 3068854272 + }, + { + "epoch": 0.86, + "learning_rate": 7.131313131313132e-05, + "loss": 2.6073, + "theoretical_loss": 3.3149830204235466, + "tokens_seen": 3068919808 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1726520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6154985427856445, + "objective/train/theoretical_loss": 3.314977602719991, + "objective/train/tokens_used": 1427844576, + "theoretical_loss": 3.314977602719991, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 7.12929292929293e-05, + "loss": 2.5718, + "theoretical_loss": 3.314977602719991, + "tokens_seen": 3068985344 + }, + { + "epoch": 0.86, + "learning_rate": 7.127272727272728e-05, + "loss": 2.5137, + "theoretical_loss": 3.314972185164518, + "tokens_seen": 3069050880 + }, + { + "epoch": 0.86, + "learning_rate": 7.125252525252525e-05, + "loss": 2.404, + "theoretical_loss": 3.31496676775712, + "tokens_seen": 3069116416 + }, + { + "epoch": 0.86, + "learning_rate": 7.123232323232323e-05, + "loss": 2.574, + "theoretical_loss": 3.314961350497791, + "tokens_seen": 3069181952 + }, + { + "epoch": 0.86, + "learning_rate": 7.121212121212121e-05, + "loss": 2.6383, + "theoretical_loss": 3.3149559333865226, + "tokens_seen": 3069247488 + }, + { + "epoch": 0.86, + "learning_rate": 7.11919191919192e-05, + "loss": 2.5715, + "theoretical_loss": 3.314950516423308, + "tokens_seen": 3069313024 + }, + { + "epoch": 0.86, + "learning_rate": 7.117171717171718e-05, + "loss": 2.6663, + "theoretical_loss": 3.3149450996081407, + "tokens_seen": 3069378560 + }, + { + "epoch": 0.86, + "learning_rate": 7.115151515151515e-05, + "loss": 2.6999, + "theoretical_loss": 3.314939682941012, + "tokens_seen": 3069444096 + }, + { + "epoch": 0.86, + "learning_rate": 7.113131313131314e-05, + "loss": 2.4541, + "theoretical_loss": 3.3149342664219166, + "tokens_seen": 3069509632 + }, + { + "epoch": 0.86, + "learning_rate": 7.11111111111111e-05, + "loss": 2.3823, + "theoretical_loss": 3.3149288500508454, + "tokens_seen": 3069575168 + }, + { + "epoch": 0.86, + "learning_rate": 7.109090909090909e-05, + "loss": 2.5848, + "theoretical_loss": 3.3149234338277926, + "tokens_seen": 3069640704 + }, + { + "epoch": 0.86, + "learning_rate": 7.107070707070707e-05, + "loss": 2.5559, + "theoretical_loss": 3.31491801775275, + "tokens_seen": 3069706240 + }, + { + "epoch": 0.86, + "learning_rate": 7.105050505050504e-05, + "loss": 2.5372, + "theoretical_loss": 3.314912601825711, + "tokens_seen": 3069771776 + }, + { + "epoch": 0.86, + "learning_rate": 7.103030303030303e-05, + "loss": 2.5754, + "theoretical_loss": 3.3149071860466686, + "tokens_seen": 3069837312 + }, + { + "epoch": 0.86, + "learning_rate": 7.101010101010101e-05, + "loss": 2.3775, + "theoretical_loss": 3.3149017704156147, + "tokens_seen": 3069902848 + }, + { + "epoch": 0.86, + "learning_rate": 7.0989898989899e-05, + "loss": 2.537, + "theoretical_loss": 3.314896354932543, + "tokens_seen": 3069968384 + }, + { + "epoch": 0.86, + "learning_rate": 7.096969696969698e-05, + "loss": 2.3072, + "theoretical_loss": 3.3148909395974457, + "tokens_seen": 3070033920 + }, + { + "epoch": 0.86, + "learning_rate": 7.094949494949495e-05, + "loss": 2.8895, + "theoretical_loss": 3.314885524410316, + "tokens_seen": 3070099456 + }, + { + "epoch": 0.86, + "learning_rate": 7.092929292929292e-05, + "loss": 2.6156, + "theoretical_loss": 3.3148801093711464, + "tokens_seen": 3070164992 + }, + { + "epoch": 0.86, + "learning_rate": 7.09090909090909e-05, + "loss": 2.7254, + "theoretical_loss": 3.31487469447993, + "tokens_seen": 3070230528 + }, + { + "epoch": 0.86, + "learning_rate": 7.088888888888889e-05, + "loss": 2.3891, + "theoretical_loss": 3.314869279736659, + "tokens_seen": 3070296064 + }, + { + "epoch": 0.86, + "learning_rate": 7.086868686868687e-05, + "loss": 2.5472, + "theoretical_loss": 3.3148638651413274, + "tokens_seen": 3070361600 + }, + { + "epoch": 0.86, + "learning_rate": 7.084848484848486e-05, + "loss": 2.4015, + "theoretical_loss": 3.3148584506939267, + "tokens_seen": 3070427136 + }, + { + "epoch": 0.86, + "learning_rate": 7.082828282828283e-05, + "loss": 2.5958, + "theoretical_loss": 3.31485303639445, + "tokens_seen": 3070492672 + }, + { + "epoch": 0.86, + "learning_rate": 7.080808080808081e-05, + "loss": 2.4489, + "theoretical_loss": 3.3148476222428904, + "tokens_seen": 3070558208 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1727712, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.597968816757202, + "objective/train/theoretical_loss": 3.314842208239241, + "objective/train/tokens_used": 1429482976, + "theoretical_loss": 3.314842208239241, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.86, + "learning_rate": 7.07878787878788e-05, + "loss": 2.631, + "theoretical_loss": 3.314842208239241, + "tokens_seen": 3070623744 + }, + { + "epoch": 0.86, + "learning_rate": 7.076767676767677e-05, + "loss": 2.4256, + "theoretical_loss": 3.314836794383494, + "tokens_seen": 3070689280 + }, + { + "epoch": 0.86, + "learning_rate": 7.074747474747475e-05, + "loss": 2.5868, + "theoretical_loss": 3.314831380675643, + "tokens_seen": 3070754816 + }, + { + "epoch": 0.86, + "learning_rate": 7.072727272727272e-05, + "loss": 2.4408, + "theoretical_loss": 3.3148259671156795, + "tokens_seen": 3070820352 + }, + { + "epoch": 0.86, + "learning_rate": 7.07070707070707e-05, + "loss": 2.6528, + "theoretical_loss": 3.3148205537035973, + "tokens_seen": 3070885888 + }, + { + "epoch": 0.86, + "learning_rate": 7.068686868686869e-05, + "loss": 2.4332, + "theoretical_loss": 3.314815140439389, + "tokens_seen": 3070951424 + }, + { + "epoch": 0.86, + "learning_rate": 7.066666666666667e-05, + "loss": 2.4596, + "theoretical_loss": 3.3148097273230475, + "tokens_seen": 3071016960 + }, + { + "epoch": 0.86, + "learning_rate": 7.064646464646466e-05, + "loss": 2.6333, + "theoretical_loss": 3.314804314354565, + "tokens_seen": 3071082496 + }, + { + "epoch": 0.86, + "learning_rate": 7.062626262626263e-05, + "loss": 2.5228, + "theoretical_loss": 3.314798901533935, + "tokens_seen": 3071148032 + }, + { + "epoch": 0.86, + "learning_rate": 7.060606060606061e-05, + "loss": 2.4982, + "theoretical_loss": 3.31479348886115, + "tokens_seen": 3071213568 + }, + { + "epoch": 0.86, + "learning_rate": 7.058585858585858e-05, + "loss": 2.5979, + "theoretical_loss": 3.314788076336203, + "tokens_seen": 3071279104 + }, + { + "epoch": 0.86, + "learning_rate": 7.056565656565656e-05, + "loss": 2.4765, + "theoretical_loss": 3.314782663959087, + "tokens_seen": 3071344640 + }, + { + "epoch": 0.86, + "learning_rate": 7.054545454545455e-05, + "loss": 2.547, + "theoretical_loss": 3.314777251729794, + "tokens_seen": 3071410176 + }, + { + "epoch": 0.86, + "learning_rate": 7.052525252525252e-05, + "loss": 2.4244, + "theoretical_loss": 3.3147718396483175, + "tokens_seen": 3071475712 + }, + { + "epoch": 0.86, + "learning_rate": 7.05050505050505e-05, + "loss": 2.5691, + "theoretical_loss": 3.31476642771465, + "tokens_seen": 3071541248 + }, + { + "epoch": 0.86, + "learning_rate": 7.048484848484849e-05, + "loss": 2.7893, + "theoretical_loss": 3.314761015928785, + "tokens_seen": 3071606784 + }, + { + "epoch": 0.86, + "learning_rate": 7.046464646464647e-05, + "loss": 2.5618, + "theoretical_loss": 3.314755604290714, + "tokens_seen": 3071672320 + }, + { + "epoch": 0.86, + "learning_rate": 7.044444444444446e-05, + "loss": 2.4691, + "theoretical_loss": 3.314750192800431, + "tokens_seen": 3071737856 + }, + { + "epoch": 0.86, + "learning_rate": 7.042424242424244e-05, + "loss": 2.6603, + "theoretical_loss": 3.314744781457928, + "tokens_seen": 3071803392 + }, + { + "epoch": 0.86, + "learning_rate": 7.04040404040404e-05, + "loss": 2.7123, + "theoretical_loss": 3.314739370263198, + "tokens_seen": 3071868928 + }, + { + "epoch": 0.86, + "learning_rate": 7.038383838383838e-05, + "loss": 2.7661, + "theoretical_loss": 3.3147339592162344, + "tokens_seen": 3071934464 + }, + { + "epoch": 0.86, + "learning_rate": 7.036363636363636e-05, + "loss": 2.5926, + "theoretical_loss": 3.3147285483170297, + "tokens_seen": 3072000000 + }, + { + "epoch": 0.86, + "learning_rate": 7.034343434343435e-05, + "loss": 2.5851, + "theoretical_loss": 3.314723137565576, + "tokens_seen": 3072065536 + }, + { + "epoch": 0.86, + "learning_rate": 7.032323232323233e-05, + "loss": 2.5035, + "theoretical_loss": 3.3147177269618675, + "tokens_seen": 3072131072 + }, + { + "epoch": 0.86, + "learning_rate": 7.03030303030303e-05, + "loss": 2.5458, + "theoretical_loss": 3.3147123165058954, + "tokens_seen": 3072196608 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1728325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.156641721725464, + "objective/train/theoretical_loss": 3.314706906197654, + "objective/train/tokens_used": 1431121376, + "theoretical_loss": 3.314706906197654, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 7.028282828282829e-05, + "loss": 2.7043, + "theoretical_loss": 3.314706906197654, + "tokens_seen": 3072262144 + }, + { + "epoch": 0.86, + "learning_rate": 7.026262626262627e-05, + "loss": 2.5777, + "theoretical_loss": 3.3147014960371353, + "tokens_seen": 3072327680 + }, + { + "epoch": 0.86, + "learning_rate": 7.024242424242424e-05, + "loss": 2.4969, + "theoretical_loss": 3.314696086024332, + "tokens_seen": 3072393216 + }, + { + "epoch": 0.86, + "learning_rate": 7.022222222222222e-05, + "loss": 2.7109, + "theoretical_loss": 3.3146906761592376, + "tokens_seen": 3072458752 + }, + { + "epoch": 0.86, + "learning_rate": 7.02020202020202e-05, + "loss": 2.5999, + "theoretical_loss": 3.3146852664418445, + "tokens_seen": 3072524288 + }, + { + "epoch": 0.86, + "learning_rate": 7.018181818181818e-05, + "loss": 2.583, + "theoretical_loss": 3.314679856872145, + "tokens_seen": 3072589824 + }, + { + "epoch": 0.86, + "learning_rate": 7.016161616161616e-05, + "loss": 2.6229, + "theoretical_loss": 3.314674447450133, + "tokens_seen": 3072655360 + }, + { + "epoch": 0.86, + "learning_rate": 7.014141414141415e-05, + "loss": 2.4547, + "theoretical_loss": 3.3146690381758006, + "tokens_seen": 3072720896 + }, + { + "epoch": 0.86, + "learning_rate": 7.012121212121213e-05, + "loss": 2.6727, + "theoretical_loss": 3.3146636290491407, + "tokens_seen": 3072786432 + }, + { + "epoch": 0.86, + "learning_rate": 7.01010101010101e-05, + "loss": 2.4919, + "theoretical_loss": 3.314658220070146, + "tokens_seen": 3072851968 + }, + { + "epoch": 0.86, + "learning_rate": 7.008080808080807e-05, + "loss": 2.6548, + "theoretical_loss": 3.3146528112388096, + "tokens_seen": 3072917504 + }, + { + "epoch": 0.86, + "learning_rate": 7.006060606060606e-05, + "loss": 2.6758, + "theoretical_loss": 3.3146474025551247, + "tokens_seen": 3072983040 + }, + { + "epoch": 0.86, + "learning_rate": 7.004040404040404e-05, + "loss": 2.6393, + "theoretical_loss": 3.3146419940190834, + "tokens_seen": 3073048576 + }, + { + "epoch": 0.86, + "learning_rate": 7.002020202020202e-05, + "loss": 2.6339, + "theoretical_loss": 3.3146365856306783, + "tokens_seen": 3073114112 + }, + { + "epoch": 0.86, + "learning_rate": 7.000000000000001e-05, + "loss": 2.5468, + "theoretical_loss": 3.3146311773899035, + "tokens_seen": 3073179648 + }, + { + "epoch": 0.86, + "learning_rate": 6.997979797979798e-05, + "loss": 2.4748, + "theoretical_loss": 3.3146257692967507, + "tokens_seen": 3073245184 + }, + { + "epoch": 0.86, + "learning_rate": 6.995959595959596e-05, + "loss": 2.6086, + "theoretical_loss": 3.314620361351213, + "tokens_seen": 3073310720 + }, + { + "epoch": 0.86, + "learning_rate": 6.993939393939395e-05, + "loss": 2.7157, + "theoretical_loss": 3.314614953553283, + "tokens_seen": 3073376256 + }, + { + "epoch": 0.86, + "learning_rate": 6.991919191919193e-05, + "loss": 2.5982, + "theoretical_loss": 3.3146095459029543, + "tokens_seen": 3073441792 + }, + { + "epoch": 0.86, + "learning_rate": 6.98989898989899e-05, + "loss": 2.6368, + "theoretical_loss": 3.314604138400219, + "tokens_seen": 3073507328 + }, + { + "epoch": 0.86, + "learning_rate": 6.987878787878787e-05, + "loss": 2.4817, + "theoretical_loss": 3.31459873104507, + "tokens_seen": 3073572864 + }, + { + "epoch": 0.86, + "learning_rate": 6.985858585858585e-05, + "loss": 2.5915, + "theoretical_loss": 3.3145933238375003, + "tokens_seen": 3073638400 + }, + { + "epoch": 0.86, + "learning_rate": 6.983838383838384e-05, + "loss": 2.685, + "theoretical_loss": 3.314587916777503, + "tokens_seen": 3073703936 + }, + { + "epoch": 0.86, + "learning_rate": 6.981818181818182e-05, + "loss": 2.4038, + "theoretical_loss": 3.3145825098650703, + "tokens_seen": 3073769472 + }, + { + "epoch": 0.86, + "learning_rate": 6.97979797979798e-05, + "loss": 2.5093, + "theoretical_loss": 3.314577103100196, + "tokens_seen": 3073835008 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1729584, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9323203563690186, + "objective/train/theoretical_loss": 3.3145716964828713, + "objective/train/tokens_used": 1432759776, + "theoretical_loss": 3.3145716964828713, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.86, + "learning_rate": 6.977777777777778e-05, + "loss": 2.7178, + "theoretical_loss": 3.3145716964828713, + "tokens_seen": 3073900544 + }, + { + "epoch": 0.86, + "learning_rate": 6.975757575757576e-05, + "loss": 2.5172, + "theoretical_loss": 3.3145662900130906, + "tokens_seen": 3073966080 + }, + { + "epoch": 0.86, + "learning_rate": 6.973737373737374e-05, + "loss": 2.6269, + "theoretical_loss": 3.314560883690846, + "tokens_seen": 3074031616 + }, + { + "epoch": 0.86, + "learning_rate": 6.971717171717172e-05, + "loss": 2.4961, + "theoretical_loss": 3.31455547751613, + "tokens_seen": 3074097152 + }, + { + "epoch": 0.86, + "learning_rate": 6.96969696969697e-05, + "loss": 2.6387, + "theoretical_loss": 3.3145500714889367, + "tokens_seen": 3074162688 + }, + { + "epoch": 0.86, + "learning_rate": 6.967676767676767e-05, + "loss": 2.6716, + "theoretical_loss": 3.3145446656092576, + "tokens_seen": 3074228224 + }, + { + "epoch": 0.86, + "learning_rate": 6.965656565656565e-05, + "loss": 2.4121, + "theoretical_loss": 3.3145392598770864, + "tokens_seen": 3074293760 + }, + { + "epoch": 0.86, + "learning_rate": 6.963636363636364e-05, + "loss": 2.6636, + "theoretical_loss": 3.3145338542924154, + "tokens_seen": 3074359296 + }, + { + "epoch": 0.86, + "learning_rate": 6.961616161616162e-05, + "loss": 2.5832, + "theoretical_loss": 3.3145284488552376, + "tokens_seen": 3074424832 + }, + { + "epoch": 0.86, + "learning_rate": 6.95959595959596e-05, + "loss": 2.6935, + "theoretical_loss": 3.3145230435655457, + "tokens_seen": 3074490368 + }, + { + "epoch": 0.86, + "learning_rate": 6.957575757575759e-05, + "loss": 2.5005, + "theoretical_loss": 3.314517638423333, + "tokens_seen": 3074555904 + }, + { + "epoch": 0.86, + "learning_rate": 6.955555555555555e-05, + "loss": 2.4382, + "theoretical_loss": 3.3145122334285917, + "tokens_seen": 3074621440 + }, + { + "epoch": 0.86, + "learning_rate": 6.953535353535353e-05, + "loss": 2.5086, + "theoretical_loss": 3.314506828581315, + "tokens_seen": 3074686976 + }, + { + "epoch": 0.86, + "learning_rate": 6.951515151515151e-05, + "loss": 2.4555, + "theoretical_loss": 3.3145014238814956, + "tokens_seen": 3074752512 + }, + { + "epoch": 0.86, + "learning_rate": 6.94949494949495e-05, + "loss": 2.385, + "theoretical_loss": 3.3144960193291264, + "tokens_seen": 3074818048 + }, + { + "epoch": 0.86, + "learning_rate": 6.947474747474748e-05, + "loss": 2.5224, + "theoretical_loss": 3.3144906149242006, + "tokens_seen": 3074883584 + }, + { + "epoch": 0.86, + "learning_rate": 6.945454545454545e-05, + "loss": 2.389, + "theoretical_loss": 3.31448521066671, + "tokens_seen": 3074949120 + }, + { + "epoch": 0.86, + "learning_rate": 6.943434343434344e-05, + "loss": 2.6187, + "theoretical_loss": 3.314479806556649, + "tokens_seen": 3075014656 + }, + { + "epoch": 0.86, + "learning_rate": 6.941414141414142e-05, + "loss": 2.6612, + "theoretical_loss": 3.314474402594009, + "tokens_seen": 3075080192 + }, + { + "epoch": 0.86, + "learning_rate": 6.93939393939394e-05, + "loss": 2.383, + "theoretical_loss": 3.3144689987787834, + "tokens_seen": 3075145728 + }, + { + "epoch": 0.86, + "learning_rate": 6.937373737373737e-05, + "loss": 2.5475, + "theoretical_loss": 3.3144635951109653, + "tokens_seen": 3075211264 + }, + { + "epoch": 0.86, + "learning_rate": 6.935353535353535e-05, + "loss": 2.6183, + "theoretical_loss": 3.314458191590547, + "tokens_seen": 3075276800 + }, + { + "epoch": 0.86, + "learning_rate": 6.933333333333333e-05, + "loss": 2.6775, + "theoretical_loss": 3.3144527882175217, + "tokens_seen": 3075342336 + }, + { + "epoch": 0.86, + "learning_rate": 6.931313131313131e-05, + "loss": 2.6655, + "theoretical_loss": 3.3144473849918823, + "tokens_seen": 3075407872 + }, + { + "epoch": 0.86, + "learning_rate": 6.92929292929293e-05, + "loss": 2.4272, + "theoretical_loss": 3.314441981913621, + "tokens_seen": 3075473408 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1730199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2659480571746826, + "objective/train/theoretical_loss": 3.3144365789827317, + "objective/train/tokens_used": 1434398176, + "theoretical_loss": 3.3144365789827317, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 6.927272727272728e-05, + "loss": 2.4386, + "theoretical_loss": 3.3144365789827317, + "tokens_seen": 3075538944 + }, + { + "epoch": 0.86, + "learning_rate": 6.925252525252525e-05, + "loss": 2.4135, + "theoretical_loss": 3.3144311761992062, + "tokens_seen": 3075604480 + }, + { + "epoch": 0.86, + "learning_rate": 6.923232323232324e-05, + "loss": 2.4871, + "theoretical_loss": 3.314425773563038, + "tokens_seen": 3075670016 + }, + { + "epoch": 0.86, + "learning_rate": 6.92121212121212e-05, + "loss": 2.4491, + "theoretical_loss": 3.3144203710742195, + "tokens_seen": 3075735552 + }, + { + "epoch": 0.86, + "learning_rate": 6.919191919191919e-05, + "loss": 2.4931, + "theoretical_loss": 3.314414968732744, + "tokens_seen": 3075801088 + }, + { + "epoch": 0.86, + "learning_rate": 6.917171717171717e-05, + "loss": 2.5375, + "theoretical_loss": 3.3144095665386044, + "tokens_seen": 3075866624 + }, + { + "epoch": 0.86, + "learning_rate": 6.915151515151516e-05, + "loss": 2.7463, + "theoretical_loss": 3.3144041644917928, + "tokens_seen": 3075932160 + }, + { + "epoch": 0.86, + "learning_rate": 6.913131313131313e-05, + "loss": 2.4166, + "theoretical_loss": 3.314398762592303, + "tokens_seen": 3075997696 + }, + { + "epoch": 0.86, + "learning_rate": 6.911111111111111e-05, + "loss": 2.582, + "theoretical_loss": 3.3143933608401266, + "tokens_seen": 3076063232 + }, + { + "epoch": 0.86, + "learning_rate": 6.90909090909091e-05, + "loss": 2.6363, + "theoretical_loss": 3.314387959235258, + "tokens_seen": 3076128768 + }, + { + "epoch": 0.86, + "learning_rate": 6.907070707070708e-05, + "loss": 2.6888, + "theoretical_loss": 3.3143825577776886, + "tokens_seen": 3076194304 + }, + { + "epoch": 0.86, + "learning_rate": 6.905050505050506e-05, + "loss": 2.4279, + "theoretical_loss": 3.3143771564674123, + "tokens_seen": 3076259840 + }, + { + "epoch": 0.86, + "learning_rate": 6.903030303030302e-05, + "loss": 2.5647, + "theoretical_loss": 3.314371755304421, + "tokens_seen": 3076325376 + }, + { + "epoch": 0.86, + "learning_rate": 6.9010101010101e-05, + "loss": 2.4622, + "theoretical_loss": 3.3143663542887087, + "tokens_seen": 3076390912 + }, + { + "epoch": 0.86, + "learning_rate": 6.898989898989899e-05, + "loss": 2.5507, + "theoretical_loss": 3.3143609534202674, + "tokens_seen": 3076456448 + }, + { + "epoch": 0.86, + "learning_rate": 6.896969696969697e-05, + "loss": 2.5577, + "theoretical_loss": 3.31435555269909, + "tokens_seen": 3076521984 + }, + { + "epoch": 0.86, + "learning_rate": 6.894949494949496e-05, + "loss": 2.476, + "theoretical_loss": 3.31435015212517, + "tokens_seen": 3076587520 + }, + { + "epoch": 0.86, + "learning_rate": 6.892929292929293e-05, + "loss": 2.4386, + "theoretical_loss": 3.314344751698499, + "tokens_seen": 3076653056 + }, + { + "epoch": 0.86, + "learning_rate": 6.890909090909091e-05, + "loss": 2.3472, + "theoretical_loss": 3.3143393514190715, + "tokens_seen": 3076718592 + }, + { + "epoch": 0.86, + "learning_rate": 6.88888888888889e-05, + "loss": 2.419, + "theoretical_loss": 3.314333951286879, + "tokens_seen": 3076784128 + }, + { + "epoch": 0.86, + "learning_rate": 6.886868686868688e-05, + "loss": 2.5338, + "theoretical_loss": 3.314328551301915, + "tokens_seen": 3076849664 + }, + { + "epoch": 0.86, + "learning_rate": 6.884848484848485e-05, + "loss": 2.4138, + "theoretical_loss": 3.314323151464172, + "tokens_seen": 3076915200 + }, + { + "epoch": 0.86, + "learning_rate": 6.882828282828282e-05, + "loss": 2.241, + "theoretical_loss": 3.314317751773643, + "tokens_seen": 3076980736 + }, + { + "epoch": 0.86, + "learning_rate": 6.88080808080808e-05, + "loss": 2.3939, + "theoretical_loss": 3.3143123522303206, + "tokens_seen": 3077046272 + }, + { + "epoch": 0.86, + "learning_rate": 6.878787878787879e-05, + "loss": 2.4355, + "theoretical_loss": 3.3143069528341984, + "tokens_seen": 3077111808 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1730859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5200440883636475, + "objective/train/theoretical_loss": 3.3143015535852687, + "objective/train/tokens_used": 1436036576, + "theoretical_loss": 3.3143015535852687, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.86, + "learning_rate": 6.876767676767677e-05, + "loss": 2.425, + "theoretical_loss": 3.3143015535852687, + "tokens_seen": 3077177344 + }, + { + "epoch": 0.86, + "learning_rate": 6.874747474747476e-05, + "loss": 2.6306, + "theoretical_loss": 3.3142961544835243, + "tokens_seen": 3077242880 + }, + { + "epoch": 0.86, + "learning_rate": 6.872727272727274e-05, + "loss": 2.602, + "theoretical_loss": 3.3142907555289582, + "tokens_seen": 3077308416 + }, + { + "epoch": 0.86, + "learning_rate": 6.870707070707071e-05, + "loss": 2.4645, + "theoretical_loss": 3.3142853567215633, + "tokens_seen": 3077373952 + }, + { + "epoch": 0.86, + "learning_rate": 6.868686868686868e-05, + "loss": 2.3398, + "theoretical_loss": 3.3142799580613325, + "tokens_seen": 3077439488 + }, + { + "epoch": 0.86, + "learning_rate": 6.866666666666666e-05, + "loss": 2.393, + "theoretical_loss": 3.314274559548258, + "tokens_seen": 3077505024 + }, + { + "epoch": 0.86, + "learning_rate": 6.864646464646465e-05, + "loss": 2.537, + "theoretical_loss": 3.3142691611823336, + "tokens_seen": 3077570560 + }, + { + "epoch": 0.86, + "learning_rate": 6.862626262626263e-05, + "loss": 2.439, + "theoretical_loss": 3.314263762963552, + "tokens_seen": 3077636096 + }, + { + "epoch": 0.86, + "learning_rate": 6.86060606060606e-05, + "loss": 2.6535, + "theoretical_loss": 3.3142583648919053, + "tokens_seen": 3077701632 + }, + { + "epoch": 0.86, + "learning_rate": 6.858585858585859e-05, + "loss": 2.5989, + "theoretical_loss": 3.3142529669673872, + "tokens_seen": 3077767168 + }, + { + "epoch": 0.86, + "learning_rate": 6.856565656565657e-05, + "loss": 2.52, + "theoretical_loss": 3.31424756918999, + "tokens_seen": 3077832704 + }, + { + "epoch": 0.86, + "learning_rate": 6.854545454545455e-05, + "loss": 2.4704, + "theoretical_loss": 3.314242171559707, + "tokens_seen": 3077898240 + }, + { + "epoch": 0.86, + "learning_rate": 6.852525252525254e-05, + "loss": 2.6694, + "theoretical_loss": 3.3142367740765306, + "tokens_seen": 3077963776 + }, + { + "epoch": 0.86, + "learning_rate": 6.85050505050505e-05, + "loss": 2.5412, + "theoretical_loss": 3.3142313767404543, + "tokens_seen": 3078029312 + }, + { + "epoch": 0.86, + "learning_rate": 6.848484848484848e-05, + "loss": 2.4145, + "theoretical_loss": 3.31422597955147, + "tokens_seen": 3078094848 + }, + { + "epoch": 0.86, + "learning_rate": 6.846464646464646e-05, + "loss": 2.4478, + "theoretical_loss": 3.3142205825095714, + "tokens_seen": 3078160384 + }, + { + "epoch": 0.86, + "learning_rate": 6.844444444444445e-05, + "loss": 2.5429, + "theoretical_loss": 3.314215185614751, + "tokens_seen": 3078225920 + }, + { + "epoch": 0.86, + "learning_rate": 6.842424242424243e-05, + "loss": 2.4982, + "theoretical_loss": 3.314209788867002, + "tokens_seen": 3078291456 + }, + { + "epoch": 0.86, + "learning_rate": 6.84040404040404e-05, + "loss": 2.4403, + "theoretical_loss": 3.3142043922663165, + "tokens_seen": 3078356992 + }, + { + "epoch": 0.86, + "learning_rate": 6.838383838383839e-05, + "loss": 2.5131, + "theoretical_loss": 3.3141989958126885, + "tokens_seen": 3078422528 + }, + { + "epoch": 0.86, + "learning_rate": 6.836363636363637e-05, + "loss": 2.6254, + "theoretical_loss": 3.3141935995061096, + "tokens_seen": 3078488064 + }, + { + "epoch": 0.86, + "learning_rate": 6.834343434343434e-05, + "loss": 2.6861, + "theoretical_loss": 3.314188203346574, + "tokens_seen": 3078553600 + }, + { + "epoch": 0.86, + "learning_rate": 6.832323232323232e-05, + "loss": 2.449, + "theoretical_loss": 3.314182807334073, + "tokens_seen": 3078619136 + }, + { + "epoch": 0.86, + "learning_rate": 6.830303030303031e-05, + "loss": 2.3121, + "theoretical_loss": 3.314177411468601, + "tokens_seen": 3078684672 + }, + { + "epoch": 0.86, + "learning_rate": 6.828282828282828e-05, + "loss": 2.2468, + "theoretical_loss": 3.3141720157501497, + "tokens_seen": 3078750208 + }, + { + "epoch": 0.86, + "objective/train/docs_used": 1732124, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325119972229004, + "objective/train/theoretical_loss": 3.3141666201787126, + "objective/train/tokens_used": 1437674976, + "theoretical_loss": 3.3141666201787126, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.86, + "learning_rate": 6.826262626262626e-05, + "loss": 2.469, + "theoretical_loss": 3.3141666201787126, + "tokens_seen": 3078815744 + }, + { + "epoch": 0.87, + "learning_rate": 6.824242424242425e-05, + "loss": 2.6016, + "theoretical_loss": 3.314161224754282, + "tokens_seen": 3078881280 + }, + { + "epoch": 0.87, + "learning_rate": 6.822222222222223e-05, + "loss": 2.5673, + "theoretical_loss": 3.314155829476852, + "tokens_seen": 3078946816 + }, + { + "epoch": 0.87, + "learning_rate": 6.820202020202021e-05, + "loss": 2.6041, + "theoretical_loss": 3.314150434346414, + "tokens_seen": 3079012352 + }, + { + "epoch": 0.87, + "learning_rate": 6.818181818181818e-05, + "loss": 2.3402, + "theoretical_loss": 3.314145039362962, + "tokens_seen": 3079077888 + }, + { + "epoch": 0.87, + "learning_rate": 6.816161616161615e-05, + "loss": 2.5098, + "theoretical_loss": 3.314139644526488, + "tokens_seen": 3079143424 + }, + { + "epoch": 0.87, + "learning_rate": 6.814141414141414e-05, + "loss": 2.601, + "theoretical_loss": 3.3141342498369855, + "tokens_seen": 3079208960 + }, + { + "epoch": 0.87, + "learning_rate": 6.812121212121212e-05, + "loss": 2.5165, + "theoretical_loss": 3.314128855294447, + "tokens_seen": 3079274496 + }, + { + "epoch": 0.87, + "learning_rate": 6.81010101010101e-05, + "loss": 2.4163, + "theoretical_loss": 3.3141234608988652, + "tokens_seen": 3079340032 + }, + { + "epoch": 0.87, + "learning_rate": 6.808080808080808e-05, + "loss": 2.5234, + "theoretical_loss": 3.3141180666502335, + "tokens_seen": 3079405568 + }, + { + "epoch": 0.87, + "learning_rate": 6.806060606060606e-05, + "loss": 2.5318, + "theoretical_loss": 3.3141126725485446, + "tokens_seen": 3079471104 + }, + { + "epoch": 0.87, + "learning_rate": 6.804040404040405e-05, + "loss": 2.4588, + "theoretical_loss": 3.314107278593791, + "tokens_seen": 3079536640 + }, + { + "epoch": 0.87, + "learning_rate": 6.802020202020203e-05, + "loss": 2.7245, + "theoretical_loss": 3.3141018847859662, + "tokens_seen": 3079602176 + }, + { + "epoch": 0.87, + "learning_rate": 6.800000000000001e-05, + "loss": 2.744, + "theoretical_loss": 3.3140964911250625, + "tokens_seen": 3079667712 + }, + { + "epoch": 0.87, + "learning_rate": 6.797979797979797e-05, + "loss": 2.3618, + "theoretical_loss": 3.3140910976110733, + "tokens_seen": 3079733248 + }, + { + "epoch": 0.87, + "learning_rate": 6.795959595959595e-05, + "loss": 2.5323, + "theoretical_loss": 3.3140857042439906, + "tokens_seen": 3079798784 + }, + { + "epoch": 0.87, + "learning_rate": 6.793939393939394e-05, + "loss": 2.489, + "theoretical_loss": 3.314080311023808, + "tokens_seen": 3079864320 + }, + { + "epoch": 0.87, + "learning_rate": 6.791919191919192e-05, + "loss": 2.3923, + "theoretical_loss": 3.314074917950519, + "tokens_seen": 3079929856 + }, + { + "epoch": 0.87, + "learning_rate": 6.78989898989899e-05, + "loss": 2.5128, + "theoretical_loss": 3.314069525024115, + "tokens_seen": 3079995392 + }, + { + "epoch": 0.87, + "learning_rate": 6.787878787878789e-05, + "loss": 2.3162, + "theoretical_loss": 3.314064132244589, + "tokens_seen": 3080060928 + }, + { + "epoch": 0.87, + "learning_rate": 6.785858585858586e-05, + "loss": 2.4728, + "theoretical_loss": 3.3140587396119354, + "tokens_seen": 3080126464 + }, + { + "epoch": 0.87, + "learning_rate": 6.783838383838384e-05, + "loss": 2.4687, + "theoretical_loss": 3.314053347126146, + "tokens_seen": 3080192000 + }, + { + "epoch": 0.87, + "learning_rate": 6.781818181818181e-05, + "loss": 2.4409, + "theoretical_loss": 3.3140479547872133, + "tokens_seen": 3080257536 + }, + { + "epoch": 0.87, + "learning_rate": 6.77979797979798e-05, + "loss": 2.3521, + "theoretical_loss": 3.314042562595131, + "tokens_seen": 3080323072 + }, + { + "epoch": 0.87, + "learning_rate": 6.777777777777778e-05, + "loss": 2.4158, + "theoretical_loss": 3.3140371705498914, + "tokens_seen": 3080388608 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1732703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.002183675765991, + "objective/train/theoretical_loss": 3.314031778651488, + "objective/train/tokens_used": 1439313376, + "theoretical_loss": 3.314031778651488, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.87, + "learning_rate": 6.775757575757575e-05, + "loss": 2.5009, + "theoretical_loss": 3.314031778651488, + "tokens_seen": 3080454144 + }, + { + "epoch": 0.87, + "learning_rate": 6.773737373737374e-05, + "loss": 2.4098, + "theoretical_loss": 3.314026386899913, + "tokens_seen": 3080519680 + }, + { + "epoch": 0.87, + "learning_rate": 6.771717171717172e-05, + "loss": 2.3991, + "theoretical_loss": 3.3140209952951594, + "tokens_seen": 3080585216 + }, + { + "epoch": 0.87, + "learning_rate": 6.76969696969697e-05, + "loss": 2.7914, + "theoretical_loss": 3.3140156038372206, + "tokens_seen": 3080650752 + }, + { + "epoch": 0.87, + "learning_rate": 6.767676767676769e-05, + "loss": 2.6606, + "theoretical_loss": 3.314010212526089, + "tokens_seen": 3080716288 + }, + { + "epoch": 0.87, + "learning_rate": 6.765656565656566e-05, + "loss": 2.404, + "theoretical_loss": 3.314004821361758, + "tokens_seen": 3080781824 + }, + { + "epoch": 0.87, + "learning_rate": 6.763636363636363e-05, + "loss": 2.584, + "theoretical_loss": 3.3139994303442197, + "tokens_seen": 3080847360 + }, + { + "epoch": 0.87, + "learning_rate": 6.761616161616161e-05, + "loss": 2.7178, + "theoretical_loss": 3.3139940394734673, + "tokens_seen": 3080912896 + }, + { + "epoch": 0.87, + "learning_rate": 6.75959595959596e-05, + "loss": 2.4085, + "theoretical_loss": 3.313988648749494, + "tokens_seen": 3080978432 + }, + { + "epoch": 0.87, + "learning_rate": 6.757575757575758e-05, + "loss": 2.4653, + "theoretical_loss": 3.313983258172292, + "tokens_seen": 3081043968 + }, + { + "epoch": 0.87, + "learning_rate": 6.755555555555555e-05, + "loss": 2.3756, + "theoretical_loss": 3.3139778677418548, + "tokens_seen": 3081109504 + }, + { + "epoch": 0.87, + "learning_rate": 6.753535353535354e-05, + "loss": 2.5429, + "theoretical_loss": 3.3139724774581754, + "tokens_seen": 3081175040 + }, + { + "epoch": 0.87, + "learning_rate": 6.751515151515152e-05, + "loss": 2.3832, + "theoretical_loss": 3.313967087321246, + "tokens_seen": 3081240576 + }, + { + "epoch": 0.87, + "learning_rate": 6.74949494949495e-05, + "loss": 2.419, + "theoretical_loss": 3.31396169733106, + "tokens_seen": 3081306112 + }, + { + "epoch": 0.87, + "learning_rate": 6.747474747474747e-05, + "loss": 2.2057, + "theoretical_loss": 3.3139563074876106, + "tokens_seen": 3081371648 + }, + { + "epoch": 0.87, + "learning_rate": 6.745454545454546e-05, + "loss": 2.7254, + "theoretical_loss": 3.3139509177908897, + "tokens_seen": 3081437184 + }, + { + "epoch": 0.87, + "learning_rate": 6.743434343434343e-05, + "loss": 2.4839, + "theoretical_loss": 3.3139455282408905, + "tokens_seen": 3081502720 + }, + { + "epoch": 0.87, + "learning_rate": 6.741414141414141e-05, + "loss": 2.5096, + "theoretical_loss": 3.313940138837607, + "tokens_seen": 3081568256 + }, + { + "epoch": 0.87, + "learning_rate": 6.73939393939394e-05, + "loss": 2.6445, + "theoretical_loss": 3.3139347495810303, + "tokens_seen": 3081633792 + }, + { + "epoch": 0.87, + "learning_rate": 6.737373737373738e-05, + "loss": 2.5282, + "theoretical_loss": 3.3139293604711546, + "tokens_seen": 3081699328 + }, + { + "epoch": 0.87, + "learning_rate": 6.735353535353536e-05, + "loss": 2.3605, + "theoretical_loss": 3.3139239715079722, + "tokens_seen": 3081764864 + }, + { + "epoch": 0.87, + "learning_rate": 6.733333333333333e-05, + "loss": 2.4047, + "theoretical_loss": 3.313918582691476, + "tokens_seen": 3081830400 + }, + { + "epoch": 0.87, + "learning_rate": 6.731313131313132e-05, + "loss": 2.5896, + "theoretical_loss": 3.3139131940216595, + "tokens_seen": 3081895936 + }, + { + "epoch": 0.87, + "learning_rate": 6.729292929292929e-05, + "loss": 2.453, + "theoretical_loss": 3.313907805498515, + "tokens_seen": 3081961472 + }, + { + "epoch": 0.87, + "learning_rate": 6.727272727272727e-05, + "loss": 2.485, + "theoretical_loss": 3.3139024171220353, + "tokens_seen": 3082027008 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1733946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7956299781799316, + "objective/train/theoretical_loss": 3.313897028892214, + "objective/train/tokens_used": 1440951776, + "theoretical_loss": 3.313897028892214, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 6.725252525252526e-05, + "loss": 2.8091, + "theoretical_loss": 3.313897028892214, + "tokens_seen": 3082092544 + }, + { + "epoch": 0.87, + "learning_rate": 6.723232323232323e-05, + "loss": 2.418, + "theoretical_loss": 3.3138916408090426, + "tokens_seen": 3082158080 + }, + { + "epoch": 0.87, + "learning_rate": 6.721212121212121e-05, + "loss": 2.6122, + "theoretical_loss": 3.3138862528725155, + "tokens_seen": 3082223616 + }, + { + "epoch": 0.87, + "learning_rate": 6.71919191919192e-05, + "loss": 2.535, + "theoretical_loss": 3.3138808650826252, + "tokens_seen": 3082289152 + }, + { + "epoch": 0.87, + "learning_rate": 6.717171717171718e-05, + "loss": 2.2962, + "theoretical_loss": 3.313875477439364, + "tokens_seen": 3082354688 + }, + { + "epoch": 0.87, + "learning_rate": 6.715151515151516e-05, + "loss": 2.3232, + "theoretical_loss": 3.313870089942725, + "tokens_seen": 3082420224 + }, + { + "epoch": 0.87, + "learning_rate": 6.713131313131312e-05, + "loss": 2.4464, + "theoretical_loss": 3.3138647025927015, + "tokens_seen": 3082485760 + }, + { + "epoch": 0.87, + "learning_rate": 6.71111111111111e-05, + "loss": 2.8273, + "theoretical_loss": 3.313859315389286, + "tokens_seen": 3082551296 + }, + { + "epoch": 0.87, + "learning_rate": 6.709090909090909e-05, + "loss": 2.365, + "theoretical_loss": 3.313853928332472, + "tokens_seen": 3082616832 + }, + { + "epoch": 0.87, + "learning_rate": 6.707070707070707e-05, + "loss": 2.6017, + "theoretical_loss": 3.313848541422251, + "tokens_seen": 3082682368 + }, + { + "epoch": 0.87, + "learning_rate": 6.705050505050506e-05, + "loss": 2.7636, + "theoretical_loss": 3.313843154658618, + "tokens_seen": 3082747904 + }, + { + "epoch": 0.87, + "learning_rate": 6.703030303030303e-05, + "loss": 2.6992, + "theoretical_loss": 3.313837768041564, + "tokens_seen": 3082813440 + }, + { + "epoch": 0.87, + "learning_rate": 6.701010101010101e-05, + "loss": 2.7382, + "theoretical_loss": 3.313832381571083, + "tokens_seen": 3082878976 + }, + { + "epoch": 0.87, + "learning_rate": 6.6989898989899e-05, + "loss": 2.4255, + "theoretical_loss": 3.313826995247167, + "tokens_seen": 3082944512 + }, + { + "epoch": 0.87, + "learning_rate": 6.696969696969698e-05, + "loss": 2.4818, + "theoretical_loss": 3.3138216090698096, + "tokens_seen": 3083010048 + }, + { + "epoch": 0.87, + "learning_rate": 6.694949494949495e-05, + "loss": 2.4091, + "theoretical_loss": 3.313816223039004, + "tokens_seen": 3083075584 + }, + { + "epoch": 0.87, + "learning_rate": 6.692929292929293e-05, + "loss": 2.4235, + "theoretical_loss": 3.313810837154742, + "tokens_seen": 3083141120 + }, + { + "epoch": 0.87, + "learning_rate": 6.69090909090909e-05, + "loss": 2.3684, + "theoretical_loss": 3.3138054514170174, + "tokens_seen": 3083206656 + }, + { + "epoch": 0.87, + "learning_rate": 6.688888888888889e-05, + "loss": 2.297, + "theoretical_loss": 3.3138000658258226, + "tokens_seen": 3083272192 + }, + { + "epoch": 0.87, + "learning_rate": 6.686868686868687e-05, + "loss": 2.5244, + "theoretical_loss": 3.313794680381151, + "tokens_seen": 3083337728 + }, + { + "epoch": 0.87, + "learning_rate": 6.684848484848485e-05, + "loss": 2.595, + "theoretical_loss": 3.313789295082995, + "tokens_seen": 3083403264 + }, + { + "epoch": 0.87, + "learning_rate": 6.682828282828284e-05, + "loss": 2.3433, + "theoretical_loss": 3.3137839099313475, + "tokens_seen": 3083468800 + }, + { + "epoch": 0.87, + "learning_rate": 6.680808080808081e-05, + "loss": 2.65, + "theoretical_loss": 3.313778524926202, + "tokens_seen": 3083534336 + }, + { + "epoch": 0.87, + "learning_rate": 6.678787878787878e-05, + "loss": 2.5395, + "theoretical_loss": 3.3137731400675507, + "tokens_seen": 3083599872 + }, + { + "epoch": 0.87, + "learning_rate": 6.676767676767676e-05, + "loss": 2.5827, + "theoretical_loss": 3.313767755355387, + "tokens_seen": 3083665408 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1734728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.267428159713745, + "objective/train/theoretical_loss": 3.3137623707897035, + "objective/train/tokens_used": 1442590176, + "theoretical_loss": 3.3137623707897035, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.87, + "learning_rate": 6.674747474747475e-05, + "loss": 2.4126, + "theoretical_loss": 3.3137623707897035, + "tokens_seen": 3083730944 + }, + { + "epoch": 0.87, + "learning_rate": 6.672727272727273e-05, + "loss": 2.6275, + "theoretical_loss": 3.3137569863704934, + "tokens_seen": 3083796480 + }, + { + "epoch": 0.87, + "learning_rate": 6.67070707070707e-05, + "loss": 2.509, + "theoretical_loss": 3.3137516020977493, + "tokens_seen": 3083862016 + }, + { + "epoch": 0.87, + "learning_rate": 6.668686868686869e-05, + "loss": 2.5223, + "theoretical_loss": 3.313746217971464, + "tokens_seen": 3083927552 + }, + { + "epoch": 0.87, + "learning_rate": 6.666666666666667e-05, + "loss": 2.5352, + "theoretical_loss": 3.3137408339916306, + "tokens_seen": 3083993088 + }, + { + "epoch": 0.87, + "learning_rate": 6.664646464646465e-05, + "loss": 2.6832, + "theoretical_loss": 3.3137354501582426, + "tokens_seen": 3084058624 + }, + { + "epoch": 0.87, + "learning_rate": 6.662626262626264e-05, + "loss": 2.6075, + "theoretical_loss": 3.3137300664712916, + "tokens_seen": 3084124160 + }, + { + "epoch": 0.87, + "learning_rate": 6.66060606060606e-05, + "loss": 2.4296, + "theoretical_loss": 3.3137246829307716, + "tokens_seen": 3084189696 + }, + { + "epoch": 0.87, + "learning_rate": 6.658585858585858e-05, + "loss": 2.5757, + "theoretical_loss": 3.3137192995366753, + "tokens_seen": 3084255232 + }, + { + "epoch": 0.87, + "learning_rate": 6.656565656565656e-05, + "loss": 2.5927, + "theoretical_loss": 3.3137139162889953, + "tokens_seen": 3084320768 + }, + { + "epoch": 0.87, + "learning_rate": 6.654545454545455e-05, + "loss": 2.7055, + "theoretical_loss": 3.3137085331877243, + "tokens_seen": 3084386304 + }, + { + "epoch": 0.87, + "learning_rate": 6.652525252525253e-05, + "loss": 2.6426, + "theoretical_loss": 3.313703150232856, + "tokens_seen": 3084451840 + }, + { + "epoch": 0.87, + "learning_rate": 6.650505050505051e-05, + "loss": 2.3898, + "theoretical_loss": 3.313697767424383, + "tokens_seen": 3084517376 + }, + { + "epoch": 0.87, + "learning_rate": 6.648484848484848e-05, + "loss": 2.4548, + "theoretical_loss": 3.3136923847622977, + "tokens_seen": 3084582912 + }, + { + "epoch": 0.87, + "learning_rate": 6.646464646464647e-05, + "loss": 2.6446, + "theoretical_loss": 3.3136870022465934, + "tokens_seen": 3084648448 + }, + { + "epoch": 0.87, + "learning_rate": 6.644444444444445e-05, + "loss": 2.6279, + "theoretical_loss": 3.313681619877263, + "tokens_seen": 3084713984 + }, + { + "epoch": 0.87, + "learning_rate": 6.642424242424242e-05, + "loss": 2.4205, + "theoretical_loss": 3.3136762376543, + "tokens_seen": 3084779520 + }, + { + "epoch": 0.87, + "learning_rate": 6.640404040404041e-05, + "loss": 2.3668, + "theoretical_loss": 3.313670855577696, + "tokens_seen": 3084845056 + }, + { + "epoch": 0.87, + "learning_rate": 6.638383838383838e-05, + "loss": 2.732, + "theoretical_loss": 3.3136654736474447, + "tokens_seen": 3084910592 + }, + { + "epoch": 0.87, + "learning_rate": 6.636363636363636e-05, + "loss": 2.4646, + "theoretical_loss": 3.313660091863539, + "tokens_seen": 3084976128 + }, + { + "epoch": 0.87, + "learning_rate": 6.634343434343435e-05, + "loss": 2.6678, + "theoretical_loss": 3.313654710225972, + "tokens_seen": 3085041664 + }, + { + "epoch": 0.87, + "learning_rate": 6.632323232323233e-05, + "loss": 2.5678, + "theoretical_loss": 3.313649328734736, + "tokens_seen": 3085107200 + }, + { + "epoch": 0.87, + "learning_rate": 6.630303030303031e-05, + "loss": 2.766, + "theoretical_loss": 3.3136439473898243, + "tokens_seen": 3085172736 + }, + { + "epoch": 0.87, + "learning_rate": 6.628282828282828e-05, + "loss": 2.4139, + "theoretical_loss": 3.31363856619123, + "tokens_seen": 3085238272 + }, + { + "epoch": 0.87, + "learning_rate": 6.626262626262625e-05, + "loss": 2.6456, + "theoretical_loss": 3.313633185138946, + "tokens_seen": 3085303808 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1736039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.297853946685791, + "objective/train/theoretical_loss": 3.3136278042329645, + "objective/train/tokens_used": 1444228576, + "theoretical_loss": 3.3136278042329645, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 6.624242424242424e-05, + "loss": 2.5233, + "theoretical_loss": 3.3136278042329645, + "tokens_seen": 3085369344 + }, + { + "epoch": 0.87, + "learning_rate": 6.622222222222222e-05, + "loss": 2.5805, + "theoretical_loss": 3.313622423473279, + "tokens_seen": 3085434880 + }, + { + "epoch": 0.87, + "learning_rate": 6.62020202020202e-05, + "loss": 2.6657, + "theoretical_loss": 3.3136170428598826, + "tokens_seen": 3085500416 + }, + { + "epoch": 0.87, + "learning_rate": 6.618181818181818e-05, + "loss": 2.457, + "theoretical_loss": 3.313611662392768, + "tokens_seen": 3085565952 + }, + { + "epoch": 0.87, + "learning_rate": 6.616161616161616e-05, + "loss": 2.7221, + "theoretical_loss": 3.313606282071928, + "tokens_seen": 3085631488 + }, + { + "epoch": 0.87, + "learning_rate": 6.614141414141414e-05, + "loss": 2.5068, + "theoretical_loss": 3.3136009018973556, + "tokens_seen": 3085697024 + }, + { + "epoch": 0.87, + "learning_rate": 6.612121212121213e-05, + "loss": 2.3641, + "theoretical_loss": 3.3135955218690434, + "tokens_seen": 3085762560 + }, + { + "epoch": 0.87, + "learning_rate": 6.610101010101011e-05, + "loss": 2.5883, + "theoretical_loss": 3.313590141986985, + "tokens_seen": 3085828096 + }, + { + "epoch": 0.87, + "learning_rate": 6.608080808080808e-05, + "loss": 2.6822, + "theoretical_loss": 3.313584762251173, + "tokens_seen": 3085893632 + }, + { + "epoch": 0.87, + "learning_rate": 6.606060606060605e-05, + "loss": 2.6091, + "theoretical_loss": 3.3135793826616, + "tokens_seen": 3085959168 + }, + { + "epoch": 0.87, + "learning_rate": 6.604040404040404e-05, + "loss": 2.4705, + "theoretical_loss": 3.3135740032182595, + "tokens_seen": 3086024704 + }, + { + "epoch": 0.87, + "learning_rate": 6.602020202020202e-05, + "loss": 2.315, + "theoretical_loss": 3.3135686239211437, + "tokens_seen": 3086090240 + }, + { + "epoch": 0.87, + "learning_rate": 6.6e-05, + "loss": 2.5233, + "theoretical_loss": 3.313563244770246, + "tokens_seen": 3086155776 + }, + { + "epoch": 0.87, + "learning_rate": 6.597979797979799e-05, + "loss": 2.5984, + "theoretical_loss": 3.3135578657655596, + "tokens_seen": 3086221312 + }, + { + "epoch": 0.87, + "learning_rate": 6.595959595959596e-05, + "loss": 2.485, + "theoretical_loss": 3.3135524869070765, + "tokens_seen": 3086286848 + }, + { + "epoch": 0.87, + "learning_rate": 6.593939393939394e-05, + "loss": 2.3676, + "theoretical_loss": 3.3135471081947907, + "tokens_seen": 3086352384 + }, + { + "epoch": 0.87, + "learning_rate": 6.591919191919191e-05, + "loss": 2.3459, + "theoretical_loss": 3.313541729628694, + "tokens_seen": 3086417920 + }, + { + "epoch": 0.87, + "learning_rate": 6.58989898989899e-05, + "loss": 2.4186, + "theoretical_loss": 3.3135363512087808, + "tokens_seen": 3086483456 + }, + { + "epoch": 0.87, + "learning_rate": 6.587878787878788e-05, + "loss": 2.313, + "theoretical_loss": 3.3135309729350424, + "tokens_seen": 3086548992 + }, + { + "epoch": 0.87, + "learning_rate": 6.585858585858585e-05, + "loss": 2.6263, + "theoretical_loss": 3.313525594807473, + "tokens_seen": 3086614528 + }, + { + "epoch": 0.87, + "learning_rate": 6.583838383838384e-05, + "loss": 2.4666, + "theoretical_loss": 3.3135202168260647, + "tokens_seen": 3086680064 + }, + { + "epoch": 0.87, + "learning_rate": 6.581818181818182e-05, + "loss": 2.557, + "theoretical_loss": 3.3135148389908107, + "tokens_seen": 3086745600 + }, + { + "epoch": 0.87, + "learning_rate": 6.57979797979798e-05, + "loss": 2.3193, + "theoretical_loss": 3.313509461301704, + "tokens_seen": 3086811136 + }, + { + "epoch": 0.87, + "learning_rate": 6.577777777777779e-05, + "loss": 2.2955, + "theoretical_loss": 3.3135040837587377, + "tokens_seen": 3086876672 + }, + { + "epoch": 0.87, + "learning_rate": 6.575757575757576e-05, + "loss": 2.5382, + "theoretical_loss": 3.313498706361904, + "tokens_seen": 3086942208 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1736664, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.908751964569092, + "objective/train/theoretical_loss": 3.313493329111197, + "objective/train/tokens_used": 1445866976, + "theoretical_loss": 3.313493329111197, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.87, + "learning_rate": 6.573737373737373e-05, + "loss": 2.6552, + "theoretical_loss": 3.313493329111197, + "tokens_seen": 3087007744 + }, + { + "epoch": 0.87, + "learning_rate": 6.571717171717171e-05, + "loss": 2.5758, + "theoretical_loss": 3.313487952006609, + "tokens_seen": 3087073280 + }, + { + "epoch": 0.87, + "learning_rate": 6.56969696969697e-05, + "loss": 2.2378, + "theoretical_loss": 3.3134825750481323, + "tokens_seen": 3087138816 + }, + { + "epoch": 0.87, + "learning_rate": 6.567676767676768e-05, + "loss": 2.4288, + "theoretical_loss": 3.3134771982357605, + "tokens_seen": 3087204352 + }, + { + "epoch": 0.87, + "learning_rate": 6.565656565656566e-05, + "loss": 2.6831, + "theoretical_loss": 3.3134718215694865, + "tokens_seen": 3087269888 + }, + { + "epoch": 0.87, + "learning_rate": 6.563636363636364e-05, + "loss": 2.4425, + "theoretical_loss": 3.313466445049303, + "tokens_seen": 3087335424 + }, + { + "epoch": 0.87, + "learning_rate": 6.561616161616162e-05, + "loss": 2.5433, + "theoretical_loss": 3.3134610686752035, + "tokens_seen": 3087400960 + }, + { + "epoch": 0.87, + "learning_rate": 6.55959595959596e-05, + "loss": 2.5574, + "theoretical_loss": 3.3134556924471807, + "tokens_seen": 3087466496 + }, + { + "epoch": 0.87, + "learning_rate": 6.557575757575759e-05, + "loss": 2.5844, + "theoretical_loss": 3.313450316365227, + "tokens_seen": 3087532032 + }, + { + "epoch": 0.87, + "learning_rate": 6.555555555555556e-05, + "loss": 2.678, + "theoretical_loss": 3.313444940429336, + "tokens_seen": 3087597568 + }, + { + "epoch": 0.87, + "learning_rate": 6.553535353535353e-05, + "loss": 2.587, + "theoretical_loss": 3.3134395646394994, + "tokens_seen": 3087663104 + }, + { + "epoch": 0.87, + "learning_rate": 6.551515151515151e-05, + "loss": 2.3078, + "theoretical_loss": 3.313434188995712, + "tokens_seen": 3087728640 + }, + { + "epoch": 0.87, + "learning_rate": 6.54949494949495e-05, + "loss": 2.5617, + "theoretical_loss": 3.3134288134979655, + "tokens_seen": 3087794176 + }, + { + "epoch": 0.87, + "learning_rate": 6.547474747474748e-05, + "loss": 2.6153, + "theoretical_loss": 3.313423438146253, + "tokens_seen": 3087859712 + }, + { + "epoch": 0.87, + "learning_rate": 6.545454545454546e-05, + "loss": 2.2932, + "theoretical_loss": 3.313418062940568, + "tokens_seen": 3087925248 + }, + { + "epoch": 0.87, + "learning_rate": 6.543434343434343e-05, + "loss": 2.5068, + "theoretical_loss": 3.3134126878809025, + "tokens_seen": 3087990784 + }, + { + "epoch": 0.87, + "learning_rate": 6.541414141414142e-05, + "loss": 2.6818, + "theoretical_loss": 3.31340731296725, + "tokens_seen": 3088056320 + }, + { + "epoch": 0.87, + "learning_rate": 6.539393939393939e-05, + "loss": 2.5099, + "theoretical_loss": 3.313401938199603, + "tokens_seen": 3088121856 + }, + { + "epoch": 0.87, + "learning_rate": 6.537373737373737e-05, + "loss": 2.5554, + "theoretical_loss": 3.3133965635779554, + "tokens_seen": 3088187392 + }, + { + "epoch": 0.87, + "learning_rate": 6.535353535353536e-05, + "loss": 2.4881, + "theoretical_loss": 3.3133911891022994, + "tokens_seen": 3088252928 + }, + { + "epoch": 0.87, + "learning_rate": 6.533333333333333e-05, + "loss": 2.3435, + "theoretical_loss": 3.3133858147726283, + "tokens_seen": 3088318464 + }, + { + "epoch": 0.87, + "learning_rate": 6.531313131313131e-05, + "loss": 2.6055, + "theoretical_loss": 3.313380440588934, + "tokens_seen": 3088384000 + }, + { + "epoch": 0.87, + "learning_rate": 6.52929292929293e-05, + "loss": 2.7592, + "theoretical_loss": 3.3133750665512114, + "tokens_seen": 3088449536 + }, + { + "epoch": 0.87, + "learning_rate": 6.527272727272728e-05, + "loss": 2.343, + "theoretical_loss": 3.3133696926594514, + "tokens_seen": 3088515072 + }, + { + "epoch": 0.87, + "learning_rate": 6.525252525252526e-05, + "loss": 2.204, + "theoretical_loss": 3.3133643189136484, + "tokens_seen": 3088580608 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1737965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6117160320281982, + "objective/train/theoretical_loss": 3.313358945313794, + "objective/train/tokens_used": 1447505376, + "theoretical_loss": 3.313358945313794, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 6.523232323232325e-05, + "loss": 2.6185, + "theoretical_loss": 3.313358945313794, + "tokens_seen": 3088646144 + }, + { + "epoch": 0.87, + "learning_rate": 6.52121212121212e-05, + "loss": 2.4325, + "theoretical_loss": 3.3133535718598823, + "tokens_seen": 3088711680 + }, + { + "epoch": 0.87, + "learning_rate": 6.519191919191919e-05, + "loss": 2.6184, + "theoretical_loss": 3.313348198551906, + "tokens_seen": 3088777216 + }, + { + "epoch": 0.87, + "learning_rate": 6.517171717171717e-05, + "loss": 2.6265, + "theoretical_loss": 3.313342825389858, + "tokens_seen": 3088842752 + }, + { + "epoch": 0.87, + "learning_rate": 6.515151515151516e-05, + "loss": 2.4719, + "theoretical_loss": 3.3133374523737307, + "tokens_seen": 3088908288 + }, + { + "epoch": 0.87, + "learning_rate": 6.513131313131314e-05, + "loss": 2.5774, + "theoretical_loss": 3.3133320795035175, + "tokens_seen": 3088973824 + }, + { + "epoch": 0.87, + "learning_rate": 6.511111111111111e-05, + "loss": 2.3762, + "theoretical_loss": 3.313326706779211, + "tokens_seen": 3089039360 + }, + { + "epoch": 0.87, + "learning_rate": 6.50909090909091e-05, + "loss": 2.7027, + "theoretical_loss": 3.313321334200805, + "tokens_seen": 3089104896 + }, + { + "epoch": 0.87, + "learning_rate": 6.507070707070708e-05, + "loss": 2.3788, + "theoretical_loss": 3.313315961768292, + "tokens_seen": 3089170432 + }, + { + "epoch": 0.87, + "learning_rate": 6.505050505050505e-05, + "loss": 2.5264, + "theoretical_loss": 3.3133105894816643, + "tokens_seen": 3089235968 + }, + { + "epoch": 0.87, + "learning_rate": 6.503030303030303e-05, + "loss": 2.4918, + "theoretical_loss": 3.3133052173409157, + "tokens_seen": 3089301504 + }, + { + "epoch": 0.87, + "learning_rate": 6.5010101010101e-05, + "loss": 2.5273, + "theoretical_loss": 3.3132998453460387, + "tokens_seen": 3089367040 + }, + { + "epoch": 0.87, + "learning_rate": 6.498989898989899e-05, + "loss": 2.6343, + "theoretical_loss": 3.3132944734970264, + "tokens_seen": 3089432576 + }, + { + "epoch": 0.87, + "learning_rate": 6.496969696969697e-05, + "loss": 2.4158, + "theoretical_loss": 3.3132891017938717, + "tokens_seen": 3089498112 + }, + { + "epoch": 0.87, + "learning_rate": 6.494949494949495e-05, + "loss": 2.3543, + "theoretical_loss": 3.3132837302365674, + "tokens_seen": 3089563648 + }, + { + "epoch": 0.87, + "learning_rate": 6.492929292929294e-05, + "loss": 2.5648, + "theoretical_loss": 3.313278358825107, + "tokens_seen": 3089629184 + }, + { + "epoch": 0.87, + "learning_rate": 6.490909090909091e-05, + "loss": 2.664, + "theoretical_loss": 3.3132729875594826, + "tokens_seen": 3089694720 + }, + { + "epoch": 0.87, + "learning_rate": 6.488888888888889e-05, + "loss": 2.453, + "theoretical_loss": 3.313267616439688, + "tokens_seen": 3089760256 + }, + { + "epoch": 0.87, + "learning_rate": 6.486868686868686e-05, + "loss": 2.2923, + "theoretical_loss": 3.3132622454657152, + "tokens_seen": 3089825792 + }, + { + "epoch": 0.87, + "learning_rate": 6.484848484848485e-05, + "loss": 2.3202, + "theoretical_loss": 3.3132568746375584, + "tokens_seen": 3089891328 + }, + { + "epoch": 0.87, + "learning_rate": 6.482828282828283e-05, + "loss": 2.7404, + "theoretical_loss": 3.3132515039552093, + "tokens_seen": 3089956864 + }, + { + "epoch": 0.87, + "learning_rate": 6.480808080808081e-05, + "loss": 2.6562, + "theoretical_loss": 3.3132461334186614, + "tokens_seen": 3090022400 + }, + { + "epoch": 0.87, + "learning_rate": 6.478787878787879e-05, + "loss": 2.5449, + "theoretical_loss": 3.313240763027908, + "tokens_seen": 3090087936 + }, + { + "epoch": 0.87, + "learning_rate": 6.476767676767677e-05, + "loss": 2.762, + "theoretical_loss": 3.3132353927829414, + "tokens_seen": 3090153472 + }, + { + "epoch": 0.87, + "learning_rate": 6.474747474747475e-05, + "loss": 2.4292, + "theoretical_loss": 3.313230022683755, + "tokens_seen": 3090219008 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1738543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.631016969680786, + "objective/train/theoretical_loss": 3.3132246527303413, + "objective/train/tokens_used": 1449143776, + "theoretical_loss": 3.3132246527303413, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.87, + "learning_rate": 6.472727272727274e-05, + "loss": 2.6554, + "theoretical_loss": 3.3132246527303413, + "tokens_seen": 3090284544 + }, + { + "epoch": 0.87, + "learning_rate": 6.470707070707071e-05, + "loss": 2.5335, + "theoretical_loss": 3.313219282922694, + "tokens_seen": 3090350080 + }, + { + "epoch": 0.87, + "learning_rate": 6.468686868686868e-05, + "loss": 2.4654, + "theoretical_loss": 3.313213913260805, + "tokens_seen": 3090415616 + }, + { + "epoch": 0.87, + "learning_rate": 6.466666666666666e-05, + "loss": 2.3744, + "theoretical_loss": 3.3132085437446683, + "tokens_seen": 3090481152 + }, + { + "epoch": 0.87, + "learning_rate": 6.464646464646465e-05, + "loss": 2.5363, + "theoretical_loss": 3.3132031743742765, + "tokens_seen": 3090546688 + }, + { + "epoch": 0.87, + "learning_rate": 6.462626262626263e-05, + "loss": 2.3892, + "theoretical_loss": 3.313197805149622, + "tokens_seen": 3090612224 + }, + { + "epoch": 0.87, + "learning_rate": 6.460606060606061e-05, + "loss": 2.4085, + "theoretical_loss": 3.3131924360706986, + "tokens_seen": 3090677760 + }, + { + "epoch": 0.87, + "learning_rate": 6.458585858585858e-05, + "loss": 2.4101, + "theoretical_loss": 3.3131870671374983, + "tokens_seen": 3090743296 + }, + { + "epoch": 0.87, + "learning_rate": 6.456565656565657e-05, + "loss": 2.4525, + "theoretical_loss": 3.3131816983500153, + "tokens_seen": 3090808832 + }, + { + "epoch": 0.87, + "learning_rate": 6.454545454545455e-05, + "loss": 2.5708, + "theoretical_loss": 3.3131763297082415, + "tokens_seen": 3090874368 + }, + { + "epoch": 0.87, + "learning_rate": 6.452525252525252e-05, + "loss": 2.5862, + "theoretical_loss": 3.3131709612121703, + "tokens_seen": 3090939904 + }, + { + "epoch": 0.87, + "learning_rate": 6.45050505050505e-05, + "loss": 2.4015, + "theoretical_loss": 3.3131655928617945, + "tokens_seen": 3091005440 + }, + { + "epoch": 0.87, + "learning_rate": 6.448484848484848e-05, + "loss": 2.4594, + "theoretical_loss": 3.313160224657107, + "tokens_seen": 3091070976 + }, + { + "epoch": 0.87, + "learning_rate": 6.446464646464646e-05, + "loss": 2.6077, + "theoretical_loss": 3.3131548565981013, + "tokens_seen": 3091136512 + }, + { + "epoch": 0.87, + "learning_rate": 6.444444444444444e-05, + "loss": 2.4006, + "theoretical_loss": 3.3131494886847697, + "tokens_seen": 3091202048 + }, + { + "epoch": 0.87, + "learning_rate": 6.442424242424243e-05, + "loss": 2.6667, + "theoretical_loss": 3.3131441209171055, + "tokens_seen": 3091267584 + }, + { + "epoch": 0.87, + "learning_rate": 6.440404040404041e-05, + "loss": 2.505, + "theoretical_loss": 3.3131387532951013, + "tokens_seen": 3091333120 + }, + { + "epoch": 0.87, + "learning_rate": 6.43838383838384e-05, + "loss": 2.1763, + "theoretical_loss": 3.313133385818751, + "tokens_seen": 3091398656 + }, + { + "epoch": 0.87, + "learning_rate": 6.436363636363635e-05, + "loss": 2.5357, + "theoretical_loss": 3.313128018488046, + "tokens_seen": 3091464192 + }, + { + "epoch": 0.87, + "learning_rate": 6.434343434343434e-05, + "loss": 2.5876, + "theoretical_loss": 3.313122651302981, + "tokens_seen": 3091529728 + }, + { + "epoch": 0.87, + "learning_rate": 6.432323232323232e-05, + "loss": 2.3183, + "theoretical_loss": 3.3131172842635475, + "tokens_seen": 3091595264 + }, + { + "epoch": 0.87, + "learning_rate": 6.43030303030303e-05, + "loss": 2.3858, + "theoretical_loss": 3.3131119173697394, + "tokens_seen": 3091660800 + }, + { + "epoch": 0.87, + "learning_rate": 6.428282828282829e-05, + "loss": 2.6347, + "theoretical_loss": 3.313106550621549, + "tokens_seen": 3091726336 + }, + { + "epoch": 0.87, + "learning_rate": 6.426262626262626e-05, + "loss": 2.5022, + "theoretical_loss": 3.31310118401897, + "tokens_seen": 3091791872 + }, + { + "epoch": 0.87, + "learning_rate": 6.424242424242424e-05, + "loss": 2.5197, + "theoretical_loss": 3.3130958175619947, + "tokens_seen": 3091857408 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1739219, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.759056568145752, + "objective/train/theoretical_loss": 3.3130904512506163, + "objective/train/tokens_used": 1450782176, + "theoretical_loss": 3.3130904512506163, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 6.422222222222223e-05, + "loss": 2.5436, + "theoretical_loss": 3.3130904512506163, + "tokens_seen": 3091922944 + }, + { + "epoch": 0.87, + "learning_rate": 6.420202020202021e-05, + "loss": 2.5661, + "theoretical_loss": 3.313085085084828, + "tokens_seen": 3091988480 + }, + { + "epoch": 0.87, + "learning_rate": 6.418181818181818e-05, + "loss": 2.2592, + "theoretical_loss": 3.3130797190646226, + "tokens_seen": 3092054016 + }, + { + "epoch": 0.87, + "learning_rate": 6.416161616161615e-05, + "loss": 2.5614, + "theoretical_loss": 3.3130743531899927, + "tokens_seen": 3092119552 + }, + { + "epoch": 0.87, + "learning_rate": 6.414141414141414e-05, + "loss": 2.3151, + "theoretical_loss": 3.313068987460932, + "tokens_seen": 3092185088 + }, + { + "epoch": 0.87, + "learning_rate": 6.412121212121212e-05, + "loss": 2.3599, + "theoretical_loss": 3.3130636218774328, + "tokens_seen": 3092250624 + }, + { + "epoch": 0.87, + "learning_rate": 6.41010101010101e-05, + "loss": 2.7464, + "theoretical_loss": 3.313058256439488, + "tokens_seen": 3092316160 + }, + { + "epoch": 0.87, + "learning_rate": 6.408080808080809e-05, + "loss": 2.4746, + "theoretical_loss": 3.3130528911470916, + "tokens_seen": 3092381696 + }, + { + "epoch": 0.87, + "learning_rate": 6.406060606060606e-05, + "loss": 2.3717, + "theoretical_loss": 3.3130475260002354, + "tokens_seen": 3092447232 + }, + { + "epoch": 0.87, + "learning_rate": 6.404040404040404e-05, + "loss": 2.5687, + "theoretical_loss": 3.3130421609989127, + "tokens_seen": 3092512768 + }, + { + "epoch": 0.87, + "learning_rate": 6.402020202020203e-05, + "loss": 2.6491, + "theoretical_loss": 3.313036796143117, + "tokens_seen": 3092578304 + }, + { + "epoch": 0.87, + "learning_rate": 6.4e-05, + "loss": 2.6375, + "theoretical_loss": 3.3130314314328406, + "tokens_seen": 3092643840 + }, + { + "epoch": 0.87, + "learning_rate": 6.397979797979798e-05, + "loss": 2.6626, + "theoretical_loss": 3.3130260668680767, + "tokens_seen": 3092709376 + }, + { + "epoch": 0.87, + "learning_rate": 6.395959595959597e-05, + "loss": 2.6356, + "theoretical_loss": 3.313020702448818, + "tokens_seen": 3092774912 + }, + { + "epoch": 0.87, + "learning_rate": 6.393939393939394e-05, + "loss": 2.3546, + "theoretical_loss": 3.313015338175058, + "tokens_seen": 3092840448 + }, + { + "epoch": 0.87, + "learning_rate": 6.391919191919192e-05, + "loss": 2.5331, + "theoretical_loss": 3.31300997404679, + "tokens_seen": 3092905984 + }, + { + "epoch": 0.87, + "learning_rate": 6.38989898989899e-05, + "loss": 2.6192, + "theoretical_loss": 3.313004610064006, + "tokens_seen": 3092971520 + }, + { + "epoch": 0.87, + "learning_rate": 6.387878787878789e-05, + "loss": 2.463, + "theoretical_loss": 3.312999246226699, + "tokens_seen": 3093037056 + }, + { + "epoch": 0.87, + "learning_rate": 6.385858585858587e-05, + "loss": 2.6423, + "theoretical_loss": 3.3129938825348626, + "tokens_seen": 3093102592 + }, + { + "epoch": 0.87, + "learning_rate": 6.383838383838383e-05, + "loss": 2.5327, + "theoretical_loss": 3.31298851898849, + "tokens_seen": 3093168128 + }, + { + "epoch": 0.87, + "learning_rate": 6.381818181818181e-05, + "loss": 2.3956, + "theoretical_loss": 3.312983155587573, + "tokens_seen": 3093233664 + }, + { + "epoch": 0.87, + "learning_rate": 6.37979797979798e-05, + "loss": 2.4984, + "theoretical_loss": 3.3129777923321058, + "tokens_seen": 3093299200 + }, + { + "epoch": 0.87, + "learning_rate": 6.377777777777778e-05, + "loss": 2.4456, + "theoretical_loss": 3.3129724292220804, + "tokens_seen": 3093364736 + }, + { + "epoch": 0.87, + "learning_rate": 6.375757575757576e-05, + "loss": 2.7153, + "theoretical_loss": 3.3129670662574906, + "tokens_seen": 3093430272 + }, + { + "epoch": 0.87, + "learning_rate": 6.373737373737373e-05, + "loss": 2.6224, + "theoretical_loss": 3.3129617034383285, + "tokens_seen": 3093495808 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1739859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7754733562469482, + "objective/train/theoretical_loss": 3.312956340764588, + "objective/train/tokens_used": 1452420576, + "theoretical_loss": 3.312956340764588, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.87, + "learning_rate": 6.371717171717172e-05, + "loss": 2.4565, + "theoretical_loss": 3.312956340764588, + "tokens_seen": 3093561344 + }, + { + "epoch": 0.87, + "learning_rate": 6.36969696969697e-05, + "loss": 2.6186, + "theoretical_loss": 3.3129509782362616, + "tokens_seen": 3093626880 + }, + { + "epoch": 0.87, + "learning_rate": 6.367676767676769e-05, + "loss": 2.5658, + "theoretical_loss": 3.3129456158533426, + "tokens_seen": 3093692416 + }, + { + "epoch": 0.87, + "learning_rate": 6.365656565656566e-05, + "loss": 2.5932, + "theoretical_loss": 3.3129402536158232, + "tokens_seen": 3093757952 + }, + { + "epoch": 0.87, + "learning_rate": 6.363636363636363e-05, + "loss": 2.5073, + "theoretical_loss": 3.312934891523697, + "tokens_seen": 3093823488 + }, + { + "epoch": 0.87, + "learning_rate": 6.361616161616161e-05, + "loss": 2.6369, + "theoretical_loss": 3.312929529576957, + "tokens_seen": 3093889024 + }, + { + "epoch": 0.87, + "learning_rate": 6.35959595959596e-05, + "loss": 2.6026, + "theoretical_loss": 3.312924167775596, + "tokens_seen": 3093954560 + }, + { + "epoch": 0.87, + "learning_rate": 6.357575757575758e-05, + "loss": 2.4154, + "theoretical_loss": 3.3129188061196064, + "tokens_seen": 3094020096 + }, + { + "epoch": 0.87, + "learning_rate": 6.355555555555556e-05, + "loss": 2.5509, + "theoretical_loss": 3.3129134446089825, + "tokens_seen": 3094085632 + }, + { + "epoch": 0.87, + "learning_rate": 6.353535353535355e-05, + "loss": 2.5085, + "theoretical_loss": 3.3129080832437166, + "tokens_seen": 3094151168 + }, + { + "epoch": 0.87, + "learning_rate": 6.351515151515152e-05, + "loss": 2.6856, + "theoretical_loss": 3.3129027220238014, + "tokens_seen": 3094216704 + }, + { + "epoch": 0.87, + "learning_rate": 6.349494949494949e-05, + "loss": 2.7023, + "theoretical_loss": 3.31289736094923, + "tokens_seen": 3094282240 + }, + { + "epoch": 0.87, + "learning_rate": 6.347474747474747e-05, + "loss": 2.4824, + "theoretical_loss": 3.312892000019996, + "tokens_seen": 3094347776 + }, + { + "epoch": 0.87, + "learning_rate": 6.345454545454546e-05, + "loss": 2.4421, + "theoretical_loss": 3.3128866392360914, + "tokens_seen": 3094413312 + }, + { + "epoch": 0.87, + "learning_rate": 6.343434343434344e-05, + "loss": 2.4018, + "theoretical_loss": 3.31288127859751, + "tokens_seen": 3094478848 + }, + { + "epoch": 0.87, + "learning_rate": 6.341414141414141e-05, + "loss": 2.4497, + "theoretical_loss": 3.312875918104244, + "tokens_seen": 3094544384 + }, + { + "epoch": 0.87, + "learning_rate": 6.33939393939394e-05, + "loss": 2.6438, + "theoretical_loss": 3.3128705577562876, + "tokens_seen": 3094609920 + }, + { + "epoch": 0.87, + "learning_rate": 6.337373737373738e-05, + "loss": 2.5147, + "theoretical_loss": 3.3128651975536325, + "tokens_seen": 3094675456 + }, + { + "epoch": 0.87, + "learning_rate": 6.335353535353536e-05, + "loss": 2.5444, + "theoretical_loss": 3.3128598374962723, + "tokens_seen": 3094740992 + }, + { + "epoch": 0.87, + "learning_rate": 6.333333333333335e-05, + "loss": 2.5814, + "theoretical_loss": 3.3128544775842, + "tokens_seen": 3094806528 + }, + { + "epoch": 0.87, + "learning_rate": 6.33131313131313e-05, + "loss": 2.4268, + "theoretical_loss": 3.312849117817408, + "tokens_seen": 3094872064 + }, + { + "epoch": 0.87, + "learning_rate": 6.329292929292929e-05, + "loss": 2.5895, + "theoretical_loss": 3.3128437581958905, + "tokens_seen": 3094937600 + }, + { + "epoch": 0.87, + "learning_rate": 6.327272727272727e-05, + "loss": 2.4472, + "theoretical_loss": 3.3128383987196397, + "tokens_seen": 3095003136 + }, + { + "epoch": 0.87, + "learning_rate": 6.325252525252525e-05, + "loss": 2.4099, + "theoretical_loss": 3.3128330393886483, + "tokens_seen": 3095068672 + }, + { + "epoch": 0.87, + "learning_rate": 6.323232323232324e-05, + "loss": 2.3216, + "theoretical_loss": 3.3128276802029095, + "tokens_seen": 3095134208 + }, + { + "epoch": 0.87, + "objective/train/docs_used": 1741115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0964629650115967, + "objective/train/theoretical_loss": 3.312822321162417, + "objective/train/tokens_used": 1454058976, + "theoretical_loss": 3.312822321162417, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 6.321212121212121e-05, + "loss": 2.3072, + "theoretical_loss": 3.312822321162417, + "tokens_seen": 3095199744 + }, + { + "epoch": 0.88, + "learning_rate": 6.319191919191919e-05, + "loss": 2.397, + "theoretical_loss": 3.3128169622671626, + "tokens_seen": 3095265280 + }, + { + "epoch": 0.88, + "learning_rate": 6.317171717171718e-05, + "loss": 2.423, + "theoretical_loss": 3.31281160351714, + "tokens_seen": 3095330816 + }, + { + "epoch": 0.88, + "learning_rate": 6.315151515151516e-05, + "loss": 2.4161, + "theoretical_loss": 3.312806244912342, + "tokens_seen": 3095396352 + }, + { + "epoch": 0.88, + "learning_rate": 6.313131313131313e-05, + "loss": 2.4155, + "theoretical_loss": 3.312800886452762, + "tokens_seen": 3095461888 + }, + { + "epoch": 0.88, + "learning_rate": 6.311111111111112e-05, + "loss": 2.6274, + "theoretical_loss": 3.312795528138393, + "tokens_seen": 3095527424 + }, + { + "epoch": 0.88, + "learning_rate": 6.309090909090909e-05, + "loss": 2.8062, + "theoretical_loss": 3.312790169969227, + "tokens_seen": 3095592960 + }, + { + "epoch": 0.88, + "learning_rate": 6.307070707070707e-05, + "loss": 2.5431, + "theoretical_loss": 3.312784811945258, + "tokens_seen": 3095658496 + }, + { + "epoch": 0.88, + "learning_rate": 6.305050505050505e-05, + "loss": 2.4119, + "theoretical_loss": 3.3127794540664786, + "tokens_seen": 3095724032 + }, + { + "epoch": 0.88, + "learning_rate": 6.303030303030304e-05, + "loss": 2.6248, + "theoretical_loss": 3.3127740963328813, + "tokens_seen": 3095789568 + }, + { + "epoch": 0.88, + "learning_rate": 6.301010101010102e-05, + "loss": 2.6243, + "theoretical_loss": 3.31276873874446, + "tokens_seen": 3095855104 + }, + { + "epoch": 0.88, + "learning_rate": 6.298989898989899e-05, + "loss": 2.5738, + "theoretical_loss": 3.3127633813012074, + "tokens_seen": 3095920640 + }, + { + "epoch": 0.88, + "learning_rate": 6.296969696969696e-05, + "loss": 2.4104, + "theoretical_loss": 3.3127580240031165, + "tokens_seen": 3095986176 + }, + { + "epoch": 0.88, + "learning_rate": 6.294949494949495e-05, + "loss": 2.4998, + "theoretical_loss": 3.31275266685018, + "tokens_seen": 3096051712 + }, + { + "epoch": 0.88, + "learning_rate": 6.292929292929293e-05, + "loss": 2.4121, + "theoretical_loss": 3.312747309842391, + "tokens_seen": 3096117248 + }, + { + "epoch": 0.88, + "learning_rate": 6.290909090909091e-05, + "loss": 2.3668, + "theoretical_loss": 3.312741952979743, + "tokens_seen": 3096182784 + }, + { + "epoch": 0.88, + "learning_rate": 6.288888888888888e-05, + "loss": 2.5546, + "theoretical_loss": 3.312736596262228, + "tokens_seen": 3096248320 + }, + { + "epoch": 0.88, + "learning_rate": 6.286868686868687e-05, + "loss": 2.4703, + "theoretical_loss": 3.3127312396898403, + "tokens_seen": 3096313856 + }, + { + "epoch": 0.88, + "learning_rate": 6.284848484848485e-05, + "loss": 2.351, + "theoretical_loss": 3.312725883262572, + "tokens_seen": 3096379392 + }, + { + "epoch": 0.88, + "learning_rate": 6.282828282828284e-05, + "loss": 2.6067, + "theoretical_loss": 3.312720526980416, + "tokens_seen": 3096444928 + }, + { + "epoch": 0.88, + "learning_rate": 6.280808080808082e-05, + "loss": 2.3226, + "theoretical_loss": 3.3127151708433655, + "tokens_seen": 3096510464 + }, + { + "epoch": 0.88, + "learning_rate": 6.278787878787878e-05, + "loss": 2.6439, + "theoretical_loss": 3.312709814851414, + "tokens_seen": 3096576000 + }, + { + "epoch": 0.88, + "learning_rate": 6.276767676767676e-05, + "loss": 2.4317, + "theoretical_loss": 3.3127044590045536, + "tokens_seen": 3096641536 + }, + { + "epoch": 0.88, + "learning_rate": 6.274747474747475e-05, + "loss": 2.2546, + "theoretical_loss": 3.312699103302778, + "tokens_seen": 3096707072 + }, + { + "epoch": 0.88, + "learning_rate": 6.272727272727273e-05, + "loss": 2.5922, + "theoretical_loss": 3.31269374774608, + "tokens_seen": 3096772608 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1741744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4827017784118652, + "objective/train/theoretical_loss": 3.3126883923344526, + "objective/train/tokens_used": 1455697376, + "theoretical_loss": 3.3126883923344526, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.88, + "learning_rate": 6.270707070707071e-05, + "loss": 2.4658, + "theoretical_loss": 3.3126883923344526, + "tokens_seen": 3096838144 + }, + { + "epoch": 0.88, + "learning_rate": 6.26868686868687e-05, + "loss": 2.5116, + "theoretical_loss": 3.312683037067889, + "tokens_seen": 3096903680 + }, + { + "epoch": 0.88, + "learning_rate": 6.266666666666667e-05, + "loss": 2.6781, + "theoretical_loss": 3.3126776819463815, + "tokens_seen": 3096969216 + }, + { + "epoch": 0.88, + "learning_rate": 6.264646464646465e-05, + "loss": 2.6636, + "theoretical_loss": 3.312672326969924, + "tokens_seen": 3097034752 + }, + { + "epoch": 0.88, + "learning_rate": 6.262626262626262e-05, + "loss": 2.4925, + "theoretical_loss": 3.3126669721385085, + "tokens_seen": 3097100288 + }, + { + "epoch": 0.88, + "learning_rate": 6.26060606060606e-05, + "loss": 2.7068, + "theoretical_loss": 3.312661617452129, + "tokens_seen": 3097165824 + }, + { + "epoch": 0.88, + "learning_rate": 6.258585858585859e-05, + "loss": 2.5456, + "theoretical_loss": 3.312656262910778, + "tokens_seen": 3097231360 + }, + { + "epoch": 0.88, + "learning_rate": 6.256565656565656e-05, + "loss": 2.652, + "theoretical_loss": 3.3126509085144487, + "tokens_seen": 3097296896 + }, + { + "epoch": 0.88, + "learning_rate": 6.254545454545454e-05, + "loss": 2.4699, + "theoretical_loss": 3.312645554263134, + "tokens_seen": 3097362432 + }, + { + "epoch": 0.88, + "learning_rate": 6.252525252525253e-05, + "loss": 2.4718, + "theoretical_loss": 3.3126402001568267, + "tokens_seen": 3097427968 + }, + { + "epoch": 0.88, + "learning_rate": 6.250505050505051e-05, + "loss": 2.7101, + "theoretical_loss": 3.31263484619552, + "tokens_seen": 3097493504 + }, + { + "epoch": 0.88, + "learning_rate": 6.248484848484848e-05, + "loss": 2.5696, + "theoretical_loss": 3.312629492379207, + "tokens_seen": 3097559040 + }, + { + "epoch": 0.88, + "learning_rate": 6.246464646464647e-05, + "loss": 2.4185, + "theoretical_loss": 3.3126241387078803, + "tokens_seen": 3097624576 + }, + { + "epoch": 0.88, + "learning_rate": 6.244444444444444e-05, + "loss": 2.532, + "theoretical_loss": 3.3126187851815336, + "tokens_seen": 3097690112 + }, + { + "epoch": 0.88, + "learning_rate": 6.242424242424242e-05, + "loss": 2.4345, + "theoretical_loss": 3.3126134318001594, + "tokens_seen": 3097755648 + }, + { + "epoch": 0.88, + "learning_rate": 6.24040404040404e-05, + "loss": 2.4074, + "theoretical_loss": 3.312608078563751, + "tokens_seen": 3097821184 + }, + { + "epoch": 0.88, + "learning_rate": 6.238383838383839e-05, + "loss": 2.5106, + "theoretical_loss": 3.312602725472301, + "tokens_seen": 3097886720 + }, + { + "epoch": 0.88, + "learning_rate": 6.236363636363636e-05, + "loss": 2.4899, + "theoretical_loss": 3.312597372525803, + "tokens_seen": 3097952256 + }, + { + "epoch": 0.88, + "learning_rate": 6.234343434343434e-05, + "loss": 2.5718, + "theoretical_loss": 3.3125920197242493, + "tokens_seen": 3098017792 + }, + { + "epoch": 0.88, + "learning_rate": 6.232323232323233e-05, + "loss": 2.4418, + "theoretical_loss": 3.3125866670676336, + "tokens_seen": 3098083328 + }, + { + "epoch": 0.88, + "learning_rate": 6.230303030303031e-05, + "loss": 2.434, + "theoretical_loss": 3.312581314555948, + "tokens_seen": 3098148864 + }, + { + "epoch": 0.88, + "learning_rate": 6.228282828282828e-05, + "loss": 2.4922, + "theoretical_loss": 3.3125759621891864, + "tokens_seen": 3098214400 + }, + { + "epoch": 0.88, + "learning_rate": 6.226262626262627e-05, + "loss": 2.7483, + "theoretical_loss": 3.3125706099673415, + "tokens_seen": 3098279936 + }, + { + "epoch": 0.88, + "learning_rate": 6.224242424242425e-05, + "loss": 2.5877, + "theoretical_loss": 3.3125652578904066, + "tokens_seen": 3098345472 + }, + { + "epoch": 0.88, + "learning_rate": 6.222222222222222e-05, + "loss": 2.7137, + "theoretical_loss": 3.312559905958374, + "tokens_seen": 3098411008 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1742793, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8015713691711426, + "objective/train/theoretical_loss": 3.3125545541712373, + "objective/train/tokens_used": 1457335776, + "theoretical_loss": 3.3125545541712373, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 6.22020202020202e-05, + "loss": 2.5319, + "theoretical_loss": 3.3125545541712373, + "tokens_seen": 3098476544 + }, + { + "epoch": 0.88, + "learning_rate": 6.218181818181817e-05, + "loss": 2.4591, + "theoretical_loss": 3.3125492025289893, + "tokens_seen": 3098542080 + }, + { + "epoch": 0.88, + "learning_rate": 6.216161616161616e-05, + "loss": 2.4316, + "theoretical_loss": 3.3125438510316227, + "tokens_seen": 3098607616 + }, + { + "epoch": 0.88, + "learning_rate": 6.214141414141414e-05, + "loss": 2.5619, + "theoretical_loss": 3.3125384996791314, + "tokens_seen": 3098673152 + }, + { + "epoch": 0.88, + "learning_rate": 6.212121212121213e-05, + "loss": 2.6786, + "theoretical_loss": 3.3125331484715077, + "tokens_seen": 3098738688 + }, + { + "epoch": 0.88, + "learning_rate": 6.21010101010101e-05, + "loss": 2.5499, + "theoretical_loss": 3.3125277974087446, + "tokens_seen": 3098804224 + }, + { + "epoch": 0.88, + "learning_rate": 6.208080808080808e-05, + "loss": 2.5863, + "theoretical_loss": 3.312522446490836, + "tokens_seen": 3098869760 + }, + { + "epoch": 0.88, + "learning_rate": 6.206060606060606e-05, + "loss": 2.5742, + "theoretical_loss": 3.3125170957177734, + "tokens_seen": 3098935296 + }, + { + "epoch": 0.88, + "learning_rate": 6.204040404040405e-05, + "loss": 2.5229, + "theoretical_loss": 3.312511745089551, + "tokens_seen": 3099000832 + }, + { + "epoch": 0.88, + "learning_rate": 6.202020202020202e-05, + "loss": 2.6763, + "theoretical_loss": 3.3125063946061615, + "tokens_seen": 3099066368 + }, + { + "epoch": 0.88, + "learning_rate": 6.2e-05, + "loss": 2.4955, + "theoretical_loss": 3.3125010442675977, + "tokens_seen": 3099131904 + }, + { + "epoch": 0.88, + "learning_rate": 6.197979797979799e-05, + "loss": 2.4924, + "theoretical_loss": 3.3124956940738532, + "tokens_seen": 3099197440 + }, + { + "epoch": 0.88, + "learning_rate": 6.195959595959596e-05, + "loss": 2.5919, + "theoretical_loss": 3.31249034402492, + "tokens_seen": 3099262976 + }, + { + "epoch": 0.88, + "learning_rate": 6.193939393939394e-05, + "loss": 2.6182, + "theoretical_loss": 3.3124849941207923, + "tokens_seen": 3099328512 + }, + { + "epoch": 0.88, + "learning_rate": 6.191919191919192e-05, + "loss": 2.435, + "theoretical_loss": 3.312479644361462, + "tokens_seen": 3099394048 + }, + { + "epoch": 0.88, + "learning_rate": 6.18989898989899e-05, + "loss": 2.6947, + "theoretical_loss": 3.3124742947469232, + "tokens_seen": 3099459584 + }, + { + "epoch": 0.88, + "learning_rate": 6.187878787878788e-05, + "loss": 2.7179, + "theoretical_loss": 3.312468945277168, + "tokens_seen": 3099525120 + }, + { + "epoch": 0.88, + "learning_rate": 6.185858585858586e-05, + "loss": 2.6799, + "theoretical_loss": 3.31246359595219, + "tokens_seen": 3099590656 + }, + { + "epoch": 0.88, + "learning_rate": 6.183838383838383e-05, + "loss": 2.6012, + "theoretical_loss": 3.312458246771982, + "tokens_seen": 3099656192 + }, + { + "epoch": 0.88, + "learning_rate": 6.181818181818182e-05, + "loss": 2.6479, + "theoretical_loss": 3.312452897736537, + "tokens_seen": 3099721728 + }, + { + "epoch": 0.88, + "learning_rate": 6.17979797979798e-05, + "loss": 2.4048, + "theoretical_loss": 3.3124475488458485, + "tokens_seen": 3099787264 + }, + { + "epoch": 0.88, + "learning_rate": 6.177777777777779e-05, + "loss": 2.5341, + "theoretical_loss": 3.3124422000999085, + "tokens_seen": 3099852800 + }, + { + "epoch": 0.88, + "learning_rate": 6.175757575757576e-05, + "loss": 2.4307, + "theoretical_loss": 3.3124368514987106, + "tokens_seen": 3099918336 + }, + { + "epoch": 0.88, + "learning_rate": 6.173737373737374e-05, + "loss": 2.5267, + "theoretical_loss": 3.3124315030422484, + "tokens_seen": 3099983872 + }, + { + "epoch": 0.88, + "learning_rate": 6.171717171717172e-05, + "loss": 2.6746, + "theoretical_loss": 3.312426154730514, + "tokens_seen": 3100049408 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1743460, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7238881587982178, + "objective/train/theoretical_loss": 3.312420806563501, + "objective/train/tokens_used": 1458974176, + "theoretical_loss": 3.312420806563501, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.88, + "learning_rate": 6.16969696969697e-05, + "loss": 2.5989, + "theoretical_loss": 3.312420806563501, + "tokens_seen": 3100114944 + }, + { + "epoch": 0.88, + "learning_rate": 6.167676767676768e-05, + "loss": 2.3897, + "theoretical_loss": 3.312415458541202, + "tokens_seen": 3100180480 + }, + { + "epoch": 0.88, + "learning_rate": 6.165656565656566e-05, + "loss": 2.4124, + "theoretical_loss": 3.31241011066361, + "tokens_seen": 3100246016 + }, + { + "epoch": 0.88, + "learning_rate": 6.163636363636363e-05, + "loss": 2.4489, + "theoretical_loss": 3.3124047629307185, + "tokens_seen": 3100311552 + }, + { + "epoch": 0.88, + "learning_rate": 6.161616161616162e-05, + "loss": 2.3826, + "theoretical_loss": 3.3123994153425205, + "tokens_seen": 3100377088 + }, + { + "epoch": 0.88, + "learning_rate": 6.15959595959596e-05, + "loss": 2.5926, + "theoretical_loss": 3.3123940678990085, + "tokens_seen": 3100442624 + }, + { + "epoch": 0.88, + "learning_rate": 6.157575757575757e-05, + "loss": 2.6867, + "theoretical_loss": 3.3123887206001763, + "tokens_seen": 3100508160 + }, + { + "epoch": 0.88, + "learning_rate": 6.155555555555555e-05, + "loss": 2.4138, + "theoretical_loss": 3.312383373446016, + "tokens_seen": 3100573696 + }, + { + "epoch": 0.88, + "learning_rate": 6.153535353535354e-05, + "loss": 2.5595, + "theoretical_loss": 3.3123780264365212, + "tokens_seen": 3100639232 + }, + { + "epoch": 0.88, + "learning_rate": 6.151515151515152e-05, + "loss": 2.6028, + "theoretical_loss": 3.312372679571685, + "tokens_seen": 3100704768 + }, + { + "epoch": 0.88, + "learning_rate": 6.14949494949495e-05, + "loss": 2.7322, + "theoretical_loss": 3.3123673328515, + "tokens_seen": 3100770304 + }, + { + "epoch": 0.88, + "learning_rate": 6.147474747474748e-05, + "loss": 2.3659, + "theoretical_loss": 3.3123619862759597, + "tokens_seen": 3100835840 + }, + { + "epoch": 0.88, + "learning_rate": 6.145454545454546e-05, + "loss": 2.5211, + "theoretical_loss": 3.3123566398450572, + "tokens_seen": 3100901376 + }, + { + "epoch": 0.88, + "learning_rate": 6.143434343434343e-05, + "loss": 2.6101, + "theoretical_loss": 3.312351293558785, + "tokens_seen": 3100966912 + }, + { + "epoch": 0.88, + "learning_rate": 6.141414141414142e-05, + "loss": 2.5557, + "theoretical_loss": 3.312345947417136, + "tokens_seen": 3101032448 + }, + { + "epoch": 0.88, + "learning_rate": 6.13939393939394e-05, + "loss": 2.5018, + "theoretical_loss": 3.312340601420104, + "tokens_seen": 3101097984 + }, + { + "epoch": 0.88, + "learning_rate": 6.137373737373737e-05, + "loss": 2.6437, + "theoretical_loss": 3.3123352555676813, + "tokens_seen": 3101163520 + }, + { + "epoch": 0.88, + "learning_rate": 6.135353535353535e-05, + "loss": 2.7014, + "theoretical_loss": 3.3123299098598618, + "tokens_seen": 3101229056 + }, + { + "epoch": 0.88, + "learning_rate": 6.133333333333334e-05, + "loss": 2.4387, + "theoretical_loss": 3.3123245642966377, + "tokens_seen": 3101294592 + }, + { + "epoch": 0.88, + "learning_rate": 6.131313131313131e-05, + "loss": 2.499, + "theoretical_loss": 3.3123192188780024, + "tokens_seen": 3101360128 + }, + { + "epoch": 0.88, + "learning_rate": 6.129292929292929e-05, + "loss": 2.624, + "theoretical_loss": 3.312313873603949, + "tokens_seen": 3101425664 + }, + { + "epoch": 0.88, + "learning_rate": 6.127272727272728e-05, + "loss": 2.6758, + "theoretical_loss": 3.3123085284744698, + "tokens_seen": 3101491200 + }, + { + "epoch": 0.88, + "learning_rate": 6.125252525252526e-05, + "loss": 2.5555, + "theoretical_loss": 3.312303183489559, + "tokens_seen": 3101556736 + }, + { + "epoch": 0.88, + "learning_rate": 6.123232323232323e-05, + "loss": 2.6138, + "theoretical_loss": 3.312297838649209, + "tokens_seen": 3101622272 + }, + { + "epoch": 0.88, + "learning_rate": 6.121212121212121e-05, + "loss": 2.6047, + "theoretical_loss": 3.3122924939534126, + "tokens_seen": 3101687808 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1744846, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.895564317703247, + "objective/train/theoretical_loss": 3.3122871494021635, + "objective/train/tokens_used": 1460612576, + "theoretical_loss": 3.3122871494021635, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 6.11919191919192e-05, + "loss": 2.658, + "theoretical_loss": 3.3122871494021635, + "tokens_seen": 3101753344 + }, + { + "epoch": 0.88, + "learning_rate": 6.117171717171717e-05, + "loss": 2.532, + "theoretical_loss": 3.3122818049954543, + "tokens_seen": 3101818880 + }, + { + "epoch": 0.88, + "learning_rate": 6.115151515151515e-05, + "loss": 2.4086, + "theoretical_loss": 3.3122764607332784, + "tokens_seen": 3101884416 + }, + { + "epoch": 0.88, + "learning_rate": 6.113131313131314e-05, + "loss": 2.5691, + "theoretical_loss": 3.3122711166156282, + "tokens_seen": 3101949952 + }, + { + "epoch": 0.88, + "learning_rate": 6.111111111111111e-05, + "loss": 2.4844, + "theoretical_loss": 3.3122657726424976, + "tokens_seen": 3102015488 + }, + { + "epoch": 0.88, + "learning_rate": 6.109090909090909e-05, + "loss": 2.5479, + "theoretical_loss": 3.3122604288138784, + "tokens_seen": 3102081024 + }, + { + "epoch": 0.88, + "learning_rate": 6.107070707070708e-05, + "loss": 2.5291, + "theoretical_loss": 3.312255085129765, + "tokens_seen": 3102146560 + }, + { + "epoch": 0.88, + "learning_rate": 6.105050505050505e-05, + "loss": 2.2848, + "theoretical_loss": 3.3122497415901493, + "tokens_seen": 3102212096 + }, + { + "epoch": 0.88, + "learning_rate": 6.1030303030303036e-05, + "loss": 2.5828, + "theoretical_loss": 3.312244398195025, + "tokens_seen": 3102277632 + }, + { + "epoch": 0.88, + "learning_rate": 6.101010101010101e-05, + "loss": 2.3019, + "theoretical_loss": 3.312239054944385, + "tokens_seen": 3102343168 + }, + { + "epoch": 0.88, + "learning_rate": 6.098989898989899e-05, + "loss": 2.5425, + "theoretical_loss": 3.3122337118382226, + "tokens_seen": 3102408704 + }, + { + "epoch": 0.88, + "learning_rate": 6.0969696969696975e-05, + "loss": 2.5315, + "theoretical_loss": 3.3122283688765304, + "tokens_seen": 3102474240 + }, + { + "epoch": 0.88, + "learning_rate": 6.094949494949495e-05, + "loss": 2.6307, + "theoretical_loss": 3.3122230260593017, + "tokens_seen": 3102539776 + }, + { + "epoch": 0.88, + "learning_rate": 6.092929292929293e-05, + "loss": 2.6894, + "theoretical_loss": 3.3122176833865296, + "tokens_seen": 3102605312 + }, + { + "epoch": 0.88, + "learning_rate": 6.0909090909090906e-05, + "loss": 2.5769, + "theoretical_loss": 3.3122123408582067, + "tokens_seen": 3102670848 + }, + { + "epoch": 0.88, + "learning_rate": 6.088888888888889e-05, + "loss": 2.708, + "theoretical_loss": 3.3122069984743265, + "tokens_seen": 3102736384 + }, + { + "epoch": 0.88, + "learning_rate": 6.0868686868686874e-05, + "loss": 2.5043, + "theoretical_loss": 3.312201656234882, + "tokens_seen": 3102801920 + }, + { + "epoch": 0.88, + "learning_rate": 6.0848484848484845e-05, + "loss": 2.5324, + "theoretical_loss": 3.312196314139866, + "tokens_seen": 3102867456 + }, + { + "epoch": 0.88, + "learning_rate": 6.082828282828283e-05, + "loss": 2.6356, + "theoretical_loss": 3.3121909721892715, + "tokens_seen": 3102932992 + }, + { + "epoch": 0.88, + "learning_rate": 6.080808080808081e-05, + "loss": 2.5125, + "theoretical_loss": 3.312185630383092, + "tokens_seen": 3102998528 + }, + { + "epoch": 0.88, + "learning_rate": 6.078787878787879e-05, + "loss": 2.5413, + "theoretical_loss": 3.3121802887213203, + "tokens_seen": 3103064064 + }, + { + "epoch": 0.88, + "learning_rate": 6.0767676767676774e-05, + "loss": 2.6933, + "theoretical_loss": 3.3121749472039492, + "tokens_seen": 3103129600 + }, + { + "epoch": 0.88, + "learning_rate": 6.0747474747474744e-05, + "loss": 2.4023, + "theoretical_loss": 3.312169605830972, + "tokens_seen": 3103195136 + }, + { + "epoch": 0.88, + "learning_rate": 6.072727272727273e-05, + "loss": 2.5581, + "theoretical_loss": 3.312164264602382, + "tokens_seen": 3103260672 + }, + { + "epoch": 0.88, + "learning_rate": 6.070707070707071e-05, + "loss": 2.7351, + "theoretical_loss": 3.312158923518172, + "tokens_seen": 3103326208 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1745390, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4726874828338623, + "objective/train/theoretical_loss": 3.312153582578335, + "objective/train/tokens_used": 1462250976, + "theoretical_loss": 3.312153582578335, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.88, + "learning_rate": 6.068686868686869e-05, + "loss": 2.5271, + "theoretical_loss": 3.312153582578335, + "tokens_seen": 3103391744 + }, + { + "epoch": 0.88, + "learning_rate": 6.0666666666666666e-05, + "loss": 2.5635, + "theoretical_loss": 3.3121482417828636, + "tokens_seen": 3103457280 + }, + { + "epoch": 0.88, + "learning_rate": 6.064646464646465e-05, + "loss": 2.6698, + "theoretical_loss": 3.3121429011317516, + "tokens_seen": 3103522816 + }, + { + "epoch": 0.88, + "learning_rate": 6.062626262626263e-05, + "loss": 2.5852, + "theoretical_loss": 3.3121375606249917, + "tokens_seen": 3103588352 + }, + { + "epoch": 0.88, + "learning_rate": 6.060606060606061e-05, + "loss": 2.4532, + "theoretical_loss": 3.312132220262577, + "tokens_seen": 3103653888 + }, + { + "epoch": 0.88, + "learning_rate": 6.058585858585858e-05, + "loss": 2.2964, + "theoretical_loss": 3.3121268800445005, + "tokens_seen": 3103719424 + }, + { + "epoch": 0.88, + "learning_rate": 6.0565656565656566e-05, + "loss": 2.4096, + "theoretical_loss": 3.3121215399707555, + "tokens_seen": 3103784960 + }, + { + "epoch": 0.88, + "learning_rate": 6.054545454545455e-05, + "loss": 2.579, + "theoretical_loss": 3.312116200041335, + "tokens_seen": 3103850496 + }, + { + "epoch": 0.88, + "learning_rate": 6.052525252525253e-05, + "loss": 2.7382, + "theoretical_loss": 3.3121108602562317, + "tokens_seen": 3103916032 + }, + { + "epoch": 0.88, + "learning_rate": 6.0505050505050504e-05, + "loss": 2.5087, + "theoretical_loss": 3.312105520615439, + "tokens_seen": 3103981568 + }, + { + "epoch": 0.88, + "learning_rate": 6.048484848484848e-05, + "loss": 2.7989, + "theoretical_loss": 3.3121001811189497, + "tokens_seen": 3104047104 + }, + { + "epoch": 0.88, + "learning_rate": 6.0464646464646465e-05, + "loss": 2.7248, + "theoretical_loss": 3.312094841766757, + "tokens_seen": 3104112640 + }, + { + "epoch": 0.88, + "learning_rate": 6.044444444444445e-05, + "loss": 2.6044, + "theoretical_loss": 3.312089502558854, + "tokens_seen": 3104178176 + }, + { + "epoch": 0.88, + "learning_rate": 6.0424242424242426e-05, + "loss": 2.6448, + "theoretical_loss": 3.312084163495234, + "tokens_seen": 3104243712 + }, + { + "epoch": 0.88, + "learning_rate": 6.0404040404040404e-05, + "loss": 2.6649, + "theoretical_loss": 3.312078824575889, + "tokens_seen": 3104309248 + }, + { + "epoch": 0.88, + "learning_rate": 6.038383838383839e-05, + "loss": 2.7602, + "theoretical_loss": 3.312073485800813, + "tokens_seen": 3104374784 + }, + { + "epoch": 0.88, + "learning_rate": 6.0363636363636365e-05, + "loss": 2.3731, + "theoretical_loss": 3.3120681471699993, + "tokens_seen": 3104440320 + }, + { + "epoch": 0.88, + "learning_rate": 6.034343434343435e-05, + "loss": 2.5229, + "theoretical_loss": 3.3120628086834403, + "tokens_seen": 3104505856 + }, + { + "epoch": 0.88, + "learning_rate": 6.032323232323232e-05, + "loss": 2.5279, + "theoretical_loss": 3.312057470341129, + "tokens_seen": 3104571392 + }, + { + "epoch": 0.88, + "learning_rate": 6.03030303030303e-05, + "loss": 2.5013, + "theoretical_loss": 3.312052132143059, + "tokens_seen": 3104636928 + }, + { + "epoch": 0.88, + "learning_rate": 6.028282828282829e-05, + "loss": 2.5615, + "theoretical_loss": 3.312046794089223, + "tokens_seen": 3104702464 + }, + { + "epoch": 0.88, + "learning_rate": 6.0262626262626264e-05, + "loss": 2.3978, + "theoretical_loss": 3.3120414561796143, + "tokens_seen": 3104768000 + }, + { + "epoch": 0.88, + "learning_rate": 6.024242424242424e-05, + "loss": 2.2772, + "theoretical_loss": 3.3120361184142255, + "tokens_seen": 3104833536 + }, + { + "epoch": 0.88, + "learning_rate": 6.0222222222222225e-05, + "loss": 2.3348, + "theoretical_loss": 3.3120307807930502, + "tokens_seen": 3104899072 + }, + { + "epoch": 0.88, + "learning_rate": 6.02020202020202e-05, + "loss": 2.4468, + "theoretical_loss": 3.312025443316081, + "tokens_seen": 3104964608 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1746455, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054536819458008, + "objective/train/theoretical_loss": 3.3120201059833114, + "objective/train/tokens_used": 1463889376, + "theoretical_loss": 3.3120201059833114, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 6.0181818181818187e-05, + "loss": 2.717, + "theoretical_loss": 3.3120201059833114, + "tokens_seen": 3105030144 + }, + { + "epoch": 0.88, + "learning_rate": 6.016161616161616e-05, + "loss": 2.6503, + "theoretical_loss": 3.312014768794734, + "tokens_seen": 3105095680 + }, + { + "epoch": 0.88, + "learning_rate": 6.014141414141414e-05, + "loss": 2.4431, + "theoretical_loss": 3.3120094317503423, + "tokens_seen": 3105161216 + }, + { + "epoch": 0.88, + "learning_rate": 6.0121212121212125e-05, + "loss": 2.4233, + "theoretical_loss": 3.3120040948501295, + "tokens_seen": 3105226752 + }, + { + "epoch": 0.88, + "learning_rate": 6.01010101010101e-05, + "loss": 2.6507, + "theoretical_loss": 3.311998758094088, + "tokens_seen": 3105292288 + }, + { + "epoch": 0.88, + "learning_rate": 6.0080808080808086e-05, + "loss": 2.7621, + "theoretical_loss": 3.311993421482211, + "tokens_seen": 3105357824 + }, + { + "epoch": 0.88, + "learning_rate": 6.0060606060606056e-05, + "loss": 2.7252, + "theoretical_loss": 3.311988085014492, + "tokens_seen": 3105423360 + }, + { + "epoch": 0.88, + "learning_rate": 6.004040404040404e-05, + "loss": 2.6959, + "theoretical_loss": 3.3119827486909235, + "tokens_seen": 3105488896 + }, + { + "epoch": 0.88, + "learning_rate": 6.0020202020202024e-05, + "loss": 2.8994, + "theoretical_loss": 3.3119774125114994, + "tokens_seen": 3105554432 + }, + { + "epoch": 0.88, + "learning_rate": 6e-05, + "loss": 2.3585, + "theoretical_loss": 3.3119720764762115, + "tokens_seen": 3105619968 + }, + { + "epoch": 0.88, + "learning_rate": 5.997979797979798e-05, + "loss": 2.7153, + "theoretical_loss": 3.311966740585054, + "tokens_seen": 3105685504 + }, + { + "epoch": 0.88, + "learning_rate": 5.995959595959596e-05, + "loss": 2.6303, + "theoretical_loss": 3.3119614048380197, + "tokens_seen": 3105751040 + }, + { + "epoch": 0.88, + "learning_rate": 5.993939393939394e-05, + "loss": 2.3792, + "theoretical_loss": 3.3119560692351016, + "tokens_seen": 3105816576 + }, + { + "epoch": 0.88, + "learning_rate": 5.9919191919191924e-05, + "loss": 2.6067, + "theoretical_loss": 3.3119507337762926, + "tokens_seen": 3105882112 + }, + { + "epoch": 0.88, + "learning_rate": 5.9898989898989894e-05, + "loss": 2.3992, + "theoretical_loss": 3.3119453984615856, + "tokens_seen": 3105947648 + }, + { + "epoch": 0.88, + "learning_rate": 5.987878787878788e-05, + "loss": 2.561, + "theoretical_loss": 3.3119400632909746, + "tokens_seen": 3106013184 + }, + { + "epoch": 0.88, + "learning_rate": 5.985858585858586e-05, + "loss": 2.7422, + "theoretical_loss": 3.3119347282644513, + "tokens_seen": 3106078720 + }, + { + "epoch": 0.88, + "learning_rate": 5.983838383838384e-05, + "loss": 2.6355, + "theoretical_loss": 3.3119293933820098, + "tokens_seen": 3106144256 + }, + { + "epoch": 0.88, + "learning_rate": 5.981818181818182e-05, + "loss": 2.5415, + "theoretical_loss": 3.311924058643643, + "tokens_seen": 3106209792 + }, + { + "epoch": 0.88, + "learning_rate": 5.97979797979798e-05, + "loss": 2.398, + "theoretical_loss": 3.3119187240493435, + "tokens_seen": 3106275328 + }, + { + "epoch": 0.88, + "learning_rate": 5.977777777777778e-05, + "loss": 2.3163, + "theoretical_loss": 3.311913389599105, + "tokens_seen": 3106340864 + }, + { + "epoch": 0.88, + "learning_rate": 5.975757575757576e-05, + "loss": 2.5836, + "theoretical_loss": 3.31190805529292, + "tokens_seen": 3106406400 + }, + { + "epoch": 0.88, + "learning_rate": 5.973737373737374e-05, + "loss": 2.5919, + "theoretical_loss": 3.311902721130782, + "tokens_seen": 3106471936 + }, + { + "epoch": 0.88, + "learning_rate": 5.9717171717171716e-05, + "loss": 2.5032, + "theoretical_loss": 3.3118973871126838, + "tokens_seen": 3106537472 + }, + { + "epoch": 0.88, + "learning_rate": 5.96969696969697e-05, + "loss": 2.5045, + "theoretical_loss": 3.3118920532386187, + "tokens_seen": 3106603008 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1746871, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.788311243057251, + "objective/train/theoretical_loss": 3.3118867195085793, + "objective/train/tokens_used": 1465527776, + "theoretical_loss": 3.3118867195085793, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.88, + "learning_rate": 5.967676767676768e-05, + "loss": 2.7488, + "theoretical_loss": 3.3118867195085793, + "tokens_seen": 3106668544 + }, + { + "epoch": 0.88, + "learning_rate": 5.965656565656566e-05, + "loss": 2.5701, + "theoretical_loss": 3.3118813859225593, + "tokens_seen": 3106734080 + }, + { + "epoch": 0.88, + "learning_rate": 5.963636363636363e-05, + "loss": 2.7578, + "theoretical_loss": 3.3118760524805517, + "tokens_seen": 3106799616 + }, + { + "epoch": 0.88, + "learning_rate": 5.9616161616161615e-05, + "loss": 2.4527, + "theoretical_loss": 3.311870719182549, + "tokens_seen": 3106865152 + }, + { + "epoch": 0.88, + "learning_rate": 5.95959595959596e-05, + "loss": 2.6796, + "theoretical_loss": 3.3118653860285447, + "tokens_seen": 3106930688 + }, + { + "epoch": 0.88, + "learning_rate": 5.957575757575758e-05, + "loss": 2.6121, + "theoretical_loss": 3.311860053018532, + "tokens_seen": 3106996224 + }, + { + "epoch": 0.88, + "learning_rate": 5.955555555555556e-05, + "loss": 2.4744, + "theoretical_loss": 3.3118547201525037, + "tokens_seen": 3107061760 + }, + { + "epoch": 0.88, + "learning_rate": 5.953535353535354e-05, + "loss": 2.6609, + "theoretical_loss": 3.3118493874304527, + "tokens_seen": 3107127296 + }, + { + "epoch": 0.88, + "learning_rate": 5.9515151515151515e-05, + "loss": 2.4529, + "theoretical_loss": 3.3118440548523727, + "tokens_seen": 3107192832 + }, + { + "epoch": 0.88, + "learning_rate": 5.94949494949495e-05, + "loss": 2.6702, + "theoretical_loss": 3.311838722418256, + "tokens_seen": 3107258368 + }, + { + "epoch": 0.88, + "learning_rate": 5.9474747474747476e-05, + "loss": 2.6118, + "theoretical_loss": 3.3118333901280965, + "tokens_seen": 3107323904 + }, + { + "epoch": 0.88, + "learning_rate": 5.945454545454545e-05, + "loss": 2.4879, + "theoretical_loss": 3.311828057981887, + "tokens_seen": 3107389440 + }, + { + "epoch": 0.88, + "learning_rate": 5.943434343434344e-05, + "loss": 2.4774, + "theoretical_loss": 3.3118227259796202, + "tokens_seen": 3107454976 + }, + { + "epoch": 0.88, + "learning_rate": 5.9414141414141414e-05, + "loss": 2.552, + "theoretical_loss": 3.3118173941212894, + "tokens_seen": 3107520512 + }, + { + "epoch": 0.88, + "learning_rate": 5.93939393939394e-05, + "loss": 2.5841, + "theoretical_loss": 3.3118120624068874, + "tokens_seen": 3107586048 + }, + { + "epoch": 0.88, + "learning_rate": 5.9373737373737376e-05, + "loss": 2.5545, + "theoretical_loss": 3.311806730836408, + "tokens_seen": 3107651584 + }, + { + "epoch": 0.88, + "learning_rate": 5.935353535353535e-05, + "loss": 2.7091, + "theoretical_loss": 3.311801399409844, + "tokens_seen": 3107717120 + }, + { + "epoch": 0.88, + "learning_rate": 5.933333333333334e-05, + "loss": 2.7394, + "theoretical_loss": 3.3117960681271885, + "tokens_seen": 3107782656 + }, + { + "epoch": 0.88, + "learning_rate": 5.9313131313131314e-05, + "loss": 2.584, + "theoretical_loss": 3.311790736988434, + "tokens_seen": 3107848192 + }, + { + "epoch": 0.88, + "learning_rate": 5.929292929292929e-05, + "loss": 2.3758, + "theoretical_loss": 3.311785405993574, + "tokens_seen": 3107913728 + }, + { + "epoch": 0.88, + "learning_rate": 5.9272727272727275e-05, + "loss": 2.6703, + "theoretical_loss": 3.3117800751426016, + "tokens_seen": 3107979264 + }, + { + "epoch": 0.88, + "learning_rate": 5.925252525252525e-05, + "loss": 2.6456, + "theoretical_loss": 3.31177474443551, + "tokens_seen": 3108044800 + }, + { + "epoch": 0.88, + "learning_rate": 5.9232323232323236e-05, + "loss": 2.7568, + "theoretical_loss": 3.311769413872292, + "tokens_seen": 3108110336 + }, + { + "epoch": 0.88, + "learning_rate": 5.921212121212121e-05, + "loss": 2.8471, + "theoretical_loss": 3.311764083452941, + "tokens_seen": 3108175872 + }, + { + "epoch": 0.88, + "learning_rate": 5.919191919191919e-05, + "loss": 2.5999, + "theoretical_loss": 3.31175875317745, + "tokens_seen": 3108241408 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1748010, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.296586513519287, + "objective/train/theoretical_loss": 3.3117534230458117, + "objective/train/tokens_used": 1467166176, + "theoretical_loss": 3.3117534230458117, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 5.9171717171717175e-05, + "loss": 2.5254, + "theoretical_loss": 3.3117534230458117, + "tokens_seen": 3108306944 + }, + { + "epoch": 0.88, + "learning_rate": 5.915151515151515e-05, + "loss": 2.2965, + "theoretical_loss": 3.3117480930580196, + "tokens_seen": 3108372480 + }, + { + "epoch": 0.88, + "learning_rate": 5.9131313131313136e-05, + "loss": 2.6441, + "theoretical_loss": 3.311742763214067, + "tokens_seen": 3108438016 + }, + { + "epoch": 0.88, + "learning_rate": 5.911111111111111e-05, + "loss": 2.534, + "theoretical_loss": 3.3117374335139464, + "tokens_seen": 3108503552 + }, + { + "epoch": 0.88, + "learning_rate": 5.909090909090909e-05, + "loss": 2.6802, + "theoretical_loss": 3.3117321039576515, + "tokens_seen": 3108569088 + }, + { + "epoch": 0.88, + "learning_rate": 5.9070707070707074e-05, + "loss": 2.354, + "theoretical_loss": 3.3117267745451744, + "tokens_seen": 3108634624 + }, + { + "epoch": 0.88, + "learning_rate": 5.905050505050505e-05, + "loss": 2.3622, + "theoretical_loss": 3.3117214452765094, + "tokens_seen": 3108700160 + }, + { + "epoch": 0.88, + "learning_rate": 5.903030303030303e-05, + "loss": 2.5687, + "theoretical_loss": 3.311716116151649, + "tokens_seen": 3108765696 + }, + { + "epoch": 0.88, + "learning_rate": 5.901010101010101e-05, + "loss": 2.6766, + "theoretical_loss": 3.3117107871705858, + "tokens_seen": 3108831232 + }, + { + "epoch": 0.88, + "learning_rate": 5.898989898989899e-05, + "loss": 2.7014, + "theoretical_loss": 3.3117054583333134, + "tokens_seen": 3108896768 + }, + { + "epoch": 0.88, + "learning_rate": 5.8969696969696973e-05, + "loss": 2.6927, + "theoretical_loss": 3.311700129639825, + "tokens_seen": 3108962304 + }, + { + "epoch": 0.88, + "learning_rate": 5.894949494949495e-05, + "loss": 2.5367, + "theoretical_loss": 3.311694801090114, + "tokens_seen": 3109027840 + }, + { + "epoch": 0.88, + "learning_rate": 5.892929292929293e-05, + "loss": 2.5695, + "theoretical_loss": 3.311689472684173, + "tokens_seen": 3109093376 + }, + { + "epoch": 0.88, + "learning_rate": 5.890909090909091e-05, + "loss": 2.5884, + "theoretical_loss": 3.311684144421995, + "tokens_seen": 3109158912 + }, + { + "epoch": 0.88, + "learning_rate": 5.888888888888889e-05, + "loss": 2.5366, + "theoretical_loss": 3.311678816303573, + "tokens_seen": 3109224448 + }, + { + "epoch": 0.88, + "learning_rate": 5.886868686868687e-05, + "loss": 2.8604, + "theoretical_loss": 3.3116734883289003, + "tokens_seen": 3109289984 + }, + { + "epoch": 0.88, + "learning_rate": 5.884848484848485e-05, + "loss": 2.6361, + "theoretical_loss": 3.3116681604979705, + "tokens_seen": 3109355520 + }, + { + "epoch": 0.88, + "learning_rate": 5.882828282828283e-05, + "loss": 2.5455, + "theoretical_loss": 3.311662832810776, + "tokens_seen": 3109421056 + }, + { + "epoch": 0.88, + "learning_rate": 5.880808080808081e-05, + "loss": 2.658, + "theoretical_loss": 3.31165750526731, + "tokens_seen": 3109486592 + }, + { + "epoch": 0.88, + "learning_rate": 5.878787878787879e-05, + "loss": 2.5178, + "theoretical_loss": 3.3116521778675656, + "tokens_seen": 3109552128 + }, + { + "epoch": 0.88, + "learning_rate": 5.8767676767676766e-05, + "loss": 2.5603, + "theoretical_loss": 3.311646850611536, + "tokens_seen": 3109617664 + }, + { + "epoch": 0.88, + "learning_rate": 5.874747474747475e-05, + "loss": 2.5296, + "theoretical_loss": 3.3116415234992145, + "tokens_seen": 3109683200 + }, + { + "epoch": 0.88, + "learning_rate": 5.872727272727273e-05, + "loss": 2.2798, + "theoretical_loss": 3.311636196530594, + "tokens_seen": 3109748736 + }, + { + "epoch": 0.88, + "learning_rate": 5.870707070707071e-05, + "loss": 2.4285, + "theoretical_loss": 3.3116308697056676, + "tokens_seen": 3109814272 + }, + { + "epoch": 0.88, + "learning_rate": 5.868686868686869e-05, + "loss": 2.5308, + "theoretical_loss": 3.3116255430244284, + "tokens_seen": 3109879808 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1748667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.140108346939087, + "objective/train/theoretical_loss": 3.311620216486869, + "objective/train/tokens_used": 1468804576, + "theoretical_loss": 3.311620216486869, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.88, + "learning_rate": 5.8666666666666665e-05, + "loss": 2.5326, + "theoretical_loss": 3.311620216486869, + "tokens_seen": 3109945344 + }, + { + "epoch": 0.88, + "learning_rate": 5.864646464646465e-05, + "loss": 2.7334, + "theoretical_loss": 3.3116148900929834, + "tokens_seen": 3110010880 + }, + { + "epoch": 0.88, + "learning_rate": 5.8626262626262626e-05, + "loss": 2.483, + "theoretical_loss": 3.3116095638427643, + "tokens_seen": 3110076416 + }, + { + "epoch": 0.88, + "learning_rate": 5.860606060606061e-05, + "loss": 2.486, + "theoretical_loss": 3.3116042377362045, + "tokens_seen": 3110141952 + }, + { + "epoch": 0.88, + "learning_rate": 5.858585858585859e-05, + "loss": 2.5462, + "theoretical_loss": 3.311598911773298, + "tokens_seen": 3110207488 + }, + { + "epoch": 0.88, + "learning_rate": 5.8565656565656565e-05, + "loss": 2.6022, + "theoretical_loss": 3.3115935859540366, + "tokens_seen": 3110273024 + }, + { + "epoch": 0.88, + "learning_rate": 5.854545454545455e-05, + "loss": 2.6616, + "theoretical_loss": 3.311588260278414, + "tokens_seen": 3110338560 + }, + { + "epoch": 0.88, + "learning_rate": 5.8525252525252526e-05, + "loss": 2.521, + "theoretical_loss": 3.311582934746424, + "tokens_seen": 3110404096 + }, + { + "epoch": 0.88, + "learning_rate": 5.85050505050505e-05, + "loss": 2.7602, + "theoretical_loss": 3.3115776093580584, + "tokens_seen": 3110469632 + }, + { + "epoch": 0.88, + "learning_rate": 5.848484848484849e-05, + "loss": 2.735, + "theoretical_loss": 3.3115722841133115, + "tokens_seen": 3110535168 + }, + { + "epoch": 0.88, + "learning_rate": 5.8464646464646464e-05, + "loss": 2.5949, + "theoretical_loss": 3.3115669590121755, + "tokens_seen": 3110600704 + }, + { + "epoch": 0.88, + "learning_rate": 5.844444444444445e-05, + "loss": 2.691, + "theoretical_loss": 3.3115616340546437, + "tokens_seen": 3110666240 + }, + { + "epoch": 0.88, + "learning_rate": 5.8424242424242425e-05, + "loss": 2.5718, + "theoretical_loss": 3.31155630924071, + "tokens_seen": 3110731776 + }, + { + "epoch": 0.88, + "learning_rate": 5.84040404040404e-05, + "loss": 2.6201, + "theoretical_loss": 3.311550984570366, + "tokens_seen": 3110797312 + }, + { + "epoch": 0.88, + "learning_rate": 5.8383838383838386e-05, + "loss": 2.674, + "theoretical_loss": 3.311545660043606, + "tokens_seen": 3110862848 + }, + { + "epoch": 0.88, + "learning_rate": 5.8363636363636364e-05, + "loss": 2.4854, + "theoretical_loss": 3.3115403356604234, + "tokens_seen": 3110928384 + }, + { + "epoch": 0.88, + "learning_rate": 5.834343434343435e-05, + "loss": 2.6375, + "theoretical_loss": 3.31153501142081, + "tokens_seen": 3110993920 + }, + { + "epoch": 0.88, + "learning_rate": 5.8323232323232325e-05, + "loss": 2.4235, + "theoretical_loss": 3.31152968732476, + "tokens_seen": 3111059456 + }, + { + "epoch": 0.88, + "learning_rate": 5.83030303030303e-05, + "loss": 2.8686, + "theoretical_loss": 3.311524363372266, + "tokens_seen": 3111124992 + }, + { + "epoch": 0.88, + "learning_rate": 5.8282828282828286e-05, + "loss": 2.5195, + "theoretical_loss": 3.311519039563321, + "tokens_seen": 3111190528 + }, + { + "epoch": 0.88, + "learning_rate": 5.826262626262627e-05, + "loss": 2.4588, + "theoretical_loss": 3.3115137158979184, + "tokens_seen": 3111256064 + }, + { + "epoch": 0.88, + "learning_rate": 5.824242424242424e-05, + "loss": 2.5991, + "theoretical_loss": 3.311508392376051, + "tokens_seen": 3111321600 + }, + { + "epoch": 0.88, + "learning_rate": 5.8222222222222224e-05, + "loss": 2.505, + "theoretical_loss": 3.3115030689977125, + "tokens_seen": 3111387136 + }, + { + "epoch": 0.88, + "learning_rate": 5.82020202020202e-05, + "loss": 2.6266, + "theoretical_loss": 3.311497745762895, + "tokens_seen": 3111452672 + }, + { + "epoch": 0.88, + "learning_rate": 5.8181818181818185e-05, + "loss": 2.6622, + "theoretical_loss": 3.311492422671593, + "tokens_seen": 3111518208 + }, + { + "epoch": 0.88, + "objective/train/docs_used": 1749977, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5630366802215576, + "objective/train/theoretical_loss": 3.3114870997237986, + "objective/train/tokens_used": 1470442976, + "theoretical_loss": 3.3114870997237986, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 5.816161616161616e-05, + "loss": 2.611, + "theoretical_loss": 3.3114870997237986, + "tokens_seen": 3111583744 + }, + { + "epoch": 0.89, + "learning_rate": 5.814141414141414e-05, + "loss": 2.4946, + "theoretical_loss": 3.311481776919505, + "tokens_seen": 3111649280 + }, + { + "epoch": 0.89, + "learning_rate": 5.8121212121212124e-05, + "loss": 2.5003, + "theoretical_loss": 3.3114764542587056, + "tokens_seen": 3111714816 + }, + { + "epoch": 0.89, + "learning_rate": 5.81010101010101e-05, + "loss": 2.1859, + "theoretical_loss": 3.311471131741393, + "tokens_seen": 3111780352 + }, + { + "epoch": 0.89, + "learning_rate": 5.808080808080808e-05, + "loss": 2.573, + "theoretical_loss": 3.311465809367561, + "tokens_seen": 3111845888 + }, + { + "epoch": 0.89, + "learning_rate": 5.806060606060606e-05, + "loss": 2.6599, + "theoretical_loss": 3.3114604871372024, + "tokens_seen": 3111911424 + }, + { + "epoch": 0.89, + "learning_rate": 5.804040404040404e-05, + "loss": 2.726, + "theoretical_loss": 3.31145516505031, + "tokens_seen": 3111976960 + }, + { + "epoch": 0.89, + "learning_rate": 5.802020202020202e-05, + "loss": 2.4554, + "theoretical_loss": 3.3114498431068773, + "tokens_seen": 3112042496 + }, + { + "epoch": 0.89, + "learning_rate": 5.800000000000001e-05, + "loss": 2.6528, + "theoretical_loss": 3.311444521306898, + "tokens_seen": 3112108032 + }, + { + "epoch": 0.89, + "learning_rate": 5.797979797979798e-05, + "loss": 2.6491, + "theoretical_loss": 3.311439199650364, + "tokens_seen": 3112173568 + }, + { + "epoch": 0.89, + "learning_rate": 5.795959595959596e-05, + "loss": 2.715, + "theoretical_loss": 3.3114338781372687, + "tokens_seen": 3112239104 + }, + { + "epoch": 0.89, + "learning_rate": 5.793939393939394e-05, + "loss": 2.5528, + "theoretical_loss": 3.3114285567676056, + "tokens_seen": 3112304640 + }, + { + "epoch": 0.89, + "learning_rate": 5.791919191919192e-05, + "loss": 2.6819, + "theoretical_loss": 3.3114232355413673, + "tokens_seen": 3112370176 + }, + { + "epoch": 0.89, + "learning_rate": 5.78989898989899e-05, + "loss": 2.6573, + "theoretical_loss": 3.311417914458548, + "tokens_seen": 3112435712 + }, + { + "epoch": 0.89, + "learning_rate": 5.787878787878788e-05, + "loss": 2.5535, + "theoretical_loss": 3.31141259351914, + "tokens_seen": 3112501248 + }, + { + "epoch": 0.89, + "learning_rate": 5.785858585858586e-05, + "loss": 2.3284, + "theoretical_loss": 3.311407272723136, + "tokens_seen": 3112566784 + }, + { + "epoch": 0.89, + "learning_rate": 5.7838383838383845e-05, + "loss": 2.6149, + "theoretical_loss": 3.31140195207053, + "tokens_seen": 3112632320 + }, + { + "epoch": 0.89, + "learning_rate": 5.7818181818181815e-05, + "loss": 2.4798, + "theoretical_loss": 3.3113966315613146, + "tokens_seen": 3112697856 + }, + { + "epoch": 0.89, + "learning_rate": 5.77979797979798e-05, + "loss": 2.5172, + "theoretical_loss": 3.311391311195483, + "tokens_seen": 3112763392 + }, + { + "epoch": 0.89, + "learning_rate": 5.7777777777777776e-05, + "loss": 2.7133, + "theoretical_loss": 3.311385990973029, + "tokens_seen": 3112828928 + }, + { + "epoch": 0.89, + "learning_rate": 5.775757575757576e-05, + "loss": 2.7984, + "theoretical_loss": 3.3113806708939446, + "tokens_seen": 3112894464 + }, + { + "epoch": 0.89, + "learning_rate": 5.7737373737373744e-05, + "loss": 2.5749, + "theoretical_loss": 3.3113753509582233, + "tokens_seen": 3112960000 + }, + { + "epoch": 0.89, + "learning_rate": 5.7717171717171715e-05, + "loss": 2.6864, + "theoretical_loss": 3.3113700311658585, + "tokens_seen": 3113025536 + }, + { + "epoch": 0.89, + "learning_rate": 5.76969696969697e-05, + "loss": 2.659, + "theoretical_loss": 3.3113647115168434, + "tokens_seen": 3113091072 + }, + { + "epoch": 0.89, + "learning_rate": 5.7676767676767676e-05, + "loss": 2.5185, + "theoretical_loss": 3.3113593920111706, + "tokens_seen": 3113156608 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1750566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7993392944335938, + "objective/train/theoretical_loss": 3.3113540726488333, + "objective/train/tokens_used": 1472081376, + "theoretical_loss": 3.3113540726488333, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.89, + "learning_rate": 5.765656565656566e-05, + "loss": 2.6406, + "theoretical_loss": 3.3113540726488333, + "tokens_seen": 3113222144 + }, + { + "epoch": 0.89, + "learning_rate": 5.763636363636364e-05, + "loss": 2.6617, + "theoretical_loss": 3.3113487534298254, + "tokens_seen": 3113287680 + }, + { + "epoch": 0.89, + "learning_rate": 5.7616161616161614e-05, + "loss": 2.656, + "theoretical_loss": 3.3113434343541392, + "tokens_seen": 3113353216 + }, + { + "epoch": 0.89, + "learning_rate": 5.75959595959596e-05, + "loss": 2.5115, + "theoretical_loss": 3.311338115421768, + "tokens_seen": 3113418752 + }, + { + "epoch": 0.89, + "learning_rate": 5.757575757575758e-05, + "loss": 2.6196, + "theoretical_loss": 3.311332796632705, + "tokens_seen": 3113484288 + }, + { + "epoch": 0.89, + "learning_rate": 5.755555555555555e-05, + "loss": 2.416, + "theoretical_loss": 3.311327477986943, + "tokens_seen": 3113549824 + }, + { + "epoch": 0.89, + "learning_rate": 5.7535353535353537e-05, + "loss": 2.1652, + "theoretical_loss": 3.3113221594844755, + "tokens_seen": 3113615360 + }, + { + "epoch": 0.89, + "learning_rate": 5.7515151515151514e-05, + "loss": 2.4952, + "theoretical_loss": 3.311316841125296, + "tokens_seen": 3113680896 + }, + { + "epoch": 0.89, + "learning_rate": 5.74949494949495e-05, + "loss": 2.3928, + "theoretical_loss": 3.3113115229093966, + "tokens_seen": 3113746432 + }, + { + "epoch": 0.89, + "learning_rate": 5.7474747474747475e-05, + "loss": 2.6933, + "theoretical_loss": 3.311306204836771, + "tokens_seen": 3113811968 + }, + { + "epoch": 0.89, + "learning_rate": 5.745454545454545e-05, + "loss": 2.7341, + "theoretical_loss": 3.311300886907413, + "tokens_seen": 3113877504 + }, + { + "epoch": 0.89, + "learning_rate": 5.7434343434343436e-05, + "loss": 2.5679, + "theoretical_loss": 3.311295569121315, + "tokens_seen": 3113943040 + }, + { + "epoch": 0.89, + "learning_rate": 5.741414141414142e-05, + "loss": 2.482, + "theoretical_loss": 3.3112902514784697, + "tokens_seen": 3114008576 + }, + { + "epoch": 0.89, + "learning_rate": 5.73939393939394e-05, + "loss": 2.7524, + "theoretical_loss": 3.3112849339788704, + "tokens_seen": 3114074112 + }, + { + "epoch": 0.89, + "learning_rate": 5.7373737373737374e-05, + "loss": 2.5268, + "theoretical_loss": 3.311279616622511, + "tokens_seen": 3114139648 + }, + { + "epoch": 0.89, + "learning_rate": 5.735353535353535e-05, + "loss": 2.6072, + "theoretical_loss": 3.311274299409384, + "tokens_seen": 3114205184 + }, + { + "epoch": 0.89, + "learning_rate": 5.7333333333333336e-05, + "loss": 2.6578, + "theoretical_loss": 3.3112689823394827, + "tokens_seen": 3114270720 + }, + { + "epoch": 0.89, + "learning_rate": 5.731313131313132e-05, + "loss": 2.6436, + "theoretical_loss": 3.3112636654128007, + "tokens_seen": 3114336256 + }, + { + "epoch": 0.89, + "learning_rate": 5.729292929292929e-05, + "loss": 2.5636, + "theoretical_loss": 3.31125834862933, + "tokens_seen": 3114401792 + }, + { + "epoch": 0.89, + "learning_rate": 5.7272727272727274e-05, + "loss": 2.4206, + "theoretical_loss": 3.3112530319890645, + "tokens_seen": 3114467328 + }, + { + "epoch": 0.89, + "learning_rate": 5.725252525252525e-05, + "loss": 2.511, + "theoretical_loss": 3.3112477154919975, + "tokens_seen": 3114532864 + }, + { + "epoch": 0.89, + "learning_rate": 5.7232323232323235e-05, + "loss": 2.3362, + "theoretical_loss": 3.3112423991381217, + "tokens_seen": 3114598400 + }, + { + "epoch": 0.89, + "learning_rate": 5.721212121212121e-05, + "loss": 2.7106, + "theoretical_loss": 3.31123708292743, + "tokens_seen": 3114663936 + }, + { + "epoch": 0.89, + "learning_rate": 5.719191919191919e-05, + "loss": 2.5036, + "theoretical_loss": 3.3112317668599163, + "tokens_seen": 3114729472 + }, + { + "epoch": 0.89, + "learning_rate": 5.717171717171717e-05, + "loss": 2.5496, + "theoretical_loss": 3.311226450935573, + "tokens_seen": 3114795008 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1751732, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4826302528381348, + "objective/train/theoretical_loss": 3.311221135154394, + "objective/train/tokens_used": 1473719776, + "theoretical_loss": 3.311221135154394, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 5.715151515151516e-05, + "loss": 2.4438, + "theoretical_loss": 3.311221135154394, + "tokens_seen": 3114860544 + }, + { + "epoch": 0.89, + "learning_rate": 5.7131313131313134e-05, + "loss": 2.5559, + "theoretical_loss": 3.3112158195163715, + "tokens_seen": 3114926080 + }, + { + "epoch": 0.89, + "learning_rate": 5.711111111111111e-05, + "loss": 2.3677, + "theoretical_loss": 3.311210504021499, + "tokens_seen": 3114991616 + }, + { + "epoch": 0.89, + "learning_rate": 5.709090909090909e-05, + "loss": 2.2788, + "theoretical_loss": 3.3112051886697706, + "tokens_seen": 3115057152 + }, + { + "epoch": 0.89, + "learning_rate": 5.707070707070707e-05, + "loss": 2.6419, + "theoretical_loss": 3.311199873461178, + "tokens_seen": 3115122688 + }, + { + "epoch": 0.89, + "learning_rate": 5.705050505050506e-05, + "loss": 2.5663, + "theoretical_loss": 3.311194558395715, + "tokens_seen": 3115188224 + }, + { + "epoch": 0.89, + "learning_rate": 5.703030303030303e-05, + "loss": 2.5675, + "theoretical_loss": 3.3111892434733745, + "tokens_seen": 3115253760 + }, + { + "epoch": 0.89, + "learning_rate": 5.701010101010101e-05, + "loss": 2.5083, + "theoretical_loss": 3.31118392869415, + "tokens_seen": 3115319296 + }, + { + "epoch": 0.89, + "learning_rate": 5.6989898989898995e-05, + "loss": 2.5787, + "theoretical_loss": 3.3111786140580346, + "tokens_seen": 3115384832 + }, + { + "epoch": 0.89, + "learning_rate": 5.696969696969697e-05, + "loss": 2.5921, + "theoretical_loss": 3.311173299565021, + "tokens_seen": 3115450368 + }, + { + "epoch": 0.89, + "learning_rate": 5.694949494949495e-05, + "loss": 2.598, + "theoretical_loss": 3.3111679852151026, + "tokens_seen": 3115515904 + }, + { + "epoch": 0.89, + "learning_rate": 5.692929292929293e-05, + "loss": 2.55, + "theoretical_loss": 3.3111626710082724, + "tokens_seen": 3115581440 + }, + { + "epoch": 0.89, + "learning_rate": 5.690909090909091e-05, + "loss": 2.4823, + "theoretical_loss": 3.311157356944524, + "tokens_seen": 3115646976 + }, + { + "epoch": 0.89, + "learning_rate": 5.6888888888888895e-05, + "loss": 2.4568, + "theoretical_loss": 3.3111520430238497, + "tokens_seen": 3115712512 + }, + { + "epoch": 0.89, + "learning_rate": 5.6868686868686865e-05, + "loss": 2.7302, + "theoretical_loss": 3.3111467292462438, + "tokens_seen": 3115778048 + }, + { + "epoch": 0.89, + "learning_rate": 5.684848484848485e-05, + "loss": 2.6591, + "theoretical_loss": 3.311141415611698, + "tokens_seen": 3115843584 + }, + { + "epoch": 0.89, + "learning_rate": 5.6828282828282826e-05, + "loss": 2.6082, + "theoretical_loss": 3.311136102120207, + "tokens_seen": 3115909120 + }, + { + "epoch": 0.89, + "learning_rate": 5.680808080808081e-05, + "loss": 2.6066, + "theoretical_loss": 3.311130788771763, + "tokens_seen": 3115974656 + }, + { + "epoch": 0.89, + "learning_rate": 5.6787878787878794e-05, + "loss": 2.7288, + "theoretical_loss": 3.311125475566359, + "tokens_seen": 3116040192 + }, + { + "epoch": 0.89, + "learning_rate": 5.6767676767676764e-05, + "loss": 2.3587, + "theoretical_loss": 3.3111201625039888, + "tokens_seen": 3116105728 + }, + { + "epoch": 0.89, + "learning_rate": 5.674747474747475e-05, + "loss": 2.6729, + "theoretical_loss": 3.3111148495846447, + "tokens_seen": 3116171264 + }, + { + "epoch": 0.89, + "learning_rate": 5.672727272727273e-05, + "loss": 2.6981, + "theoretical_loss": 3.3111095368083205, + "tokens_seen": 3116236800 + }, + { + "epoch": 0.89, + "learning_rate": 5.670707070707071e-05, + "loss": 2.5472, + "theoretical_loss": 3.3111042241750095, + "tokens_seen": 3116302336 + }, + { + "epoch": 0.89, + "learning_rate": 5.668686868686869e-05, + "loss": 2.5853, + "theoretical_loss": 3.311098911684704, + "tokens_seen": 3116367872 + }, + { + "epoch": 0.89, + "learning_rate": 5.6666666666666664e-05, + "loss": 2.3444, + "theoretical_loss": 3.311093599337398, + "tokens_seen": 3116433408 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1752323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2536070346832275, + "objective/train/theoretical_loss": 3.3110882871330847, + "objective/train/tokens_used": 1475358176, + "theoretical_loss": 3.3110882871330847, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.89, + "learning_rate": 5.664646464646465e-05, + "loss": 2.4691, + "theoretical_loss": 3.3110882871330847, + "tokens_seen": 3116498944 + }, + { + "epoch": 0.89, + "learning_rate": 5.662626262626263e-05, + "loss": 2.4115, + "theoretical_loss": 3.3110829750717565, + "tokens_seen": 3116564480 + }, + { + "epoch": 0.89, + "learning_rate": 5.66060606060606e-05, + "loss": 2.6224, + "theoretical_loss": 3.311077663153407, + "tokens_seen": 3116630016 + }, + { + "epoch": 0.89, + "learning_rate": 5.6585858585858586e-05, + "loss": 2.4308, + "theoretical_loss": 3.3110723513780287, + "tokens_seen": 3116695552 + }, + { + "epoch": 0.89, + "learning_rate": 5.656565656565657e-05, + "loss": 2.4285, + "theoretical_loss": 3.3110670397456157, + "tokens_seen": 3116761088 + }, + { + "epoch": 0.89, + "learning_rate": 5.654545454545455e-05, + "loss": 2.4314, + "theoretical_loss": 3.3110617282561607, + "tokens_seen": 3116826624 + }, + { + "epoch": 0.89, + "learning_rate": 5.652525252525253e-05, + "loss": 2.4591, + "theoretical_loss": 3.311056416909657, + "tokens_seen": 3116892160 + }, + { + "epoch": 0.89, + "learning_rate": 5.65050505050505e-05, + "loss": 2.4011, + "theoretical_loss": 3.311051105706097, + "tokens_seen": 3116957696 + }, + { + "epoch": 0.89, + "learning_rate": 5.6484848484848486e-05, + "loss": 2.3756, + "theoretical_loss": 3.311045794645475, + "tokens_seen": 3117023232 + }, + { + "epoch": 0.89, + "learning_rate": 5.646464646464647e-05, + "loss": 2.4835, + "theoretical_loss": 3.311040483727784, + "tokens_seen": 3117088768 + }, + { + "epoch": 0.89, + "learning_rate": 5.644444444444445e-05, + "loss": 2.5277, + "theoretical_loss": 3.311035172953016, + "tokens_seen": 3117154304 + }, + { + "epoch": 0.89, + "learning_rate": 5.6424242424242424e-05, + "loss": 2.6127, + "theoretical_loss": 3.311029862321165, + "tokens_seen": 3117219840 + }, + { + "epoch": 0.89, + "learning_rate": 5.64040404040404e-05, + "loss": 2.8772, + "theoretical_loss": 3.3110245518322245, + "tokens_seen": 3117285376 + }, + { + "epoch": 0.89, + "learning_rate": 5.6383838383838385e-05, + "loss": 2.5857, + "theoretical_loss": 3.3110192414861874, + "tokens_seen": 3117350912 + }, + { + "epoch": 0.89, + "learning_rate": 5.636363636363637e-05, + "loss": 2.7049, + "theoretical_loss": 3.311013931283046, + "tokens_seen": 3117416448 + }, + { + "epoch": 0.89, + "learning_rate": 5.634343434343434e-05, + "loss": 2.5618, + "theoretical_loss": 3.3110086212227947, + "tokens_seen": 3117481984 + }, + { + "epoch": 0.89, + "learning_rate": 5.6323232323232324e-05, + "loss": 2.5696, + "theoretical_loss": 3.311003311305426, + "tokens_seen": 3117547520 + }, + { + "epoch": 0.89, + "learning_rate": 5.630303030303031e-05, + "loss": 2.6608, + "theoretical_loss": 3.3109980015309324, + "tokens_seen": 3117613056 + }, + { + "epoch": 0.89, + "learning_rate": 5.6282828282828285e-05, + "loss": 2.2851, + "theoretical_loss": 3.3109926918993087, + "tokens_seen": 3117678592 + }, + { + "epoch": 0.89, + "learning_rate": 5.626262626262627e-05, + "loss": 2.7093, + "theoretical_loss": 3.3109873824105467, + "tokens_seen": 3117744128 + }, + { + "epoch": 0.89, + "learning_rate": 5.624242424242424e-05, + "loss": 2.6061, + "theoretical_loss": 3.3109820730646398, + "tokens_seen": 3117809664 + }, + { + "epoch": 0.89, + "learning_rate": 5.622222222222222e-05, + "loss": 2.6193, + "theoretical_loss": 3.3109767638615817, + "tokens_seen": 3117875200 + }, + { + "epoch": 0.89, + "learning_rate": 5.620202020202021e-05, + "loss": 2.5402, + "theoretical_loss": 3.310971454801365, + "tokens_seen": 3117940736 + }, + { + "epoch": 0.89, + "learning_rate": 5.6181818181818184e-05, + "loss": 2.4369, + "theoretical_loss": 3.3109661458839827, + "tokens_seen": 3118006272 + }, + { + "epoch": 0.89, + "learning_rate": 5.616161616161616e-05, + "loss": 2.6181, + "theoretical_loss": 3.3109608371094286, + "tokens_seen": 3118071808 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1753489, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4775819778442383, + "objective/train/theoretical_loss": 3.3109555284776957, + "objective/train/tokens_used": 1476996576, + "theoretical_loss": 3.3109555284776957, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 5.6141414141414145e-05, + "loss": 2.4935, + "theoretical_loss": 3.3109555284776957, + "tokens_seen": 3118137344 + }, + { + "epoch": 0.89, + "learning_rate": 5.612121212121212e-05, + "loss": 2.4487, + "theoretical_loss": 3.310950219988777, + "tokens_seen": 3118202880 + }, + { + "epoch": 0.89, + "learning_rate": 5.6101010101010106e-05, + "loss": 2.3578, + "theoretical_loss": 3.3109449116426655, + "tokens_seen": 3118268416 + }, + { + "epoch": 0.89, + "learning_rate": 5.608080808080808e-05, + "loss": 2.5082, + "theoretical_loss": 3.310939603439355, + "tokens_seen": 3118333952 + }, + { + "epoch": 0.89, + "learning_rate": 5.606060606060606e-05, + "loss": 2.6299, + "theoretical_loss": 3.3109342953788374, + "tokens_seen": 3118399488 + }, + { + "epoch": 0.89, + "learning_rate": 5.6040404040404045e-05, + "loss": 2.5853, + "theoretical_loss": 3.310928987461107, + "tokens_seen": 3118465024 + }, + { + "epoch": 0.89, + "learning_rate": 5.602020202020202e-05, + "loss": 2.7055, + "theoretical_loss": 3.310923679686157, + "tokens_seen": 3118530560 + }, + { + "epoch": 0.89, + "learning_rate": 5.6e-05, + "loss": 2.4513, + "theoretical_loss": 3.3109183720539797, + "tokens_seen": 3118596096 + }, + { + "epoch": 0.89, + "learning_rate": 5.5979797979797976e-05, + "loss": 2.4992, + "theoretical_loss": 3.3109130645645686, + "tokens_seen": 3118661632 + }, + { + "epoch": 0.89, + "learning_rate": 5.595959595959596e-05, + "loss": 2.6164, + "theoretical_loss": 3.310907757217917, + "tokens_seen": 3118727168 + }, + { + "epoch": 0.89, + "learning_rate": 5.5939393939393944e-05, + "loss": 2.485, + "theoretical_loss": 3.3109024500140185, + "tokens_seen": 3118792704 + }, + { + "epoch": 0.89, + "learning_rate": 5.591919191919192e-05, + "loss": 2.4848, + "theoretical_loss": 3.3108971429528653, + "tokens_seen": 3118858240 + }, + { + "epoch": 0.89, + "learning_rate": 5.58989898989899e-05, + "loss": 2.7161, + "theoretical_loss": 3.3108918360344513, + "tokens_seen": 3118923776 + }, + { + "epoch": 0.89, + "learning_rate": 5.587878787878788e-05, + "loss": 2.5384, + "theoretical_loss": 3.3108865292587693, + "tokens_seen": 3118989312 + }, + { + "epoch": 0.89, + "learning_rate": 5.585858585858586e-05, + "loss": 2.4785, + "theoretical_loss": 3.3108812226258126, + "tokens_seen": 3119054848 + }, + { + "epoch": 0.89, + "learning_rate": 5.5838383838383844e-05, + "loss": 2.6025, + "theoretical_loss": 3.3108759161355743, + "tokens_seen": 3119120384 + }, + { + "epoch": 0.89, + "learning_rate": 5.5818181818181814e-05, + "loss": 2.5995, + "theoretical_loss": 3.310870609788048, + "tokens_seen": 3119185920 + }, + { + "epoch": 0.89, + "learning_rate": 5.57979797979798e-05, + "loss": 2.6059, + "theoretical_loss": 3.310865303583226, + "tokens_seen": 3119251456 + }, + { + "epoch": 0.89, + "learning_rate": 5.577777777777778e-05, + "loss": 2.5996, + "theoretical_loss": 3.310859997521102, + "tokens_seen": 3119316992 + }, + { + "epoch": 0.89, + "learning_rate": 5.575757575757576e-05, + "loss": 2.6636, + "theoretical_loss": 3.3108546916016692, + "tokens_seen": 3119382528 + }, + { + "epoch": 0.89, + "learning_rate": 5.5737373737373736e-05, + "loss": 2.7654, + "theoretical_loss": 3.3108493858249206, + "tokens_seen": 3119448064 + }, + { + "epoch": 0.89, + "learning_rate": 5.5717171717171714e-05, + "loss": 2.5765, + "theoretical_loss": 3.3108440801908494, + "tokens_seen": 3119513600 + }, + { + "epoch": 0.89, + "learning_rate": 5.56969696969697e-05, + "loss": 2.6388, + "theoretical_loss": 3.310838774699449, + "tokens_seen": 3119579136 + }, + { + "epoch": 0.89, + "learning_rate": 5.567676767676768e-05, + "loss": 2.3593, + "theoretical_loss": 3.310833469350712, + "tokens_seen": 3119644672 + }, + { + "epoch": 0.89, + "learning_rate": 5.565656565656565e-05, + "loss": 2.6107, + "theoretical_loss": 3.3108281641446324, + "tokens_seen": 3119710208 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1754072, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.788405179977417, + "objective/train/theoretical_loss": 3.3108228590812026, + "objective/train/tokens_used": 1478634976, + "theoretical_loss": 3.3108228590812026, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.89, + "learning_rate": 5.5636363636363636e-05, + "loss": 2.741, + "theoretical_loss": 3.3108228590812026, + "tokens_seen": 3119775744 + }, + { + "epoch": 0.89, + "learning_rate": 5.561616161616162e-05, + "loss": 2.6959, + "theoretical_loss": 3.3108175541604163, + "tokens_seen": 3119841280 + }, + { + "epoch": 0.89, + "learning_rate": 5.55959595959596e-05, + "loss": 2.4466, + "theoretical_loss": 3.3108122493822663, + "tokens_seen": 3119906816 + }, + { + "epoch": 0.89, + "learning_rate": 5.557575757575758e-05, + "loss": 2.3802, + "theoretical_loss": 3.3108069447467456, + "tokens_seen": 3119972352 + }, + { + "epoch": 0.89, + "learning_rate": 5.555555555555555e-05, + "loss": 2.727, + "theoretical_loss": 3.310801640253848, + "tokens_seen": 3120037888 + }, + { + "epoch": 0.89, + "learning_rate": 5.5535353535353535e-05, + "loss": 2.2968, + "theoretical_loss": 3.310796335903566, + "tokens_seen": 3120103424 + }, + { + "epoch": 0.89, + "learning_rate": 5.551515151515152e-05, + "loss": 2.6224, + "theoretical_loss": 3.3107910316958935, + "tokens_seen": 3120168960 + }, + { + "epoch": 0.89, + "learning_rate": 5.5494949494949497e-05, + "loss": 2.8206, + "theoretical_loss": 3.3107857276308232, + "tokens_seen": 3120234496 + }, + { + "epoch": 0.89, + "learning_rate": 5.5474747474747474e-05, + "loss": 2.4617, + "theoretical_loss": 3.3107804237083482, + "tokens_seen": 3120300032 + }, + { + "epoch": 0.89, + "learning_rate": 5.545454545454546e-05, + "loss": 2.5168, + "theoretical_loss": 3.310775119928462, + "tokens_seen": 3120365568 + }, + { + "epoch": 0.89, + "learning_rate": 5.5434343434343435e-05, + "loss": 2.5715, + "theoretical_loss": 3.310769816291158, + "tokens_seen": 3120431104 + }, + { + "epoch": 0.89, + "learning_rate": 5.541414141414142e-05, + "loss": 2.4327, + "theoretical_loss": 3.3107645127964282, + "tokens_seen": 3120496640 + }, + { + "epoch": 0.89, + "learning_rate": 5.539393939393939e-05, + "loss": 2.6662, + "theoretical_loss": 3.310759209444267, + "tokens_seen": 3120562176 + }, + { + "epoch": 0.89, + "learning_rate": 5.537373737373737e-05, + "loss": 2.5041, + "theoretical_loss": 3.3107539062346665, + "tokens_seen": 3120627712 + }, + { + "epoch": 0.89, + "learning_rate": 5.535353535353536e-05, + "loss": 2.6466, + "theoretical_loss": 3.310748603167621, + "tokens_seen": 3120693248 + }, + { + "epoch": 0.89, + "learning_rate": 5.5333333333333334e-05, + "loss": 2.2076, + "theoretical_loss": 3.3107433002431232, + "tokens_seen": 3120758784 + }, + { + "epoch": 0.89, + "learning_rate": 5.531313131313132e-05, + "loss": 2.4733, + "theoretical_loss": 3.310737997461166, + "tokens_seen": 3120824320 + }, + { + "epoch": 0.89, + "learning_rate": 5.529292929292929e-05, + "loss": 2.864, + "theoretical_loss": 3.310732694821743, + "tokens_seen": 3120889856 + }, + { + "epoch": 0.89, + "learning_rate": 5.527272727272727e-05, + "loss": 2.5146, + "theoretical_loss": 3.3107273923248473, + "tokens_seen": 3120955392 + }, + { + "epoch": 0.89, + "learning_rate": 5.525252525252526e-05, + "loss": 2.7065, + "theoretical_loss": 3.3107220899704717, + "tokens_seen": 3121020928 + }, + { + "epoch": 0.89, + "learning_rate": 5.5232323232323234e-05, + "loss": 2.4459, + "theoretical_loss": 3.3107167877586097, + "tokens_seen": 3121086464 + }, + { + "epoch": 0.89, + "learning_rate": 5.521212121212121e-05, + "loss": 2.508, + "theoretical_loss": 3.3107114856892546, + "tokens_seen": 3121152000 + }, + { + "epoch": 0.89, + "learning_rate": 5.5191919191919195e-05, + "loss": 2.6577, + "theoretical_loss": 3.310706183762399, + "tokens_seen": 3121217536 + }, + { + "epoch": 0.89, + "learning_rate": 5.517171717171717e-05, + "loss": 2.4031, + "theoretical_loss": 3.310700881978037, + "tokens_seen": 3121283072 + }, + { + "epoch": 0.89, + "learning_rate": 5.5151515151515156e-05, + "loss": 2.7056, + "theoretical_loss": 3.310695580336161, + "tokens_seen": 3121348608 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1755494, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6306960582733154, + "objective/train/theoretical_loss": 3.310690278836764, + "objective/train/tokens_used": 1480273376, + "theoretical_loss": 3.310690278836764, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 5.5131313131313127e-05, + "loss": 2.5082, + "theoretical_loss": 3.310690278836764, + "tokens_seen": 3121414144 + }, + { + "epoch": 0.89, + "learning_rate": 5.511111111111111e-05, + "loss": 2.5893, + "theoretical_loss": 3.3106849774798404, + "tokens_seen": 3121479680 + }, + { + "epoch": 0.89, + "learning_rate": 5.5090909090909094e-05, + "loss": 2.5626, + "theoretical_loss": 3.310679676265382, + "tokens_seen": 3121545216 + }, + { + "epoch": 0.89, + "learning_rate": 5.507070707070707e-05, + "loss": 2.5663, + "theoretical_loss": 3.3106743751933827, + "tokens_seen": 3121610752 + }, + { + "epoch": 0.89, + "learning_rate": 5.5050505050505056e-05, + "loss": 2.34, + "theoretical_loss": 3.310669074263836, + "tokens_seen": 3121676288 + }, + { + "epoch": 0.89, + "learning_rate": 5.503030303030303e-05, + "loss": 2.5606, + "theoretical_loss": 3.310663773476734, + "tokens_seen": 3121741824 + }, + { + "epoch": 0.89, + "learning_rate": 5.501010101010101e-05, + "loss": 2.5695, + "theoretical_loss": 3.310658472832071, + "tokens_seen": 3121807360 + }, + { + "epoch": 0.89, + "learning_rate": 5.4989898989898994e-05, + "loss": 2.6706, + "theoretical_loss": 3.3106531723298396, + "tokens_seen": 3121872896 + }, + { + "epoch": 0.89, + "learning_rate": 5.496969696969697e-05, + "loss": 2.5111, + "theoretical_loss": 3.3106478719700325, + "tokens_seen": 3121938432 + }, + { + "epoch": 0.89, + "learning_rate": 5.494949494949495e-05, + "loss": 2.4962, + "theoretical_loss": 3.310642571752644, + "tokens_seen": 3122003968 + }, + { + "epoch": 0.89, + "learning_rate": 5.492929292929293e-05, + "loss": 2.5839, + "theoretical_loss": 3.310637271677667, + "tokens_seen": 3122069504 + }, + { + "epoch": 0.89, + "learning_rate": 5.490909090909091e-05, + "loss": 2.6157, + "theoretical_loss": 3.310631971745094, + "tokens_seen": 3122135040 + }, + { + "epoch": 0.89, + "learning_rate": 5.4888888888888893e-05, + "loss": 2.6321, + "theoretical_loss": 3.3106266719549184, + "tokens_seen": 3122200576 + }, + { + "epoch": 0.89, + "learning_rate": 5.4868686868686864e-05, + "loss": 2.6668, + "theoretical_loss": 3.3106213723071343, + "tokens_seen": 3122266112 + }, + { + "epoch": 0.89, + "learning_rate": 5.484848484848485e-05, + "loss": 2.546, + "theoretical_loss": 3.310616072801734, + "tokens_seen": 3122331648 + }, + { + "epoch": 0.89, + "learning_rate": 5.482828282828283e-05, + "loss": 2.7177, + "theoretical_loss": 3.3106107734387105, + "tokens_seen": 3122397184 + }, + { + "epoch": 0.89, + "learning_rate": 5.480808080808081e-05, + "loss": 2.4654, + "theoretical_loss": 3.3106054742180575, + "tokens_seen": 3122462720 + }, + { + "epoch": 0.89, + "learning_rate": 5.4787878787878786e-05, + "loss": 2.4153, + "theoretical_loss": 3.3106001751397685, + "tokens_seen": 3122528256 + }, + { + "epoch": 0.89, + "learning_rate": 5.476767676767677e-05, + "loss": 2.5519, + "theoretical_loss": 3.3105948762038357, + "tokens_seen": 3122593792 + }, + { + "epoch": 0.89, + "learning_rate": 5.474747474747475e-05, + "loss": 2.6024, + "theoretical_loss": 3.3105895774102527, + "tokens_seen": 3122659328 + }, + { + "epoch": 0.89, + "learning_rate": 5.472727272727273e-05, + "loss": 2.7698, + "theoretical_loss": 3.310584278759013, + "tokens_seen": 3122724864 + }, + { + "epoch": 0.89, + "learning_rate": 5.470707070707071e-05, + "loss": 2.5392, + "theoretical_loss": 3.31057898025011, + "tokens_seen": 3122790400 + }, + { + "epoch": 0.89, + "learning_rate": 5.4686868686868686e-05, + "loss": 2.4208, + "theoretical_loss": 3.310573681883536, + "tokens_seen": 3122855936 + }, + { + "epoch": 0.89, + "learning_rate": 5.466666666666667e-05, + "loss": 2.4302, + "theoretical_loss": 3.310568383659285, + "tokens_seen": 3122921472 + }, + { + "epoch": 0.89, + "learning_rate": 5.464646464646465e-05, + "loss": 2.6166, + "theoretical_loss": 3.31056308557735, + "tokens_seen": 3122987008 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1755986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9578261375427246, + "objective/train/theoretical_loss": 3.3105577876377237, + "objective/train/tokens_used": 1481911776, + "theoretical_loss": 3.3105577876377237, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.89, + "learning_rate": 5.462626262626263e-05, + "loss": 2.7649, + "theoretical_loss": 3.3105577876377237, + "tokens_seen": 3123052544 + }, + { + "epoch": 0.89, + "learning_rate": 5.460606060606061e-05, + "loss": 2.5728, + "theoretical_loss": 3.3105524898403997, + "tokens_seen": 3123118080 + }, + { + "epoch": 0.89, + "learning_rate": 5.4585858585858585e-05, + "loss": 2.3242, + "theoretical_loss": 3.3105471921853713, + "tokens_seen": 3123183616 + }, + { + "epoch": 0.89, + "learning_rate": 5.456565656565657e-05, + "loss": 2.6394, + "theoretical_loss": 3.3105418946726313, + "tokens_seen": 3123249152 + }, + { + "epoch": 0.89, + "learning_rate": 5.4545454545454546e-05, + "loss": 2.3094, + "theoretical_loss": 3.3105365973021734, + "tokens_seen": 3123314688 + }, + { + "epoch": 0.89, + "learning_rate": 5.4525252525252523e-05, + "loss": 2.3615, + "theoretical_loss": 3.31053130007399, + "tokens_seen": 3123380224 + }, + { + "epoch": 0.89, + "learning_rate": 5.450505050505051e-05, + "loss": 2.4644, + "theoretical_loss": 3.310526002988076, + "tokens_seen": 3123445760 + }, + { + "epoch": 0.89, + "learning_rate": 5.4484848484848485e-05, + "loss": 2.5025, + "theoretical_loss": 3.3105207060444224, + "tokens_seen": 3123511296 + }, + { + "epoch": 0.89, + "learning_rate": 5.446464646464647e-05, + "loss": 2.5755, + "theoretical_loss": 3.3105154092430236, + "tokens_seen": 3123576832 + }, + { + "epoch": 0.89, + "learning_rate": 5.444444444444444e-05, + "loss": 2.4897, + "theoretical_loss": 3.3105101125838727, + "tokens_seen": 3123642368 + }, + { + "epoch": 0.89, + "learning_rate": 5.442424242424242e-05, + "loss": 2.6777, + "theoretical_loss": 3.3105048160669623, + "tokens_seen": 3123707904 + }, + { + "epoch": 0.89, + "learning_rate": 5.440404040404041e-05, + "loss": 2.7403, + "theoretical_loss": 3.3104995196922866, + "tokens_seen": 3123773440 + }, + { + "epoch": 0.89, + "learning_rate": 5.4383838383838384e-05, + "loss": 2.3955, + "theoretical_loss": 3.3104942234598385, + "tokens_seen": 3123838976 + }, + { + "epoch": 0.89, + "learning_rate": 5.436363636363637e-05, + "loss": 2.5161, + "theoretical_loss": 3.3104889273696108, + "tokens_seen": 3123904512 + }, + { + "epoch": 0.89, + "learning_rate": 5.4343434343434345e-05, + "loss": 2.553, + "theoretical_loss": 3.310483631421597, + "tokens_seen": 3123970048 + }, + { + "epoch": 0.89, + "learning_rate": 5.432323232323232e-05, + "loss": 2.4295, + "theoretical_loss": 3.3104783356157896, + "tokens_seen": 3124035584 + }, + { + "epoch": 0.89, + "learning_rate": 5.4303030303030306e-05, + "loss": 2.5543, + "theoretical_loss": 3.310473039952183, + "tokens_seen": 3124101120 + }, + { + "epoch": 0.89, + "learning_rate": 5.4282828282828284e-05, + "loss": 2.1668, + "theoretical_loss": 3.3104677444307695, + "tokens_seen": 3124166656 + }, + { + "epoch": 0.89, + "learning_rate": 5.426262626262626e-05, + "loss": 2.3503, + "theoretical_loss": 3.3104624490515424, + "tokens_seen": 3124232192 + }, + { + "epoch": 0.89, + "learning_rate": 5.4242424242424245e-05, + "loss": 2.567, + "theoretical_loss": 3.3104571538144953, + "tokens_seen": 3124297728 + }, + { + "epoch": 0.89, + "learning_rate": 5.422222222222222e-05, + "loss": 2.423, + "theoretical_loss": 3.310451858719621, + "tokens_seen": 3124363264 + }, + { + "epoch": 0.89, + "learning_rate": 5.4202020202020206e-05, + "loss": 2.4868, + "theoretical_loss": 3.3104465637669134, + "tokens_seen": 3124428800 + }, + { + "epoch": 0.89, + "learning_rate": 5.418181818181818e-05, + "loss": 2.6218, + "theoretical_loss": 3.3104412689563647, + "tokens_seen": 3124494336 + }, + { + "epoch": 0.89, + "learning_rate": 5.416161616161616e-05, + "loss": 2.4627, + "theoretical_loss": 3.310435974287969, + "tokens_seen": 3124559872 + }, + { + "epoch": 0.89, + "learning_rate": 5.4141414141414144e-05, + "loss": 2.6946, + "theoretical_loss": 3.3104306797617187, + "tokens_seen": 3124625408 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1757280, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8810014724731445, + "objective/train/theoretical_loss": 3.3104253853776076, + "objective/train/tokens_used": 1483550176, + "theoretical_loss": 3.3104253853776076, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 5.412121212121212e-05, + "loss": 2.612, + "theoretical_loss": 3.3104253853776076, + "tokens_seen": 3124690944 + }, + { + "epoch": 0.89, + "learning_rate": 5.4101010101010105e-05, + "loss": 2.4223, + "theoretical_loss": 3.3104200911356285, + "tokens_seen": 3124756480 + }, + { + "epoch": 0.89, + "learning_rate": 5.408080808080808e-05, + "loss": 2.5273, + "theoretical_loss": 3.310414797035775, + "tokens_seen": 3124822016 + }, + { + "epoch": 0.89, + "learning_rate": 5.406060606060606e-05, + "loss": 2.6075, + "theoretical_loss": 3.31040950307804, + "tokens_seen": 3124887552 + }, + { + "epoch": 0.89, + "learning_rate": 5.4040404040404044e-05, + "loss": 2.3985, + "theoretical_loss": 3.3104042092624164, + "tokens_seen": 3124953088 + }, + { + "epoch": 0.89, + "learning_rate": 5.402020202020202e-05, + "loss": 2.482, + "theoretical_loss": 3.310398915588898, + "tokens_seen": 3125018624 + }, + { + "epoch": 0.89, + "learning_rate": 5.4e-05, + "loss": 2.3892, + "theoretical_loss": 3.310393622057479, + "tokens_seen": 3125084160 + }, + { + "epoch": 0.89, + "learning_rate": 5.397979797979798e-05, + "loss": 2.6802, + "theoretical_loss": 3.3103883286681497, + "tokens_seen": 3125149696 + }, + { + "epoch": 0.89, + "learning_rate": 5.395959595959596e-05, + "loss": 2.6839, + "theoretical_loss": 3.310383035420906, + "tokens_seen": 3125215232 + }, + { + "epoch": 0.89, + "learning_rate": 5.393939393939394e-05, + "loss": 2.4924, + "theoretical_loss": 3.31037774231574, + "tokens_seen": 3125280768 + }, + { + "epoch": 0.89, + "learning_rate": 5.391919191919192e-05, + "loss": 2.4246, + "theoretical_loss": 3.310372449352645, + "tokens_seen": 3125346304 + }, + { + "epoch": 0.89, + "learning_rate": 5.38989898989899e-05, + "loss": 2.6007, + "theoretical_loss": 3.3103671565316137, + "tokens_seen": 3125411840 + }, + { + "epoch": 0.89, + "learning_rate": 5.387878787878788e-05, + "loss": 2.4045, + "theoretical_loss": 3.3103618638526404, + "tokens_seen": 3125477376 + }, + { + "epoch": 0.89, + "learning_rate": 5.385858585858586e-05, + "loss": 2.702, + "theoretical_loss": 3.310356571315718, + "tokens_seen": 3125542912 + }, + { + "epoch": 0.89, + "learning_rate": 5.383838383838384e-05, + "loss": 2.594, + "theoretical_loss": 3.310351278920839, + "tokens_seen": 3125608448 + }, + { + "epoch": 0.89, + "learning_rate": 5.381818181818182e-05, + "loss": 2.5526, + "theoretical_loss": 3.310345986667997, + "tokens_seen": 3125673984 + }, + { + "epoch": 0.89, + "learning_rate": 5.37979797979798e-05, + "loss": 2.4521, + "theoretical_loss": 3.310340694557186, + "tokens_seen": 3125739520 + }, + { + "epoch": 0.89, + "learning_rate": 5.377777777777778e-05, + "loss": 2.3564, + "theoretical_loss": 3.3103354025883975, + "tokens_seen": 3125805056 + }, + { + "epoch": 0.89, + "learning_rate": 5.3757575757575765e-05, + "loss": 2.4804, + "theoretical_loss": 3.3103301107616265, + "tokens_seen": 3125870592 + }, + { + "epoch": 0.89, + "learning_rate": 5.3737373737373735e-05, + "loss": 2.5515, + "theoretical_loss": 3.310324819076865, + "tokens_seen": 3125936128 + }, + { + "epoch": 0.89, + "learning_rate": 5.371717171717172e-05, + "loss": 2.5686, + "theoretical_loss": 3.310319527534107, + "tokens_seen": 3126001664 + }, + { + "epoch": 0.89, + "learning_rate": 5.3696969696969696e-05, + "loss": 2.6274, + "theoretical_loss": 3.310314236133345, + "tokens_seen": 3126067200 + }, + { + "epoch": 0.89, + "learning_rate": 5.367676767676768e-05, + "loss": 2.4801, + "theoretical_loss": 3.310308944874573, + "tokens_seen": 3126132736 + }, + { + "epoch": 0.89, + "learning_rate": 5.365656565656566e-05, + "loss": 2.7971, + "theoretical_loss": 3.3103036537577832, + "tokens_seen": 3126198272 + }, + { + "epoch": 0.89, + "learning_rate": 5.3636363636363635e-05, + "loss": 2.5382, + "theoretical_loss": 3.31029836278297, + "tokens_seen": 3126263808 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1757671, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.212989568710327, + "objective/train/theoretical_loss": 3.3102930719501256, + "objective/train/tokens_used": 1485188576, + "theoretical_loss": 3.3102930719501256, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.89, + "learning_rate": 5.361616161616162e-05, + "loss": 2.3943, + "theoretical_loss": 3.3102930719501256, + "tokens_seen": 3126329344 + }, + { + "epoch": 0.89, + "learning_rate": 5.3595959595959596e-05, + "loss": 2.5951, + "theoretical_loss": 3.3102877812592437, + "tokens_seen": 3126394880 + }, + { + "epoch": 0.89, + "learning_rate": 5.357575757575757e-05, + "loss": 2.3691, + "theoretical_loss": 3.310282490710317, + "tokens_seen": 3126460416 + }, + { + "epoch": 0.89, + "learning_rate": 5.355555555555556e-05, + "loss": 2.6537, + "theoretical_loss": 3.31027720030334, + "tokens_seen": 3126525952 + }, + { + "epoch": 0.89, + "learning_rate": 5.3535353535353534e-05, + "loss": 2.6273, + "theoretical_loss": 3.3102719100383045, + "tokens_seen": 3126591488 + }, + { + "epoch": 0.89, + "learning_rate": 5.351515151515152e-05, + "loss": 2.399, + "theoretical_loss": 3.3102666199152044, + "tokens_seen": 3126657024 + }, + { + "epoch": 0.89, + "learning_rate": 5.34949494949495e-05, + "loss": 2.5414, + "theoretical_loss": 3.3102613299340327, + "tokens_seen": 3126722560 + }, + { + "epoch": 0.89, + "learning_rate": 5.347474747474747e-05, + "loss": 2.4912, + "theoretical_loss": 3.3102560400947834, + "tokens_seen": 3126788096 + }, + { + "epoch": 0.89, + "learning_rate": 5.3454545454545457e-05, + "loss": 2.5282, + "theoretical_loss": 3.310250750397448, + "tokens_seen": 3126853632 + }, + { + "epoch": 0.89, + "learning_rate": 5.3434343434343434e-05, + "loss": 2.6009, + "theoretical_loss": 3.3102454608420215, + "tokens_seen": 3126919168 + }, + { + "epoch": 0.89, + "learning_rate": 5.341414141414142e-05, + "loss": 2.5515, + "theoretical_loss": 3.3102401714284957, + "tokens_seen": 3126984704 + }, + { + "epoch": 0.89, + "learning_rate": 5.3393939393939395e-05, + "loss": 2.532, + "theoretical_loss": 3.3102348821568652, + "tokens_seen": 3127050240 + }, + { + "epoch": 0.89, + "learning_rate": 5.337373737373737e-05, + "loss": 2.6898, + "theoretical_loss": 3.310229593027122, + "tokens_seen": 3127115776 + }, + { + "epoch": 0.89, + "learning_rate": 5.3353535353535356e-05, + "loss": 2.5753, + "theoretical_loss": 3.3102243040392603, + "tokens_seen": 3127181312 + }, + { + "epoch": 0.89, + "learning_rate": 5.333333333333334e-05, + "loss": 2.5294, + "theoretical_loss": 3.3102190151932724, + "tokens_seen": 3127246848 + }, + { + "epoch": 0.89, + "learning_rate": 5.331313131313131e-05, + "loss": 2.3962, + "theoretical_loss": 3.3102137264891525, + "tokens_seen": 3127312384 + }, + { + "epoch": 0.89, + "learning_rate": 5.3292929292929294e-05, + "loss": 2.451, + "theoretical_loss": 3.310208437926893, + "tokens_seen": 3127377920 + }, + { + "epoch": 0.89, + "learning_rate": 5.327272727272727e-05, + "loss": 2.3094, + "theoretical_loss": 3.310203149506487, + "tokens_seen": 3127443456 + }, + { + "epoch": 0.89, + "learning_rate": 5.3252525252525255e-05, + "loss": 2.2559, + "theoretical_loss": 3.3101978612279286, + "tokens_seen": 3127508992 + }, + { + "epoch": 0.89, + "learning_rate": 5.323232323232324e-05, + "loss": 2.3499, + "theoretical_loss": 3.3101925730912107, + "tokens_seen": 3127574528 + }, + { + "epoch": 0.89, + "learning_rate": 5.321212121212121e-05, + "loss": 2.5237, + "theoretical_loss": 3.310187285096326, + "tokens_seen": 3127640064 + }, + { + "epoch": 0.89, + "learning_rate": 5.3191919191919194e-05, + "loss": 2.6232, + "theoretical_loss": 3.310181997243268, + "tokens_seen": 3127705600 + }, + { + "epoch": 0.89, + "learning_rate": 5.317171717171717e-05, + "loss": 2.5606, + "theoretical_loss": 3.3101767095320307, + "tokens_seen": 3127771136 + }, + { + "epoch": 0.89, + "learning_rate": 5.3151515151515155e-05, + "loss": 2.7337, + "theoretical_loss": 3.310171421962606, + "tokens_seen": 3127836672 + }, + { + "epoch": 0.89, + "learning_rate": 5.313131313131313e-05, + "loss": 2.5595, + "theoretical_loss": 3.310166134534988, + "tokens_seen": 3127902208 + }, + { + "epoch": 0.89, + "objective/train/docs_used": 1758353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4248931407928467, + "objective/train/theoretical_loss": 3.31016084724917, + "objective/train/tokens_used": 1486826976, + "theoretical_loss": 3.31016084724917, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 5.311111111111111e-05, + "loss": 2.3845, + "theoretical_loss": 3.31016084724917, + "tokens_seen": 3127967744 + }, + { + "epoch": 0.9, + "learning_rate": 5.309090909090909e-05, + "loss": 2.5826, + "theoretical_loss": 3.3101555601051444, + "tokens_seen": 3128033280 + }, + { + "epoch": 0.9, + "learning_rate": 5.307070707070708e-05, + "loss": 2.4199, + "theoretical_loss": 3.3101502731029053, + "tokens_seen": 3128098816 + }, + { + "epoch": 0.9, + "learning_rate": 5.305050505050505e-05, + "loss": 2.3236, + "theoretical_loss": 3.3101449862424457, + "tokens_seen": 3128164352 + }, + { + "epoch": 0.9, + "learning_rate": 5.303030303030303e-05, + "loss": 2.5169, + "theoretical_loss": 3.3101396995237584, + "tokens_seen": 3128229888 + }, + { + "epoch": 0.9, + "learning_rate": 5.301010101010101e-05, + "loss": 2.5982, + "theoretical_loss": 3.3101344129468373, + "tokens_seen": 3128295424 + }, + { + "epoch": 0.9, + "learning_rate": 5.298989898989899e-05, + "loss": 2.518, + "theoretical_loss": 3.3101291265116752, + "tokens_seen": 3128360960 + }, + { + "epoch": 0.9, + "learning_rate": 5.296969696969697e-05, + "loss": 2.52, + "theoretical_loss": 3.310123840218265, + "tokens_seen": 3128426496 + }, + { + "epoch": 0.9, + "learning_rate": 5.294949494949495e-05, + "loss": 2.5652, + "theoretical_loss": 3.3101185540666007, + "tokens_seen": 3128492032 + }, + { + "epoch": 0.9, + "learning_rate": 5.292929292929293e-05, + "loss": 2.9835, + "theoretical_loss": 3.3101132680566754, + "tokens_seen": 3128557568 + }, + { + "epoch": 0.9, + "learning_rate": 5.2909090909090915e-05, + "loss": 2.5257, + "theoretical_loss": 3.3101079821884816, + "tokens_seen": 3128623104 + }, + { + "epoch": 0.9, + "learning_rate": 5.288888888888889e-05, + "loss": 2.6543, + "theoretical_loss": 3.310102696462013, + "tokens_seen": 3128688640 + }, + { + "epoch": 0.9, + "learning_rate": 5.286868686868687e-05, + "loss": 2.503, + "theoretical_loss": 3.310097410877263, + "tokens_seen": 3128754176 + }, + { + "epoch": 0.9, + "learning_rate": 5.2848484848484847e-05, + "loss": 2.6652, + "theoretical_loss": 3.310092125434225, + "tokens_seen": 3128819712 + }, + { + "epoch": 0.9, + "learning_rate": 5.282828282828283e-05, + "loss": 2.5742, + "theoretical_loss": 3.310086840132892, + "tokens_seen": 3128885248 + }, + { + "epoch": 0.9, + "learning_rate": 5.2808080808080815e-05, + "loss": 2.599, + "theoretical_loss": 3.3100815549732565, + "tokens_seen": 3128950784 + }, + { + "epoch": 0.9, + "learning_rate": 5.2787878787878785e-05, + "loss": 2.5732, + "theoretical_loss": 3.310076269955313, + "tokens_seen": 3129016320 + }, + { + "epoch": 0.9, + "learning_rate": 5.276767676767677e-05, + "loss": 2.3672, + "theoretical_loss": 3.3100709850790535, + "tokens_seen": 3129081856 + }, + { + "epoch": 0.9, + "learning_rate": 5.2747474747474746e-05, + "loss": 2.5961, + "theoretical_loss": 3.3100657003444725, + "tokens_seen": 3129147392 + }, + { + "epoch": 0.9, + "learning_rate": 5.272727272727273e-05, + "loss": 2.5277, + "theoretical_loss": 3.310060415751562, + "tokens_seen": 3129212928 + }, + { + "epoch": 0.9, + "learning_rate": 5.270707070707071e-05, + "loss": 2.5984, + "theoretical_loss": 3.310055131300316, + "tokens_seen": 3129278464 + }, + { + "epoch": 0.9, + "learning_rate": 5.2686868686868684e-05, + "loss": 2.3937, + "theoretical_loss": 3.310049846990728, + "tokens_seen": 3129344000 + }, + { + "epoch": 0.9, + "learning_rate": 5.266666666666667e-05, + "loss": 2.4783, + "theoretical_loss": 3.3100445628227906, + "tokens_seen": 3129409536 + }, + { + "epoch": 0.9, + "learning_rate": 5.264646464646465e-05, + "loss": 2.2857, + "theoretical_loss": 3.310039278796497, + "tokens_seen": 3129475072 + }, + { + "epoch": 0.9, + "learning_rate": 5.262626262626263e-05, + "loss": 2.67, + "theoretical_loss": 3.3100339949118407, + "tokens_seen": 3129540608 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1759339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.837021827697754, + "objective/train/theoretical_loss": 3.3100287111688154, + "objective/train/tokens_used": 1488465376, + "theoretical_loss": 3.3100287111688154, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.9, + "learning_rate": 5.260606060606061e-05, + "loss": 2.683, + "theoretical_loss": 3.3100287111688154, + "tokens_seen": 3129606144 + }, + { + "epoch": 0.9, + "learning_rate": 5.2585858585858584e-05, + "loss": 2.4982, + "theoretical_loss": 3.310023427567413, + "tokens_seen": 3129671680 + }, + { + "epoch": 0.9, + "learning_rate": 5.256565656565657e-05, + "loss": 2.6151, + "theoretical_loss": 3.3100181441076284, + "tokens_seen": 3129737216 + }, + { + "epoch": 0.9, + "learning_rate": 5.254545454545455e-05, + "loss": 2.7082, + "theoretical_loss": 3.3100128607894534, + "tokens_seen": 3129802752 + }, + { + "epoch": 0.9, + "learning_rate": 5.252525252525252e-05, + "loss": 2.4482, + "theoretical_loss": 3.310007577612882, + "tokens_seen": 3129868288 + }, + { + "epoch": 0.9, + "learning_rate": 5.2505050505050506e-05, + "loss": 2.4778, + "theoretical_loss": 3.3100022945779077, + "tokens_seen": 3129933824 + }, + { + "epoch": 0.9, + "learning_rate": 5.248484848484848e-05, + "loss": 2.426, + "theoretical_loss": 3.309997011684523, + "tokens_seen": 3129999360 + }, + { + "epoch": 0.9, + "learning_rate": 5.246464646464647e-05, + "loss": 2.3938, + "theoretical_loss": 3.3099917289327214, + "tokens_seen": 3130064896 + }, + { + "epoch": 0.9, + "learning_rate": 5.2444444444444445e-05, + "loss": 2.6298, + "theoretical_loss": 3.3099864463224966, + "tokens_seen": 3130130432 + }, + { + "epoch": 0.9, + "learning_rate": 5.242424242424242e-05, + "loss": 2.5199, + "theoretical_loss": 3.3099811638538412, + "tokens_seen": 3130195968 + }, + { + "epoch": 0.9, + "learning_rate": 5.2404040404040406e-05, + "loss": 2.5593, + "theoretical_loss": 3.309975881526749, + "tokens_seen": 3130261504 + }, + { + "epoch": 0.9, + "learning_rate": 5.238383838383839e-05, + "loss": 2.343, + "theoretical_loss": 3.3099705993412125, + "tokens_seen": 3130327040 + }, + { + "epoch": 0.9, + "learning_rate": 5.236363636363636e-05, + "loss": 2.4225, + "theoretical_loss": 3.309965317297226, + "tokens_seen": 3130392576 + }, + { + "epoch": 0.9, + "learning_rate": 5.2343434343434344e-05, + "loss": 2.6867, + "theoretical_loss": 3.3099600353947816, + "tokens_seen": 3130458112 + }, + { + "epoch": 0.9, + "learning_rate": 5.232323232323232e-05, + "loss": 2.4903, + "theoretical_loss": 3.309954753633873, + "tokens_seen": 3130523648 + }, + { + "epoch": 0.9, + "learning_rate": 5.2303030303030305e-05, + "loss": 2.5112, + "theoretical_loss": 3.3099494720144937, + "tokens_seen": 3130589184 + }, + { + "epoch": 0.9, + "learning_rate": 5.228282828282829e-05, + "loss": 2.2532, + "theoretical_loss": 3.309944190536637, + "tokens_seen": 3130654720 + }, + { + "epoch": 0.9, + "learning_rate": 5.226262626262626e-05, + "loss": 2.5365, + "theoretical_loss": 3.3099389092002958, + "tokens_seen": 3130720256 + }, + { + "epoch": 0.9, + "learning_rate": 5.2242424242424243e-05, + "loss": 2.535, + "theoretical_loss": 3.309933628005463, + "tokens_seen": 3130785792 + }, + { + "epoch": 0.9, + "learning_rate": 5.222222222222223e-05, + "loss": 2.2058, + "theoretical_loss": 3.309928346952133, + "tokens_seen": 3130851328 + }, + { + "epoch": 0.9, + "learning_rate": 5.2202020202020205e-05, + "loss": 2.424, + "theoretical_loss": 3.309923066040298, + "tokens_seen": 3130916864 + }, + { + "epoch": 0.9, + "learning_rate": 5.218181818181818e-05, + "loss": 2.5875, + "theoretical_loss": 3.3099177852699517, + "tokens_seen": 3130982400 + }, + { + "epoch": 0.9, + "learning_rate": 5.216161616161616e-05, + "loss": 2.3284, + "theoretical_loss": 3.3099125046410873, + "tokens_seen": 3131047936 + }, + { + "epoch": 0.9, + "learning_rate": 5.214141414141414e-05, + "loss": 2.4388, + "theoretical_loss": 3.309907224153698, + "tokens_seen": 3131113472 + }, + { + "epoch": 0.9, + "learning_rate": 5.212121212121213e-05, + "loss": 2.589, + "theoretical_loss": 3.309901943807777, + "tokens_seen": 3131179008 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1760143, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8071322441101074, + "objective/train/theoretical_loss": 3.3098966636033174, + "objective/train/tokens_used": 1490103776, + "theoretical_loss": 3.3098966636033174, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 5.21010101010101e-05, + "loss": 2.4923, + "theoretical_loss": 3.3098966636033174, + "tokens_seen": 3131244544 + }, + { + "epoch": 0.9, + "learning_rate": 5.208080808080808e-05, + "loss": 2.5105, + "theoretical_loss": 3.3098913835403128, + "tokens_seen": 3131310080 + }, + { + "epoch": 0.9, + "learning_rate": 5.206060606060606e-05, + "loss": 2.4652, + "theoretical_loss": 3.309886103618757, + "tokens_seen": 3131375616 + }, + { + "epoch": 0.9, + "learning_rate": 5.204040404040404e-05, + "loss": 2.5718, + "theoretical_loss": 3.3098808238386415, + "tokens_seen": 3131441152 + }, + { + "epoch": 0.9, + "learning_rate": 5.2020202020202026e-05, + "loss": 2.3858, + "theoretical_loss": 3.309875544199961, + "tokens_seen": 3131506688 + }, + { + "epoch": 0.9, + "learning_rate": 5.2e-05, + "loss": 2.4182, + "theoretical_loss": 3.3098702647027083, + "tokens_seen": 3131572224 + }, + { + "epoch": 0.9, + "learning_rate": 5.197979797979798e-05, + "loss": 2.3768, + "theoretical_loss": 3.3098649853468767, + "tokens_seen": 3131637760 + }, + { + "epoch": 0.9, + "learning_rate": 5.1959595959595965e-05, + "loss": 2.6121, + "theoretical_loss": 3.3098597061324595, + "tokens_seen": 3131703296 + }, + { + "epoch": 0.9, + "learning_rate": 5.193939393939394e-05, + "loss": 2.4145, + "theoretical_loss": 3.3098544270594497, + "tokens_seen": 3131768832 + }, + { + "epoch": 0.9, + "learning_rate": 5.191919191919192e-05, + "loss": 2.2078, + "theoretical_loss": 3.3098491481278414, + "tokens_seen": 3131834368 + }, + { + "epoch": 0.9, + "learning_rate": 5.1898989898989896e-05, + "loss": 2.621, + "theoretical_loss": 3.3098438693376266, + "tokens_seen": 3131899904 + }, + { + "epoch": 0.9, + "learning_rate": 5.187878787878788e-05, + "loss": 2.2939, + "theoretical_loss": 3.3098385906887993, + "tokens_seen": 3131965440 + }, + { + "epoch": 0.9, + "learning_rate": 5.1858585858585864e-05, + "loss": 2.5508, + "theoretical_loss": 3.3098333121813526, + "tokens_seen": 3132030976 + }, + { + "epoch": 0.9, + "learning_rate": 5.1838383838383835e-05, + "loss": 2.4307, + "theoretical_loss": 3.30982803381528, + "tokens_seen": 3132096512 + }, + { + "epoch": 0.9, + "learning_rate": 5.181818181818182e-05, + "loss": 2.5311, + "theoretical_loss": 3.3098227555905746, + "tokens_seen": 3132162048 + }, + { + "epoch": 0.9, + "learning_rate": 5.17979797979798e-05, + "loss": 2.5723, + "theoretical_loss": 3.309817477507229, + "tokens_seen": 3132227584 + }, + { + "epoch": 0.9, + "learning_rate": 5.177777777777778e-05, + "loss": 2.4751, + "theoretical_loss": 3.3098121995652376, + "tokens_seen": 3132293120 + }, + { + "epoch": 0.9, + "learning_rate": 5.175757575757576e-05, + "loss": 2.6266, + "theoretical_loss": 3.3098069217645927, + "tokens_seen": 3132358656 + }, + { + "epoch": 0.9, + "learning_rate": 5.1737373737373734e-05, + "loss": 2.4988, + "theoretical_loss": 3.3098016441052884, + "tokens_seen": 3132424192 + }, + { + "epoch": 0.9, + "learning_rate": 5.171717171717172e-05, + "loss": 2.7835, + "theoretical_loss": 3.3097963665873174, + "tokens_seen": 3132489728 + }, + { + "epoch": 0.9, + "learning_rate": 5.16969696969697e-05, + "loss": 2.4713, + "theoretical_loss": 3.3097910892106728, + "tokens_seen": 3132555264 + }, + { + "epoch": 0.9, + "learning_rate": 5.167676767676768e-05, + "loss": 2.6729, + "theoretical_loss": 3.3097858119753485, + "tokens_seen": 3132620800 + }, + { + "epoch": 0.9, + "learning_rate": 5.1656565656565656e-05, + "loss": 2.5148, + "theoretical_loss": 3.309780534881337, + "tokens_seen": 3132686336 + }, + { + "epoch": 0.9, + "learning_rate": 5.1636363636363634e-05, + "loss": 2.4794, + "theoretical_loss": 3.3097752579286324, + "tokens_seen": 3132751872 + }, + { + "epoch": 0.9, + "learning_rate": 5.161616161616162e-05, + "loss": 2.5828, + "theoretical_loss": 3.309769981117227, + "tokens_seen": 3132817408 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1761098, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.540506362915039, + "objective/train/theoretical_loss": 3.3097647044471152, + "objective/train/tokens_used": 1491742176, + "theoretical_loss": 3.3097647044471152, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.9, + "learning_rate": 5.15959595959596e-05, + "loss": 2.4917, + "theoretical_loss": 3.3097647044471152, + "tokens_seen": 3132882944 + }, + { + "epoch": 0.9, + "learning_rate": 5.157575757575757e-05, + "loss": 2.4968, + "theoretical_loss": 3.309759427918289, + "tokens_seen": 3132948480 + }, + { + "epoch": 0.9, + "learning_rate": 5.1555555555555556e-05, + "loss": 2.4777, + "theoretical_loss": 3.309754151530743, + "tokens_seen": 3133014016 + }, + { + "epoch": 0.9, + "learning_rate": 5.153535353535354e-05, + "loss": 2.7054, + "theoretical_loss": 3.3097488752844693, + "tokens_seen": 3133079552 + }, + { + "epoch": 0.9, + "learning_rate": 5.151515151515152e-05, + "loss": 2.2868, + "theoretical_loss": 3.3097435991794617, + "tokens_seen": 3133145088 + }, + { + "epoch": 0.9, + "learning_rate": 5.1494949494949494e-05, + "loss": 2.4272, + "theoretical_loss": 3.309738323215713, + "tokens_seen": 3133210624 + }, + { + "epoch": 0.9, + "learning_rate": 5.147474747474747e-05, + "loss": 2.3407, + "theoretical_loss": 3.3097330473932174, + "tokens_seen": 3133276160 + }, + { + "epoch": 0.9, + "learning_rate": 5.1454545454545455e-05, + "loss": 2.4206, + "theoretical_loss": 3.3097277717119677, + "tokens_seen": 3133341696 + }, + { + "epoch": 0.9, + "learning_rate": 5.143434343434344e-05, + "loss": 2.8176, + "theoretical_loss": 3.3097224961719567, + "tokens_seen": 3133407232 + }, + { + "epoch": 0.9, + "learning_rate": 5.1414141414141416e-05, + "loss": 2.5572, + "theoretical_loss": 3.309717220773178, + "tokens_seen": 3133472768 + }, + { + "epoch": 0.9, + "learning_rate": 5.1393939393939394e-05, + "loss": 2.3091, + "theoretical_loss": 3.309711945515625, + "tokens_seen": 3133538304 + }, + { + "epoch": 0.9, + "learning_rate": 5.137373737373738e-05, + "loss": 2.5986, + "theoretical_loss": 3.309706670399291, + "tokens_seen": 3133603840 + }, + { + "epoch": 0.9, + "learning_rate": 5.1353535353535355e-05, + "loss": 2.5942, + "theoretical_loss": 3.3097013954241694, + "tokens_seen": 3133669376 + }, + { + "epoch": 0.9, + "learning_rate": 5.133333333333334e-05, + "loss": 2.7297, + "theoretical_loss": 3.3096961205902526, + "tokens_seen": 3133734912 + }, + { + "epoch": 0.9, + "learning_rate": 5.131313131313131e-05, + "loss": 2.5276, + "theoretical_loss": 3.309690845897535, + "tokens_seen": 3133800448 + }, + { + "epoch": 0.9, + "learning_rate": 5.129292929292929e-05, + "loss": 2.4054, + "theoretical_loss": 3.309685571346009, + "tokens_seen": 3133865984 + }, + { + "epoch": 0.9, + "learning_rate": 5.127272727272728e-05, + "loss": 2.6591, + "theoretical_loss": 3.3096802969356682, + "tokens_seen": 3133931520 + }, + { + "epoch": 0.9, + "learning_rate": 5.1252525252525254e-05, + "loss": 2.5723, + "theoretical_loss": 3.3096750226665064, + "tokens_seen": 3133997056 + }, + { + "epoch": 0.9, + "learning_rate": 5.123232323232323e-05, + "loss": 2.4893, + "theoretical_loss": 3.3096697485385156, + "tokens_seen": 3134062592 + }, + { + "epoch": 0.9, + "learning_rate": 5.121212121212121e-05, + "loss": 2.8196, + "theoretical_loss": 3.30966447455169, + "tokens_seen": 3134128128 + }, + { + "epoch": 0.9, + "learning_rate": 5.119191919191919e-05, + "loss": 2.7042, + "theoretical_loss": 3.3096592007060233, + "tokens_seen": 3134193664 + }, + { + "epoch": 0.9, + "learning_rate": 5.1171717171717177e-05, + "loss": 2.3783, + "theoretical_loss": 3.3096539270015075, + "tokens_seen": 3134259200 + }, + { + "epoch": 0.9, + "learning_rate": 5.115151515151515e-05, + "loss": 2.6735, + "theoretical_loss": 3.3096486534381366, + "tokens_seen": 3134324736 + }, + { + "epoch": 0.9, + "learning_rate": 5.113131313131313e-05, + "loss": 2.431, + "theoretical_loss": 3.3096433800159044, + "tokens_seen": 3134390272 + }, + { + "epoch": 0.9, + "learning_rate": 5.1111111111111115e-05, + "loss": 2.4301, + "theoretical_loss": 3.309638106734803, + "tokens_seen": 3134455808 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1761812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5387630462646484, + "objective/train/theoretical_loss": 3.309632833594826, + "objective/train/tokens_used": 1493380576, + "theoretical_loss": 3.309632833594826, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 5.109090909090909e-05, + "loss": 2.6305, + "theoretical_loss": 3.309632833594826, + "tokens_seen": 3134521344 + }, + { + "epoch": 0.9, + "learning_rate": 5.1070707070707076e-05, + "loss": 2.7768, + "theoretical_loss": 3.3096275605959677, + "tokens_seen": 3134586880 + }, + { + "epoch": 0.9, + "learning_rate": 5.1050505050505046e-05, + "loss": 2.5375, + "theoretical_loss": 3.30962228773822, + "tokens_seen": 3134652416 + }, + { + "epoch": 0.9, + "learning_rate": 5.103030303030303e-05, + "loss": 2.8198, + "theoretical_loss": 3.309617015021577, + "tokens_seen": 3134717952 + }, + { + "epoch": 0.9, + "learning_rate": 5.1010101010101014e-05, + "loss": 2.332, + "theoretical_loss": 3.3096117424460316, + "tokens_seen": 3134783488 + }, + { + "epoch": 0.9, + "learning_rate": 5.098989898989899e-05, + "loss": 2.4018, + "theoretical_loss": 3.3096064700115773, + "tokens_seen": 3134849024 + }, + { + "epoch": 0.9, + "learning_rate": 5.096969696969697e-05, + "loss": 2.5299, + "theoretical_loss": 3.3096011977182074, + "tokens_seen": 3134914560 + }, + { + "epoch": 0.9, + "learning_rate": 5.094949494949495e-05, + "loss": 2.386, + "theoretical_loss": 3.309595925565915, + "tokens_seen": 3134980096 + }, + { + "epoch": 0.9, + "learning_rate": 5.092929292929293e-05, + "loss": 2.649, + "theoretical_loss": 3.309590653554693, + "tokens_seen": 3135045632 + }, + { + "epoch": 0.9, + "learning_rate": 5.0909090909090914e-05, + "loss": 2.5545, + "theoretical_loss": 3.3095853816845358, + "tokens_seen": 3135111168 + }, + { + "epoch": 0.9, + "learning_rate": 5.0888888888888884e-05, + "loss": 2.3822, + "theoretical_loss": 3.309580109955436, + "tokens_seen": 3135176704 + }, + { + "epoch": 0.9, + "learning_rate": 5.086868686868687e-05, + "loss": 2.4336, + "theoretical_loss": 3.309574838367386, + "tokens_seen": 3135242240 + }, + { + "epoch": 0.9, + "learning_rate": 5.084848484848485e-05, + "loss": 2.5628, + "theoretical_loss": 3.3095695669203806, + "tokens_seen": 3135307776 + }, + { + "epoch": 0.9, + "learning_rate": 5.082828282828283e-05, + "loss": 2.5288, + "theoretical_loss": 3.3095642956144125, + "tokens_seen": 3135373312 + }, + { + "epoch": 0.9, + "learning_rate": 5.080808080808081e-05, + "loss": 2.2241, + "theoretical_loss": 3.3095590244494746, + "tokens_seen": 3135438848 + }, + { + "epoch": 0.9, + "learning_rate": 5.0787878787878784e-05, + "loss": 2.5234, + "theoretical_loss": 3.309553753425561, + "tokens_seen": 3135504384 + }, + { + "epoch": 0.9, + "learning_rate": 5.076767676767677e-05, + "loss": 2.657, + "theoretical_loss": 3.309548482542664, + "tokens_seen": 3135569920 + }, + { + "epoch": 0.9, + "learning_rate": 5.074747474747475e-05, + "loss": 2.3729, + "theoretical_loss": 3.3095432118007775, + "tokens_seen": 3135635456 + }, + { + "epoch": 0.9, + "learning_rate": 5.072727272727273e-05, + "loss": 2.3316, + "theoretical_loss": 3.309537941199894, + "tokens_seen": 3135700992 + }, + { + "epoch": 0.9, + "learning_rate": 5.0707070707070706e-05, + "loss": 2.4818, + "theoretical_loss": 3.3095326707400083, + "tokens_seen": 3135766528 + }, + { + "epoch": 0.9, + "learning_rate": 5.068686868686869e-05, + "loss": 2.6839, + "theoretical_loss": 3.3095274004211124, + "tokens_seen": 3135832064 + }, + { + "epoch": 0.9, + "learning_rate": 5.066666666666667e-05, + "loss": 2.4676, + "theoretical_loss": 3.3095221302432, + "tokens_seen": 3135897600 + }, + { + "epoch": 0.9, + "learning_rate": 5.064646464646465e-05, + "loss": 2.6744, + "theoretical_loss": 3.3095168602062643, + "tokens_seen": 3135963136 + }, + { + "epoch": 0.9, + "learning_rate": 5.062626262626262e-05, + "loss": 2.3766, + "theoretical_loss": 3.3095115903102985, + "tokens_seen": 3136028672 + }, + { + "epoch": 0.9, + "learning_rate": 5.0606060606060606e-05, + "loss": 2.5268, + "theoretical_loss": 3.3095063205552964, + "tokens_seen": 3136094208 + }, + { + "debugging/Self-BLEU-5": 0.47506560066232434, + "debugging/distinct-1-grams": 0.7038168750705822, + "debugging/distinct-2-grams": 0.9567806381187414, + "debugging/entropy-1-grams": 5.464426542413726, + "debugging/entropy-2-grams": 6.431994342092753, + "debugging/length": 564.0, + "debugging/num_segments": 8, + "epoch": 0.9, + "objective/train/docs_used": 1763087, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5173027515411377, + "objective/train/theoretical_loss": 3.309501050941251, + "objective/train/tokens_used": 1495018976, + "theoretical_loss": 3.309501050941251, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.9, + "learning_rate": 5.058585858585859e-05, + "loss": 2.4747, + "theoretical_loss": 3.309501050941251, + "tokens_seen": 3136159744 + }, + { + "epoch": 0.9, + "learning_rate": 5.056565656565657e-05, + "loss": 2.5765, + "theoretical_loss": 3.309495781468155, + "tokens_seen": 3136225280 + }, + { + "epoch": 0.9, + "learning_rate": 5.0545454545454544e-05, + "loss": 2.4516, + "theoretical_loss": 3.3094905121360023, + "tokens_seen": 3136290816 + }, + { + "epoch": 0.9, + "learning_rate": 5.052525252525253e-05, + "loss": 2.4783, + "theoretical_loss": 3.309485242944786, + "tokens_seen": 3136356352 + }, + { + "epoch": 0.9, + "learning_rate": 5.0505050505050505e-05, + "loss": 2.5288, + "theoretical_loss": 3.3094799738944998, + "tokens_seen": 3136421888 + }, + { + "epoch": 0.9, + "learning_rate": 5.048484848484849e-05, + "loss": 2.5684, + "theoretical_loss": 3.3094747049851363, + "tokens_seen": 3136487424 + }, + { + "epoch": 0.9, + "learning_rate": 5.0464646464646466e-05, + "loss": 2.3571, + "theoretical_loss": 3.309469436216689, + "tokens_seen": 3136552960 + }, + { + "epoch": 0.9, + "learning_rate": 5.044444444444444e-05, + "loss": 2.6986, + "theoretical_loss": 3.3094641675891516, + "tokens_seen": 3136618496 + }, + { + "epoch": 0.9, + "learning_rate": 5.042424242424243e-05, + "loss": 2.518, + "theoretical_loss": 3.309458899102517, + "tokens_seen": 3136684032 + }, + { + "epoch": 0.9, + "learning_rate": 5.0404040404040404e-05, + "loss": 2.4195, + "theoretical_loss": 3.309453630756779, + "tokens_seen": 3136749568 + }, + { + "epoch": 0.9, + "learning_rate": 5.038383838383839e-05, + "loss": 2.4025, + "theoretical_loss": 3.3094483625519295, + "tokens_seen": 3136815104 + }, + { + "epoch": 0.9, + "learning_rate": 5.036363636363636e-05, + "loss": 2.4335, + "theoretical_loss": 3.309443094487963, + "tokens_seen": 3136880640 + }, + { + "epoch": 0.9, + "learning_rate": 5.034343434343434e-05, + "loss": 2.1823, + "theoretical_loss": 3.3094378265648734, + "tokens_seen": 3136946176 + }, + { + "epoch": 0.9, + "learning_rate": 5.032323232323233e-05, + "loss": 2.4622, + "theoretical_loss": 3.3094325587826523, + "tokens_seen": 3137011712 + }, + { + "epoch": 0.9, + "learning_rate": 5.0303030303030304e-05, + "loss": 2.5634, + "theoretical_loss": 3.309427291141294, + "tokens_seen": 3137077248 + }, + { + "epoch": 0.9, + "learning_rate": 5.028282828282828e-05, + "loss": 2.6942, + "theoretical_loss": 3.3094220236407916, + "tokens_seen": 3137142784 + }, + { + "epoch": 0.9, + "learning_rate": 5.0262626262626265e-05, + "loss": 2.5032, + "theoretical_loss": 3.3094167562811387, + "tokens_seen": 3137208320 + }, + { + "epoch": 0.9, + "learning_rate": 5.024242424242424e-05, + "loss": 2.4469, + "theoretical_loss": 3.309411489062328, + "tokens_seen": 3137273856 + }, + { + "epoch": 0.9, + "learning_rate": 5.0222222222222226e-05, + "loss": 2.2323, + "theoretical_loss": 3.309406221984353, + "tokens_seen": 3137339392 + }, + { + "epoch": 0.9, + "learning_rate": 5.0202020202020203e-05, + "loss": 2.6245, + "theoretical_loss": 3.309400955047207, + "tokens_seen": 3137404928 + }, + { + "epoch": 0.9, + "learning_rate": 5.018181818181818e-05, + "loss": 2.6147, + "theoretical_loss": 3.309395688250884, + "tokens_seen": 3137470464 + }, + { + "epoch": 0.9, + "learning_rate": 5.0161616161616165e-05, + "loss": 2.6855, + "theoretical_loss": 3.309390421595376, + "tokens_seen": 3137536000 + }, + { + "epoch": 0.9, + "learning_rate": 5.014141414141414e-05, + "loss": 2.4369, + "theoretical_loss": 3.3093851550806774, + "tokens_seen": 3137601536 + }, + { + "epoch": 0.9, + "learning_rate": 5.0121212121212126e-05, + "loss": 2.537, + "theoretical_loss": 3.309379888706781, + "tokens_seen": 3137667072 + }, + { + "epoch": 0.9, + "learning_rate": 5.01010101010101e-05, + "loss": 2.4753, + "theoretical_loss": 3.3093746224736798, + "tokens_seen": 3137732608 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1763593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5707321166992188, + "objective/train/theoretical_loss": 3.3093693563813678, + "objective/train/tokens_used": 1496657376, + "theoretical_loss": 3.3093693563813678, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 5.008080808080808e-05, + "loss": 2.6654, + "theoretical_loss": 3.3093693563813678, + "tokens_seen": 3137798144 + }, + { + "epoch": 0.9, + "learning_rate": 5.0060606060606064e-05, + "loss": 2.5086, + "theoretical_loss": 3.309364090429838, + "tokens_seen": 3137863680 + }, + { + "epoch": 0.9, + "learning_rate": 5.004040404040404e-05, + "loss": 2.5628, + "theoretical_loss": 3.3093588246190833, + "tokens_seen": 3137929216 + }, + { + "epoch": 0.9, + "learning_rate": 5.002020202020202e-05, + "loss": 2.6114, + "theoretical_loss": 3.3093535589490974, + "tokens_seen": 3137994752 + }, + { + "epoch": 0.9, + "learning_rate": 5e-05, + "loss": 2.3974, + "theoretical_loss": 3.3093482934198737, + "tokens_seen": 3138060288 + }, + { + "epoch": 0.9, + "learning_rate": 4.997979797979798e-05, + "loss": 2.4562, + "theoretical_loss": 3.3093430280314053, + "tokens_seen": 3138125824 + }, + { + "epoch": 0.9, + "learning_rate": 4.9959595959595964e-05, + "loss": 2.6312, + "theoretical_loss": 3.3093377627836853, + "tokens_seen": 3138191360 + }, + { + "epoch": 0.9, + "learning_rate": 4.9939393939393934e-05, + "loss": 2.6899, + "theoretical_loss": 3.3093324976767073, + "tokens_seen": 3138256896 + }, + { + "epoch": 0.9, + "learning_rate": 4.991919191919192e-05, + "loss": 2.4273, + "theoretical_loss": 3.309327232710465, + "tokens_seen": 3138322432 + }, + { + "epoch": 0.9, + "learning_rate": 4.98989898989899e-05, + "loss": 2.6199, + "theoretical_loss": 3.3093219678849506, + "tokens_seen": 3138387968 + }, + { + "epoch": 0.9, + "learning_rate": 4.987878787878788e-05, + "loss": 2.4605, + "theoretical_loss": 3.3093167032001585, + "tokens_seen": 3138453504 + }, + { + "epoch": 0.9, + "learning_rate": 4.985858585858586e-05, + "loss": 2.3998, + "theoretical_loss": 3.3093114386560814, + "tokens_seen": 3138519040 + }, + { + "epoch": 0.9, + "learning_rate": 4.983838383838384e-05, + "loss": 2.602, + "theoretical_loss": 3.309306174252712, + "tokens_seen": 3138584576 + }, + { + "epoch": 0.9, + "learning_rate": 4.981818181818182e-05, + "loss": 2.5438, + "theoretical_loss": 3.3093009099900454, + "tokens_seen": 3138650112 + }, + { + "epoch": 0.9, + "learning_rate": 4.97979797979798e-05, + "loss": 2.5986, + "theoretical_loss": 3.3092956458680733, + "tokens_seen": 3138715648 + }, + { + "epoch": 0.9, + "learning_rate": 4.977777777777778e-05, + "loss": 2.6955, + "theoretical_loss": 3.3092903818867896, + "tokens_seen": 3138781184 + }, + { + "epoch": 0.9, + "learning_rate": 4.9757575757575756e-05, + "loss": 2.5049, + "theoretical_loss": 3.3092851180461875, + "tokens_seen": 3138846720 + }, + { + "epoch": 0.9, + "learning_rate": 4.973737373737374e-05, + "loss": 2.7211, + "theoretical_loss": 3.3092798543462605, + "tokens_seen": 3138912256 + }, + { + "epoch": 0.9, + "learning_rate": 4.971717171717172e-05, + "loss": 2.3001, + "theoretical_loss": 3.3092745907870014, + "tokens_seen": 3138977792 + }, + { + "epoch": 0.9, + "learning_rate": 4.96969696969697e-05, + "loss": 2.6498, + "theoretical_loss": 3.309269327368404, + "tokens_seen": 3139043328 + }, + { + "epoch": 0.9, + "learning_rate": 4.967676767676767e-05, + "loss": 2.3305, + "theoretical_loss": 3.3092640640904616, + "tokens_seen": 3139108864 + }, + { + "epoch": 0.9, + "learning_rate": 4.9656565656565655e-05, + "loss": 2.5936, + "theoretical_loss": 3.309258800953167, + "tokens_seen": 3139174400 + }, + { + "epoch": 0.9, + "learning_rate": 4.963636363636364e-05, + "loss": 2.5166, + "theoretical_loss": 3.309253537956514, + "tokens_seen": 3139239936 + }, + { + "epoch": 0.9, + "learning_rate": 4.9616161616161616e-05, + "loss": 2.3919, + "theoretical_loss": 3.309248275100496, + "tokens_seen": 3139305472 + }, + { + "epoch": 0.9, + "learning_rate": 4.95959595959596e-05, + "loss": 2.5065, + "theoretical_loss": 3.3092430123851058, + "tokens_seen": 3139371008 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1765019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8205020427703857, + "objective/train/theoretical_loss": 3.309237749810337, + "objective/train/tokens_used": 1498295776, + "theoretical_loss": 3.309237749810337, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.9, + "learning_rate": 4.957575757575758e-05, + "loss": 2.6063, + "theoretical_loss": 3.309237749810337, + "tokens_seen": 3139436544 + }, + { + "epoch": 0.9, + "learning_rate": 4.9555555555555555e-05, + "loss": 2.3706, + "theoretical_loss": 3.309232487376183, + "tokens_seen": 3139502080 + }, + { + "epoch": 0.9, + "learning_rate": 4.953535353535354e-05, + "loss": 2.5044, + "theoretical_loss": 3.309227225082637, + "tokens_seen": 3139567616 + }, + { + "epoch": 0.9, + "learning_rate": 4.9515151515151516e-05, + "loss": 2.4673, + "theoretical_loss": 3.3092219629296924, + "tokens_seen": 3139633152 + }, + { + "epoch": 0.9, + "learning_rate": 4.949494949494949e-05, + "loss": 2.5017, + "theoretical_loss": 3.309216700917342, + "tokens_seen": 3139698688 + }, + { + "epoch": 0.9, + "learning_rate": 4.947474747474748e-05, + "loss": 2.3993, + "theoretical_loss": 3.30921143904558, + "tokens_seen": 3139764224 + }, + { + "epoch": 0.9, + "learning_rate": 4.9454545454545454e-05, + "loss": 2.6281, + "theoretical_loss": 3.309206177314399, + "tokens_seen": 3139829760 + }, + { + "epoch": 0.9, + "learning_rate": 4.943434343434344e-05, + "loss": 2.6605, + "theoretical_loss": 3.3092009157237925, + "tokens_seen": 3139895296 + }, + { + "epoch": 0.9, + "learning_rate": 4.9414141414141415e-05, + "loss": 2.5411, + "theoretical_loss": 3.3091956542737537, + "tokens_seen": 3139960832 + }, + { + "epoch": 0.9, + "learning_rate": 4.939393939393939e-05, + "loss": 2.4349, + "theoretical_loss": 3.3091903929642763, + "tokens_seen": 3140026368 + }, + { + "epoch": 0.9, + "learning_rate": 4.9373737373737376e-05, + "loss": 2.4991, + "theoretical_loss": 3.309185131795353, + "tokens_seen": 3140091904 + }, + { + "epoch": 0.9, + "learning_rate": 4.9353535353535354e-05, + "loss": 2.4572, + "theoretical_loss": 3.309179870766978, + "tokens_seen": 3140157440 + }, + { + "epoch": 0.9, + "learning_rate": 4.933333333333334e-05, + "loss": 2.6817, + "theoretical_loss": 3.309174609879144, + "tokens_seen": 3140222976 + }, + { + "epoch": 0.9, + "learning_rate": 4.9313131313131315e-05, + "loss": 2.4166, + "theoretical_loss": 3.309169349131844, + "tokens_seen": 3140288512 + }, + { + "epoch": 0.9, + "learning_rate": 4.929292929292929e-05, + "loss": 2.6789, + "theoretical_loss": 3.309164088525072, + "tokens_seen": 3140354048 + }, + { + "epoch": 0.9, + "learning_rate": 4.9272727272727276e-05, + "loss": 2.3781, + "theoretical_loss": 3.309158828058821, + "tokens_seen": 3140419584 + }, + { + "epoch": 0.9, + "learning_rate": 4.925252525252525e-05, + "loss": 2.5349, + "theoretical_loss": 3.3091535677330848, + "tokens_seen": 3140485120 + }, + { + "epoch": 0.9, + "learning_rate": 4.923232323232323e-05, + "loss": 2.4611, + "theoretical_loss": 3.3091483075478556, + "tokens_seen": 3140550656 + }, + { + "epoch": 0.9, + "learning_rate": 4.9212121212121214e-05, + "loss": 2.5239, + "theoretical_loss": 3.3091430475031274, + "tokens_seen": 3140616192 + }, + { + "epoch": 0.9, + "learning_rate": 4.919191919191919e-05, + "loss": 2.6709, + "theoretical_loss": 3.309137787598894, + "tokens_seen": 3140681728 + }, + { + "epoch": 0.9, + "learning_rate": 4.9171717171717175e-05, + "loss": 2.2724, + "theoretical_loss": 3.3091325278351476, + "tokens_seen": 3140747264 + }, + { + "epoch": 0.9, + "learning_rate": 4.915151515151515e-05, + "loss": 2.5101, + "theoretical_loss": 3.3091272682118826, + "tokens_seen": 3140812800 + }, + { + "epoch": 0.9, + "learning_rate": 4.913131313131313e-05, + "loss": 2.6215, + "theoretical_loss": 3.3091220087290916, + "tokens_seen": 3140878336 + }, + { + "epoch": 0.9, + "learning_rate": 4.9111111111111114e-05, + "loss": 2.5611, + "theoretical_loss": 3.309116749386768, + "tokens_seen": 3140943872 + }, + { + "epoch": 0.9, + "learning_rate": 4.909090909090909e-05, + "loss": 2.6148, + "theoretical_loss": 3.3091114901849057, + "tokens_seen": 3141009408 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1765173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.625615119934082, + "objective/train/theoretical_loss": 3.309106231123497, + "objective/train/tokens_used": 1499934176, + "theoretical_loss": 3.309106231123497, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 4.907070707070707e-05, + "loss": 2.7061, + "theoretical_loss": 3.309106231123497, + "tokens_seen": 3141074944 + }, + { + "epoch": 0.9, + "learning_rate": 4.905050505050505e-05, + "loss": 2.4889, + "theoretical_loss": 3.3091009722025366, + "tokens_seen": 3141140480 + }, + { + "epoch": 0.9, + "learning_rate": 4.903030303030303e-05, + "loss": 2.6972, + "theoretical_loss": 3.3090957134220167, + "tokens_seen": 3141206016 + }, + { + "epoch": 0.9, + "learning_rate": 4.901010101010101e-05, + "loss": 2.5993, + "theoretical_loss": 3.3090904547819306, + "tokens_seen": 3141271552 + }, + { + "epoch": 0.9, + "learning_rate": 4.8989898989899e-05, + "loss": 2.4079, + "theoretical_loss": 3.3090851962822723, + "tokens_seen": 3141337088 + }, + { + "epoch": 0.9, + "learning_rate": 4.896969696969697e-05, + "loss": 2.6596, + "theoretical_loss": 3.309079937923035, + "tokens_seen": 3141402624 + }, + { + "epoch": 0.9, + "learning_rate": 4.894949494949495e-05, + "loss": 2.583, + "theoretical_loss": 3.309074679704211, + "tokens_seen": 3141468160 + }, + { + "epoch": 0.9, + "learning_rate": 4.892929292929293e-05, + "loss": 2.4139, + "theoretical_loss": 3.309069421625795, + "tokens_seen": 3141533696 + }, + { + "epoch": 0.9, + "learning_rate": 4.890909090909091e-05, + "loss": 2.4557, + "theoretical_loss": 3.30906416368778, + "tokens_seen": 3141599232 + }, + { + "epoch": 0.9, + "learning_rate": 4.888888888888889e-05, + "loss": 2.238, + "theoretical_loss": 3.3090589058901587, + "tokens_seen": 3141664768 + }, + { + "epoch": 0.9, + "learning_rate": 4.886868686868687e-05, + "loss": 2.445, + "theoretical_loss": 3.3090536482329247, + "tokens_seen": 3141730304 + }, + { + "epoch": 0.9, + "learning_rate": 4.884848484848485e-05, + "loss": 2.5833, + "theoretical_loss": 3.3090483907160717, + "tokens_seen": 3141795840 + }, + { + "epoch": 0.9, + "learning_rate": 4.882828282828283e-05, + "loss": 2.524, + "theoretical_loss": 3.3090431333395927, + "tokens_seen": 3141861376 + }, + { + "epoch": 0.9, + "learning_rate": 4.8808080808080805e-05, + "loss": 2.5632, + "theoretical_loss": 3.309037876103481, + "tokens_seen": 3141926912 + }, + { + "epoch": 0.9, + "learning_rate": 4.878787878787879e-05, + "loss": 2.4458, + "theoretical_loss": 3.30903261900773, + "tokens_seen": 3141992448 + }, + { + "epoch": 0.9, + "learning_rate": 4.8767676767676767e-05, + "loss": 2.5437, + "theoretical_loss": 3.309027362052333, + "tokens_seen": 3142057984 + }, + { + "epoch": 0.9, + "learning_rate": 4.874747474747475e-05, + "loss": 2.419, + "theoretical_loss": 3.3090221052372835, + "tokens_seen": 3142123520 + }, + { + "epoch": 0.9, + "learning_rate": 4.8727272727272734e-05, + "loss": 2.4865, + "theoretical_loss": 3.3090168485625746, + "tokens_seen": 3142189056 + }, + { + "epoch": 0.9, + "learning_rate": 4.8707070707070705e-05, + "loss": 2.4899, + "theoretical_loss": 3.3090115920281997, + "tokens_seen": 3142254592 + }, + { + "epoch": 0.9, + "learning_rate": 4.868686868686869e-05, + "loss": 2.7036, + "theoretical_loss": 3.3090063356341517, + "tokens_seen": 3142320128 + }, + { + "epoch": 0.9, + "learning_rate": 4.8666666666666666e-05, + "loss": 2.5279, + "theoretical_loss": 3.309001079380425, + "tokens_seen": 3142385664 + }, + { + "epoch": 0.9, + "learning_rate": 4.864646464646465e-05, + "loss": 2.5754, + "theoretical_loss": 3.3089958232670122, + "tokens_seen": 3142451200 + }, + { + "epoch": 0.9, + "learning_rate": 4.862626262626263e-05, + "loss": 2.7122, + "theoretical_loss": 3.308990567293906, + "tokens_seen": 3142516736 + }, + { + "epoch": 0.9, + "learning_rate": 4.8606060606060604e-05, + "loss": 2.8279, + "theoretical_loss": 3.308985311461101, + "tokens_seen": 3142582272 + }, + { + "epoch": 0.9, + "learning_rate": 4.858585858585859e-05, + "loss": 2.5458, + "theoretical_loss": 3.3089800557685902, + "tokens_seen": 3142647808 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1765556, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3280491828918457, + "objective/train/theoretical_loss": 3.308974800216366, + "objective/train/tokens_used": 1501572576, + "theoretical_loss": 3.308974800216366, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.9, + "learning_rate": 4.856565656565657e-05, + "loss": 2.3076, + "theoretical_loss": 3.308974800216366, + "tokens_seen": 3142713344 + }, + { + "epoch": 0.9, + "learning_rate": 4.854545454545454e-05, + "loss": 2.4976, + "theoretical_loss": 3.308969544804423, + "tokens_seen": 3142778880 + }, + { + "epoch": 0.9, + "learning_rate": 4.852525252525253e-05, + "loss": 2.818, + "theoretical_loss": 3.308964289532754, + "tokens_seen": 3142844416 + }, + { + "epoch": 0.9, + "learning_rate": 4.8505050505050504e-05, + "loss": 2.6887, + "theoretical_loss": 3.308959034401352, + "tokens_seen": 3142909952 + }, + { + "epoch": 0.9, + "learning_rate": 4.848484848484849e-05, + "loss": 2.7414, + "theoretical_loss": 3.308953779410211, + "tokens_seen": 3142975488 + }, + { + "epoch": 0.9, + "learning_rate": 4.8464646464646465e-05, + "loss": 2.5092, + "theoretical_loss": 3.308948524559323, + "tokens_seen": 3143041024 + }, + { + "epoch": 0.9, + "learning_rate": 4.844444444444444e-05, + "loss": 2.613, + "theoretical_loss": 3.308943269848683, + "tokens_seen": 3143106560 + }, + { + "epoch": 0.9, + "learning_rate": 4.8424242424242426e-05, + "loss": 2.6878, + "theoretical_loss": 3.308938015278284, + "tokens_seen": 3143172096 + }, + { + "epoch": 0.9, + "learning_rate": 4.84040404040404e-05, + "loss": 2.69, + "theoretical_loss": 3.3089327608481183, + "tokens_seen": 3143237632 + }, + { + "epoch": 0.9, + "learning_rate": 4.838383838383839e-05, + "loss": 2.4992, + "theoretical_loss": 3.30892750655818, + "tokens_seen": 3143303168 + }, + { + "epoch": 0.9, + "learning_rate": 4.8363636363636364e-05, + "loss": 2.3453, + "theoretical_loss": 3.3089222524084625, + "tokens_seen": 3143368704 + }, + { + "epoch": 0.9, + "learning_rate": 4.834343434343434e-05, + "loss": 2.6439, + "theoretical_loss": 3.3089169983989586, + "tokens_seen": 3143434240 + }, + { + "epoch": 0.9, + "learning_rate": 4.8323232323232326e-05, + "loss": 2.6112, + "theoretical_loss": 3.308911744529662, + "tokens_seen": 3143499776 + }, + { + "epoch": 0.9, + "learning_rate": 4.830303030303031e-05, + "loss": 2.7198, + "theoretical_loss": 3.3089064908005663, + "tokens_seen": 3143565312 + }, + { + "epoch": 0.9, + "learning_rate": 4.828282828282828e-05, + "loss": 2.4947, + "theoretical_loss": 3.3089012372116646, + "tokens_seen": 3143630848 + }, + { + "epoch": 0.9, + "learning_rate": 4.8262626262626264e-05, + "loss": 2.5727, + "theoretical_loss": 3.3088959837629504, + "tokens_seen": 3143696384 + }, + { + "epoch": 0.9, + "learning_rate": 4.824242424242424e-05, + "loss": 2.6279, + "theoretical_loss": 3.308890730454416, + "tokens_seen": 3143761920 + }, + { + "epoch": 0.9, + "learning_rate": 4.8222222222222225e-05, + "loss": 2.5761, + "theoretical_loss": 3.3088854772860565, + "tokens_seen": 3143827456 + }, + { + "epoch": 0.9, + "learning_rate": 4.82020202020202e-05, + "loss": 2.6086, + "theoretical_loss": 3.3088802242578637, + "tokens_seen": 3143892992 + }, + { + "epoch": 0.9, + "learning_rate": 4.818181818181818e-05, + "loss": 2.6696, + "theoretical_loss": 3.3088749713698316, + "tokens_seen": 3143958528 + }, + { + "epoch": 0.9, + "learning_rate": 4.8161616161616163e-05, + "loss": 2.5153, + "theoretical_loss": 3.3088697186219536, + "tokens_seen": 3144024064 + }, + { + "epoch": 0.9, + "learning_rate": 4.814141414141415e-05, + "loss": 2.5273, + "theoretical_loss": 3.308864466014223, + "tokens_seen": 3144089600 + }, + { + "epoch": 0.9, + "learning_rate": 4.8121212121212125e-05, + "loss": 2.5573, + "theoretical_loss": 3.3088592135466333, + "tokens_seen": 3144155136 + }, + { + "epoch": 0.9, + "learning_rate": 4.81010101010101e-05, + "loss": 2.4626, + "theoretical_loss": 3.308853961219177, + "tokens_seen": 3144220672 + }, + { + "epoch": 0.9, + "learning_rate": 4.808080808080808e-05, + "loss": 2.5408, + "theoretical_loss": 3.3088487090318486, + "tokens_seen": 3144286208 + }, + { + "epoch": 0.9, + "objective/train/docs_used": 1766057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4200446605682373, + "objective/train/theoretical_loss": 3.3088434569846403, + "objective/train/tokens_used": 1503210976, + "theoretical_loss": 3.3088434569846403, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 4.806060606060606e-05, + "loss": 2.4558, + "theoretical_loss": 3.3088434569846403, + "tokens_seen": 3144351744 + }, + { + "epoch": 0.91, + "learning_rate": 4.804040404040405e-05, + "loss": 2.5231, + "theoretical_loss": 3.3088382050775467, + "tokens_seen": 3144417280 + }, + { + "epoch": 0.91, + "learning_rate": 4.802020202020202e-05, + "loss": 2.6496, + "theoretical_loss": 3.30883295331056, + "tokens_seen": 3144482816 + }, + { + "epoch": 0.91, + "learning_rate": 4.8e-05, + "loss": 2.7934, + "theoretical_loss": 3.308827701683674, + "tokens_seen": 3144548352 + }, + { + "epoch": 0.91, + "learning_rate": 4.797979797979798e-05, + "loss": 2.6671, + "theoretical_loss": 3.308822450196882, + "tokens_seen": 3144613888 + }, + { + "epoch": 0.91, + "learning_rate": 4.795959595959596e-05, + "loss": 2.6634, + "theoretical_loss": 3.3088171988501776, + "tokens_seen": 3144679424 + }, + { + "epoch": 0.91, + "learning_rate": 4.793939393939394e-05, + "loss": 2.4922, + "theoretical_loss": 3.3088119476435542, + "tokens_seen": 3144744960 + }, + { + "epoch": 0.91, + "learning_rate": 4.791919191919192e-05, + "loss": 2.7125, + "theoretical_loss": 3.3088066965770047, + "tokens_seen": 3144810496 + }, + { + "epoch": 0.91, + "learning_rate": 4.78989898989899e-05, + "loss": 2.6459, + "theoretical_loss": 3.308801445650522, + "tokens_seen": 3144876032 + }, + { + "epoch": 0.91, + "learning_rate": 4.7878787878787885e-05, + "loss": 2.6681, + "theoretical_loss": 3.308796194864101, + "tokens_seen": 3144941568 + }, + { + "epoch": 0.91, + "learning_rate": 4.7858585858585855e-05, + "loss": 2.7363, + "theoretical_loss": 3.3087909442177335, + "tokens_seen": 3145007104 + }, + { + "epoch": 0.91, + "learning_rate": 4.783838383838384e-05, + "loss": 2.2816, + "theoretical_loss": 3.3087856937114135, + "tokens_seen": 3145072640 + }, + { + "epoch": 0.91, + "learning_rate": 4.7818181818181816e-05, + "loss": 2.4122, + "theoretical_loss": 3.3087804433451344, + "tokens_seen": 3145138176 + }, + { + "epoch": 0.91, + "learning_rate": 4.77979797979798e-05, + "loss": 2.7254, + "theoretical_loss": 3.3087751931188896, + "tokens_seen": 3145203712 + }, + { + "epoch": 0.91, + "learning_rate": 4.7777777777777784e-05, + "loss": 2.7794, + "theoretical_loss": 3.3087699430326722, + "tokens_seen": 3145269248 + }, + { + "epoch": 0.91, + "learning_rate": 4.7757575757575755e-05, + "loss": 2.4496, + "theoretical_loss": 3.308764693086476, + "tokens_seen": 3145334784 + }, + { + "epoch": 0.91, + "learning_rate": 4.773737373737374e-05, + "loss": 2.4188, + "theoretical_loss": 3.3087594432802936, + "tokens_seen": 3145400320 + }, + { + "epoch": 0.91, + "learning_rate": 4.771717171717172e-05, + "loss": 2.4482, + "theoretical_loss": 3.308754193614119, + "tokens_seen": 3145465856 + }, + { + "epoch": 0.91, + "learning_rate": 4.76969696969697e-05, + "loss": 2.7358, + "theoretical_loss": 3.3087489440879447, + "tokens_seen": 3145531392 + }, + { + "epoch": 0.91, + "learning_rate": 4.767676767676768e-05, + "loss": 2.6044, + "theoretical_loss": 3.308743694701765, + "tokens_seen": 3145596928 + }, + { + "epoch": 0.91, + "learning_rate": 4.7656565656565654e-05, + "loss": 2.498, + "theoretical_loss": 3.3087384454555733, + "tokens_seen": 3145662464 + }, + { + "epoch": 0.91, + "learning_rate": 4.763636363636364e-05, + "loss": 2.5975, + "theoretical_loss": 3.308733196349362, + "tokens_seen": 3145728000 + }, + { + "epoch": 0.91, + "learning_rate": 4.761616161616162e-05, + "loss": 2.6452, + "theoretical_loss": 3.308727947383125, + "tokens_seen": 3145793536 + }, + { + "epoch": 0.91, + "learning_rate": 4.759595959595959e-05, + "loss": 2.7637, + "theoretical_loss": 3.3087226985568563, + "tokens_seen": 3145859072 + }, + { + "epoch": 0.91, + "learning_rate": 4.7575757575757576e-05, + "loss": 2.4205, + "theoretical_loss": 3.308717449870548, + "tokens_seen": 3145924608 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1767374, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9139411449432373, + "objective/train/theoretical_loss": 3.3087122013241945, + "objective/train/tokens_used": 1504849376, + "theoretical_loss": 3.3087122013241945, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.91, + "learning_rate": 4.7555555555555553e-05, + "loss": 2.6603, + "theoretical_loss": 3.3087122013241945, + "tokens_seen": 3145990144 + }, + { + "epoch": 0.91, + "learning_rate": 4.753535353535354e-05, + "loss": 2.4674, + "theoretical_loss": 3.308706952917788, + "tokens_seen": 3146055680 + }, + { + "epoch": 0.91, + "learning_rate": 4.751515151515152e-05, + "loss": 2.4618, + "theoretical_loss": 3.308701704651323, + "tokens_seen": 3146121216 + }, + { + "epoch": 0.91, + "learning_rate": 4.749494949494949e-05, + "loss": 2.4116, + "theoretical_loss": 3.3086964565247925, + "tokens_seen": 3146186752 + }, + { + "epoch": 0.91, + "learning_rate": 4.7474747474747476e-05, + "loss": 2.524, + "theoretical_loss": 3.30869120853819, + "tokens_seen": 3146252288 + }, + { + "epoch": 0.91, + "learning_rate": 4.745454545454546e-05, + "loss": 2.3445, + "theoretical_loss": 3.308685960691508, + "tokens_seen": 3146317824 + }, + { + "epoch": 0.91, + "learning_rate": 4.743434343434344e-05, + "loss": 2.5175, + "theoretical_loss": 3.308680712984741, + "tokens_seen": 3146383360 + }, + { + "epoch": 0.91, + "learning_rate": 4.7414141414141414e-05, + "loss": 2.3519, + "theoretical_loss": 3.3086754654178816, + "tokens_seen": 3146448896 + }, + { + "epoch": 0.91, + "learning_rate": 4.739393939393939e-05, + "loss": 2.7261, + "theoretical_loss": 3.3086702179909233, + "tokens_seen": 3146514432 + }, + { + "epoch": 0.91, + "learning_rate": 4.7373737373737375e-05, + "loss": 2.5403, + "theoretical_loss": 3.30866497070386, + "tokens_seen": 3146579968 + }, + { + "epoch": 0.91, + "learning_rate": 4.735353535353536e-05, + "loss": 2.5392, + "theoretical_loss": 3.308659723556684, + "tokens_seen": 3146645504 + }, + { + "epoch": 0.91, + "learning_rate": 4.733333333333333e-05, + "loss": 2.422, + "theoretical_loss": 3.3086544765493895, + "tokens_seen": 3146711040 + }, + { + "epoch": 0.91, + "learning_rate": 4.7313131313131314e-05, + "loss": 2.5951, + "theoretical_loss": 3.3086492296819694, + "tokens_seen": 3146776576 + }, + { + "epoch": 0.91, + "learning_rate": 4.72929292929293e-05, + "loss": 2.6374, + "theoretical_loss": 3.3086439829544174, + "tokens_seen": 3146842112 + }, + { + "epoch": 0.91, + "learning_rate": 4.7272727272727275e-05, + "loss": 2.4268, + "theoretical_loss": 3.308638736366727, + "tokens_seen": 3146907648 + }, + { + "epoch": 0.91, + "learning_rate": 4.725252525252525e-05, + "loss": 2.6767, + "theoretical_loss": 3.308633489918891, + "tokens_seen": 3146973184 + }, + { + "epoch": 0.91, + "learning_rate": 4.723232323232323e-05, + "loss": 2.5242, + "theoretical_loss": 3.3086282436109036, + "tokens_seen": 3147038720 + }, + { + "epoch": 0.91, + "learning_rate": 4.721212121212121e-05, + "loss": 2.4168, + "theoretical_loss": 3.308622997442757, + "tokens_seen": 3147104256 + }, + { + "epoch": 0.91, + "learning_rate": 4.71919191919192e-05, + "loss": 2.6212, + "theoretical_loss": 3.3086177514144453, + "tokens_seen": 3147169792 + }, + { + "epoch": 0.91, + "learning_rate": 4.7171717171717174e-05, + "loss": 2.6072, + "theoretical_loss": 3.308612505525962, + "tokens_seen": 3147235328 + }, + { + "epoch": 0.91, + "learning_rate": 4.715151515151515e-05, + "loss": 2.5223, + "theoretical_loss": 3.3086072597773, + "tokens_seen": 3147300864 + }, + { + "epoch": 0.91, + "learning_rate": 4.713131313131313e-05, + "loss": 2.6328, + "theoretical_loss": 3.3086020141684527, + "tokens_seen": 3147366400 + }, + { + "epoch": 0.91, + "learning_rate": 4.711111111111111e-05, + "loss": 2.6565, + "theoretical_loss": 3.308596768699414, + "tokens_seen": 3147431936 + }, + { + "epoch": 0.91, + "learning_rate": 4.7090909090909097e-05, + "loss": 2.5527, + "theoretical_loss": 3.308591523370177, + "tokens_seen": 3147497472 + }, + { + "epoch": 0.91, + "learning_rate": 4.707070707070707e-05, + "loss": 2.7643, + "theoretical_loss": 3.3085862781807345, + "tokens_seen": 3147563008 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1768009, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0143842697143555, + "objective/train/theoretical_loss": 3.3085810331310803, + "objective/train/tokens_used": 1506487776, + "theoretical_loss": 3.3085810331310803, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 4.705050505050505e-05, + "loss": 2.7059, + "theoretical_loss": 3.3085810331310803, + "tokens_seen": 3147628544 + }, + { + "epoch": 0.91, + "learning_rate": 4.7030303030303035e-05, + "loss": 2.67, + "theoretical_loss": 3.3085757882212077, + "tokens_seen": 3147694080 + }, + { + "epoch": 0.91, + "learning_rate": 4.701010101010101e-05, + "loss": 2.6025, + "theoretical_loss": 3.3085705434511103, + "tokens_seen": 3147759616 + }, + { + "epoch": 0.91, + "learning_rate": 4.698989898989899e-05, + "loss": 2.3877, + "theoretical_loss": 3.3085652988207817, + "tokens_seen": 3147825152 + }, + { + "epoch": 0.91, + "learning_rate": 4.6969696969696966e-05, + "loss": 2.7667, + "theoretical_loss": 3.3085600543302145, + "tokens_seen": 3147890688 + }, + { + "epoch": 0.91, + "learning_rate": 4.694949494949495e-05, + "loss": 2.7307, + "theoretical_loss": 3.3085548099794027, + "tokens_seen": 3147956224 + }, + { + "epoch": 0.91, + "learning_rate": 4.6929292929292934e-05, + "loss": 2.453, + "theoretical_loss": 3.308549565768339, + "tokens_seen": 3148021760 + }, + { + "epoch": 0.91, + "learning_rate": 4.690909090909091e-05, + "loss": 2.5223, + "theoretical_loss": 3.308544321697017, + "tokens_seen": 3148087296 + }, + { + "epoch": 0.91, + "learning_rate": 4.688888888888889e-05, + "loss": 2.5382, + "theoretical_loss": 3.308539077765431, + "tokens_seen": 3148152832 + }, + { + "epoch": 0.91, + "learning_rate": 4.686868686868687e-05, + "loss": 2.6879, + "theoretical_loss": 3.308533833973573, + "tokens_seen": 3148218368 + }, + { + "epoch": 0.91, + "learning_rate": 4.684848484848485e-05, + "loss": 2.5579, + "theoretical_loss": 3.308528590321437, + "tokens_seen": 3148283904 + }, + { + "epoch": 0.91, + "learning_rate": 4.6828282828282834e-05, + "loss": 2.6765, + "theoretical_loss": 3.3085233468090167, + "tokens_seen": 3148349440 + }, + { + "epoch": 0.91, + "learning_rate": 4.6808080808080804e-05, + "loss": 2.2701, + "theoretical_loss": 3.308518103436305, + "tokens_seen": 3148414976 + }, + { + "epoch": 0.91, + "learning_rate": 4.678787878787879e-05, + "loss": 2.7534, + "theoretical_loss": 3.308512860203295, + "tokens_seen": 3148480512 + }, + { + "epoch": 0.91, + "learning_rate": 4.676767676767677e-05, + "loss": 2.7276, + "theoretical_loss": 3.308507617109981, + "tokens_seen": 3148546048 + }, + { + "epoch": 0.91, + "learning_rate": 4.674747474747475e-05, + "loss": 2.5695, + "theoretical_loss": 3.3085023741563555, + "tokens_seen": 3148611584 + }, + { + "epoch": 0.91, + "learning_rate": 4.6727272727272727e-05, + "loss": 2.4051, + "theoretical_loss": 3.308497131342412, + "tokens_seen": 3148677120 + }, + { + "epoch": 0.91, + "learning_rate": 4.6707070707070704e-05, + "loss": 2.3617, + "theoretical_loss": 3.3084918886681445, + "tokens_seen": 3148742656 + }, + { + "epoch": 0.91, + "learning_rate": 4.668686868686869e-05, + "loss": 2.7053, + "theoretical_loss": 3.308486646133546, + "tokens_seen": 3148808192 + }, + { + "epoch": 0.91, + "learning_rate": 4.666666666666667e-05, + "loss": 2.6545, + "theoretical_loss": 3.308481403738609, + "tokens_seen": 3148873728 + }, + { + "epoch": 0.91, + "learning_rate": 4.664646464646464e-05, + "loss": 2.6065, + "theoretical_loss": 3.3084761614833287, + "tokens_seen": 3148939264 + }, + { + "epoch": 0.91, + "learning_rate": 4.6626262626262626e-05, + "loss": 2.4755, + "theoretical_loss": 3.308470919367697, + "tokens_seen": 3149004800 + }, + { + "epoch": 0.91, + "learning_rate": 4.660606060606061e-05, + "loss": 2.3762, + "theoretical_loss": 3.3084656773917076, + "tokens_seen": 3149070336 + }, + { + "epoch": 0.91, + "learning_rate": 4.658585858585859e-05, + "loss": 2.585, + "theoretical_loss": 3.308460435555354, + "tokens_seen": 3149135872 + }, + { + "epoch": 0.91, + "learning_rate": 4.656565656565657e-05, + "loss": 2.5481, + "theoretical_loss": 3.30845519385863, + "tokens_seen": 3149201408 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1769388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6708626747131348, + "objective/train/theoretical_loss": 3.3084499523015283, + "objective/train/tokens_used": 1508126176, + "theoretical_loss": 3.3084499523015283, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.91, + "learning_rate": 4.654545454545454e-05, + "loss": 2.6618, + "theoretical_loss": 3.3084499523015283, + "tokens_seen": 3149266944 + }, + { + "epoch": 0.91, + "learning_rate": 4.6525252525252525e-05, + "loss": 2.6737, + "theoretical_loss": 3.3084447108840425, + "tokens_seen": 3149332480 + }, + { + "epoch": 0.91, + "learning_rate": 4.650505050505051e-05, + "loss": 2.3358, + "theoretical_loss": 3.308439469606166, + "tokens_seen": 3149398016 + }, + { + "epoch": 0.91, + "learning_rate": 4.6484848484848487e-05, + "loss": 2.3705, + "theoretical_loss": 3.3084342284678923, + "tokens_seen": 3149463552 + }, + { + "epoch": 0.91, + "learning_rate": 4.6464646464646464e-05, + "loss": 2.5509, + "theoretical_loss": 3.308428987469214, + "tokens_seen": 3149529088 + }, + { + "epoch": 0.91, + "learning_rate": 4.644444444444444e-05, + "loss": 2.7736, + "theoretical_loss": 3.308423746610126, + "tokens_seen": 3149594624 + }, + { + "epoch": 0.91, + "learning_rate": 4.6424242424242425e-05, + "loss": 2.7814, + "theoretical_loss": 3.3084185058906206, + "tokens_seen": 3149660160 + }, + { + "epoch": 0.91, + "learning_rate": 4.640404040404041e-05, + "loss": 2.5492, + "theoretical_loss": 3.308413265310691, + "tokens_seen": 3149725696 + }, + { + "epoch": 0.91, + "learning_rate": 4.638383838383838e-05, + "loss": 2.4167, + "theoretical_loss": 3.3084080248703316, + "tokens_seen": 3149791232 + }, + { + "epoch": 0.91, + "learning_rate": 4.636363636363636e-05, + "loss": 2.6129, + "theoretical_loss": 3.3084027845695343, + "tokens_seen": 3149856768 + }, + { + "epoch": 0.91, + "learning_rate": 4.634343434343435e-05, + "loss": 2.5188, + "theoretical_loss": 3.308397544408294, + "tokens_seen": 3149922304 + }, + { + "epoch": 0.91, + "learning_rate": 4.6323232323232324e-05, + "loss": 2.6851, + "theoretical_loss": 3.308392304386603, + "tokens_seen": 3149987840 + }, + { + "epoch": 0.91, + "learning_rate": 4.630303030303031e-05, + "loss": 2.499, + "theoretical_loss": 3.3083870645044553, + "tokens_seen": 3150053376 + }, + { + "epoch": 0.91, + "learning_rate": 4.628282828282828e-05, + "loss": 2.4559, + "theoretical_loss": 3.3083818247618444, + "tokens_seen": 3150118912 + }, + { + "epoch": 0.91, + "learning_rate": 4.626262626262626e-05, + "loss": 2.4549, + "theoretical_loss": 3.3083765851587628, + "tokens_seen": 3150184448 + }, + { + "epoch": 0.91, + "learning_rate": 4.624242424242425e-05, + "loss": 2.7422, + "theoretical_loss": 3.3083713456952046, + "tokens_seen": 3150249984 + }, + { + "epoch": 0.91, + "learning_rate": 4.6222222222222224e-05, + "loss": 2.4911, + "theoretical_loss": 3.308366106371163, + "tokens_seen": 3150315520 + }, + { + "epoch": 0.91, + "learning_rate": 4.62020202020202e-05, + "loss": 2.7277, + "theoretical_loss": 3.3083608671866314, + "tokens_seen": 3150381056 + }, + { + "epoch": 0.91, + "learning_rate": 4.6181818181818185e-05, + "loss": 2.4835, + "theoretical_loss": 3.3083556281416033, + "tokens_seen": 3150446592 + }, + { + "epoch": 0.91, + "learning_rate": 4.616161616161616e-05, + "loss": 2.45, + "theoretical_loss": 3.308350389236072, + "tokens_seen": 3150512128 + }, + { + "epoch": 0.91, + "learning_rate": 4.6141414141414146e-05, + "loss": 2.5858, + "theoretical_loss": 3.3083451504700307, + "tokens_seen": 3150577664 + }, + { + "epoch": 0.91, + "learning_rate": 4.6121212121212117e-05, + "loss": 2.707, + "theoretical_loss": 3.308339911843473, + "tokens_seen": 3150643200 + }, + { + "epoch": 0.91, + "learning_rate": 4.61010101010101e-05, + "loss": 2.4469, + "theoretical_loss": 3.3083346733563923, + "tokens_seen": 3150708736 + }, + { + "epoch": 0.91, + "learning_rate": 4.6080808080808085e-05, + "loss": 2.5654, + "theoretical_loss": 3.3083294350087815, + "tokens_seen": 3150774272 + }, + { + "epoch": 0.91, + "learning_rate": 4.606060606060606e-05, + "loss": 2.6019, + "theoretical_loss": 3.3083241968006347, + "tokens_seen": 3150839808 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1770077, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4027698040008545, + "objective/train/theoretical_loss": 3.3083189587319453, + "objective/train/tokens_used": 1509764576, + "theoretical_loss": 3.3083189587319453, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 4.604040404040404e-05, + "loss": 2.4345, + "theoretical_loss": 3.3083189587319453, + "tokens_seen": 3150905344 + }, + { + "epoch": 0.91, + "learning_rate": 4.6020202020202016e-05, + "loss": 2.5324, + "theoretical_loss": 3.3083137208027056, + "tokens_seen": 3150970880 + }, + { + "epoch": 0.91, + "learning_rate": 4.6e-05, + "loss": 2.4123, + "theoretical_loss": 3.3083084830129104, + "tokens_seen": 3151036416 + }, + { + "epoch": 0.91, + "learning_rate": 4.5979797979797984e-05, + "loss": 2.5215, + "theoretical_loss": 3.308303245362552, + "tokens_seen": 3151101952 + }, + { + "epoch": 0.91, + "learning_rate": 4.595959595959596e-05, + "loss": 2.5834, + "theoretical_loss": 3.3082980078516244, + "tokens_seen": 3151167488 + }, + { + "epoch": 0.91, + "learning_rate": 4.593939393939394e-05, + "loss": 2.4716, + "theoretical_loss": 3.308292770480121, + "tokens_seen": 3151233024 + }, + { + "epoch": 0.91, + "learning_rate": 4.591919191919192e-05, + "loss": 2.3598, + "theoretical_loss": 3.3082875332480346, + "tokens_seen": 3151298560 + }, + { + "epoch": 0.91, + "learning_rate": 4.58989898989899e-05, + "loss": 2.4249, + "theoretical_loss": 3.308282296155359, + "tokens_seen": 3151364096 + }, + { + "epoch": 0.91, + "learning_rate": 4.5878787878787883e-05, + "loss": 2.4569, + "theoretical_loss": 3.308277059202088, + "tokens_seen": 3151429632 + }, + { + "epoch": 0.91, + "learning_rate": 4.5858585858585854e-05, + "loss": 2.7465, + "theoretical_loss": 3.3082718223882144, + "tokens_seen": 3151495168 + }, + { + "epoch": 0.91, + "learning_rate": 4.583838383838384e-05, + "loss": 2.626, + "theoretical_loss": 3.308266585713732, + "tokens_seen": 3151560704 + }, + { + "epoch": 0.91, + "learning_rate": 4.581818181818182e-05, + "loss": 2.5615, + "theoretical_loss": 3.3082613491786335, + "tokens_seen": 3151626240 + }, + { + "epoch": 0.91, + "learning_rate": 4.57979797979798e-05, + "loss": 2.526, + "theoretical_loss": 3.308256112782913, + "tokens_seen": 3151691776 + }, + { + "epoch": 0.91, + "learning_rate": 4.5777777777777776e-05, + "loss": 2.5896, + "theoretical_loss": 3.3082508765265635, + "tokens_seen": 3151757312 + }, + { + "epoch": 0.91, + "learning_rate": 4.575757575757576e-05, + "loss": 2.6293, + "theoretical_loss": 3.3082456404095786, + "tokens_seen": 3151822848 + }, + { + "epoch": 0.91, + "learning_rate": 4.573737373737374e-05, + "loss": 2.4846, + "theoretical_loss": 3.3082404044319516, + "tokens_seen": 3151888384 + }, + { + "epoch": 0.91, + "learning_rate": 4.571717171717172e-05, + "loss": 2.7255, + "theoretical_loss": 3.3082351685936757, + "tokens_seen": 3151953920 + }, + { + "epoch": 0.91, + "learning_rate": 4.56969696969697e-05, + "loss": 2.3417, + "theoretical_loss": 3.308229932894745, + "tokens_seen": 3152019456 + }, + { + "epoch": 0.91, + "learning_rate": 4.5676767676767676e-05, + "loss": 2.6772, + "theoretical_loss": 3.308224697335152, + "tokens_seen": 3152084992 + }, + { + "epoch": 0.91, + "learning_rate": 4.565656565656566e-05, + "loss": 2.5872, + "theoretical_loss": 3.3082194619148906, + "tokens_seen": 3152150528 + }, + { + "epoch": 0.91, + "learning_rate": 4.563636363636364e-05, + "loss": 2.5891, + "theoretical_loss": 3.3082142266339543, + "tokens_seen": 3152216064 + }, + { + "epoch": 0.91, + "learning_rate": 4.561616161616162e-05, + "loss": 2.6979, + "theoretical_loss": 3.308208991492336, + "tokens_seen": 3152281600 + }, + { + "epoch": 0.91, + "learning_rate": 4.559595959595959e-05, + "loss": 2.6744, + "theoretical_loss": 3.3082037564900295, + "tokens_seen": 3152347136 + }, + { + "epoch": 0.91, + "learning_rate": 4.5575757575757575e-05, + "loss": 2.366, + "theoretical_loss": 3.3081985216270278, + "tokens_seen": 3152412672 + }, + { + "epoch": 0.91, + "learning_rate": 4.555555555555556e-05, + "loss": 2.3918, + "theoretical_loss": 3.3081932869033253, + "tokens_seen": 3152478208 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1771514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9596893787384033, + "objective/train/theoretical_loss": 3.308188052318914, + "objective/train/tokens_used": 1511402976, + "theoretical_loss": 3.308188052318914, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.91, + "learning_rate": 4.5535353535353536e-05, + "loss": 2.6061, + "theoretical_loss": 3.308188052318914, + "tokens_seen": 3152543744 + }, + { + "epoch": 0.91, + "learning_rate": 4.5515151515151513e-05, + "loss": 2.574, + "theoretical_loss": 3.308182817873788, + "tokens_seen": 3152609280 + }, + { + "epoch": 0.91, + "learning_rate": 4.54949494949495e-05, + "loss": 2.479, + "theoretical_loss": 3.3081775835679412, + "tokens_seen": 3152674816 + }, + { + "epoch": 0.91, + "learning_rate": 4.5474747474747475e-05, + "loss": 2.5261, + "theoretical_loss": 3.3081723494013664, + "tokens_seen": 3152740352 + }, + { + "epoch": 0.91, + "learning_rate": 4.545454545454546e-05, + "loss": 2.4386, + "theoretical_loss": 3.3081671153740566, + "tokens_seen": 3152805888 + }, + { + "epoch": 0.91, + "learning_rate": 4.543434343434343e-05, + "loss": 2.4497, + "theoretical_loss": 3.308161881486006, + "tokens_seen": 3152871424 + }, + { + "epoch": 0.91, + "learning_rate": 4.541414141414141e-05, + "loss": 2.4606, + "theoretical_loss": 3.3081566477372073, + "tokens_seen": 3152936960 + }, + { + "epoch": 0.91, + "learning_rate": 4.53939393939394e-05, + "loss": 2.6794, + "theoretical_loss": 3.308151414127655, + "tokens_seen": 3153002496 + }, + { + "epoch": 0.91, + "learning_rate": 4.5373737373737374e-05, + "loss": 2.6021, + "theoretical_loss": 3.3081461806573413, + "tokens_seen": 3153068032 + }, + { + "epoch": 0.91, + "learning_rate": 4.535353535353536e-05, + "loss": 2.6016, + "theoretical_loss": 3.30814094732626, + "tokens_seen": 3153133568 + }, + { + "epoch": 0.91, + "learning_rate": 4.5333333333333335e-05, + "loss": 2.4568, + "theoretical_loss": 3.3081357141344045, + "tokens_seen": 3153199104 + }, + { + "epoch": 0.91, + "learning_rate": 4.531313131313131e-05, + "loss": 2.538, + "theoretical_loss": 3.3081304810817684, + "tokens_seen": 3153264640 + }, + { + "epoch": 0.91, + "learning_rate": 4.5292929292929296e-05, + "loss": 2.636, + "theoretical_loss": 3.3081252481683454, + "tokens_seen": 3153330176 + }, + { + "epoch": 0.91, + "learning_rate": 4.5272727272727274e-05, + "loss": 2.5851, + "theoretical_loss": 3.308120015394128, + "tokens_seen": 3153395712 + }, + { + "epoch": 0.91, + "learning_rate": 4.525252525252525e-05, + "loss": 2.4794, + "theoretical_loss": 3.30811478275911, + "tokens_seen": 3153461248 + }, + { + "epoch": 0.91, + "learning_rate": 4.5232323232323235e-05, + "loss": 2.2398, + "theoretical_loss": 3.3081095502632856, + "tokens_seen": 3153526784 + }, + { + "epoch": 0.91, + "learning_rate": 4.521212121212121e-05, + "loss": 2.4043, + "theoretical_loss": 3.308104317906647, + "tokens_seen": 3153592320 + }, + { + "epoch": 0.91, + "learning_rate": 4.5191919191919196e-05, + "loss": 2.6458, + "theoretical_loss": 3.3080990856891876, + "tokens_seen": 3153657856 + }, + { + "epoch": 0.91, + "learning_rate": 4.5171717171717166e-05, + "loss": 2.6857, + "theoretical_loss": 3.3080938536109024, + "tokens_seen": 3153723392 + }, + { + "epoch": 0.91, + "learning_rate": 4.515151515151515e-05, + "loss": 2.497, + "theoretical_loss": 3.308088621671783, + "tokens_seen": 3153788928 + }, + { + "epoch": 0.91, + "learning_rate": 4.5131313131313134e-05, + "loss": 2.7172, + "theoretical_loss": 3.3080833898718236, + "tokens_seen": 3153854464 + }, + { + "epoch": 0.91, + "learning_rate": 4.511111111111111e-05, + "loss": 2.4372, + "theoretical_loss": 3.3080781582110177, + "tokens_seen": 3153920000 + }, + { + "epoch": 0.91, + "learning_rate": 4.5090909090909095e-05, + "loss": 2.65, + "theoretical_loss": 3.3080729266893583, + "tokens_seen": 3153985536 + }, + { + "epoch": 0.91, + "learning_rate": 4.507070707070707e-05, + "loss": 2.4883, + "theoretical_loss": 3.308067695306839, + "tokens_seen": 3154051072 + }, + { + "epoch": 0.91, + "learning_rate": 4.505050505050505e-05, + "loss": 2.5856, + "theoretical_loss": 3.3080624640634535, + "tokens_seen": 3154116608 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1772131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.818197011947632, + "objective/train/theoretical_loss": 3.3080572329591953, + "objective/train/tokens_used": 1513041376, + "theoretical_loss": 3.3080572329591953, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 4.5030303030303034e-05, + "loss": 2.5976, + "theoretical_loss": 3.3080572329591953, + "tokens_seen": 3154182144 + }, + { + "epoch": 0.91, + "learning_rate": 4.501010101010101e-05, + "loss": 2.6449, + "theoretical_loss": 3.308052001994057, + "tokens_seen": 3154247680 + }, + { + "epoch": 0.91, + "learning_rate": 4.498989898989899e-05, + "loss": 2.622, + "theoretical_loss": 3.3080467711680326, + "tokens_seen": 3154313216 + }, + { + "epoch": 0.91, + "learning_rate": 4.496969696969697e-05, + "loss": 2.5585, + "theoretical_loss": 3.308041540481115, + "tokens_seen": 3154378752 + }, + { + "epoch": 0.91, + "learning_rate": 4.494949494949495e-05, + "loss": 2.6214, + "theoretical_loss": 3.3080363099332986, + "tokens_seen": 3154444288 + }, + { + "epoch": 0.91, + "learning_rate": 4.492929292929293e-05, + "loss": 2.4835, + "theoretical_loss": 3.308031079524576, + "tokens_seen": 3154509824 + }, + { + "epoch": 0.91, + "learning_rate": 4.490909090909091e-05, + "loss": 2.2673, + "theoretical_loss": 3.308025849254941, + "tokens_seen": 3154575360 + }, + { + "epoch": 0.91, + "learning_rate": 4.488888888888889e-05, + "loss": 2.632, + "theoretical_loss": 3.3080206191243864, + "tokens_seen": 3154640896 + }, + { + "epoch": 0.91, + "learning_rate": 4.486868686868687e-05, + "loss": 2.4897, + "theoretical_loss": 3.3080153891329065, + "tokens_seen": 3154706432 + }, + { + "epoch": 0.91, + "learning_rate": 4.484848484848485e-05, + "loss": 2.6216, + "theoretical_loss": 3.308010159280494, + "tokens_seen": 3154771968 + }, + { + "epoch": 0.91, + "learning_rate": 4.4828282828282826e-05, + "loss": 2.3433, + "theoretical_loss": 3.3080049295671423, + "tokens_seen": 3154837504 + }, + { + "epoch": 0.91, + "learning_rate": 4.480808080808081e-05, + "loss": 2.2698, + "theoretical_loss": 3.3079996999928456, + "tokens_seen": 3154903040 + }, + { + "epoch": 0.91, + "learning_rate": 4.478787878787879e-05, + "loss": 2.7232, + "theoretical_loss": 3.3079944705575968, + "tokens_seen": 3154968576 + }, + { + "epoch": 0.91, + "learning_rate": 4.476767676767677e-05, + "loss": 2.2761, + "theoretical_loss": 3.3079892412613887, + "tokens_seen": 3155034112 + }, + { + "epoch": 0.91, + "learning_rate": 4.474747474747475e-05, + "loss": 2.4753, + "theoretical_loss": 3.307984012104216, + "tokens_seen": 3155099648 + }, + { + "epoch": 0.91, + "learning_rate": 4.4727272727272725e-05, + "loss": 2.4082, + "theoretical_loss": 3.3079787830860714, + "tokens_seen": 3155165184 + }, + { + "epoch": 0.91, + "learning_rate": 4.470707070707071e-05, + "loss": 2.571, + "theoretical_loss": 3.3079735542069484, + "tokens_seen": 3155230720 + }, + { + "epoch": 0.91, + "learning_rate": 4.4686868686868686e-05, + "loss": 2.3446, + "theoretical_loss": 3.30796832546684, + "tokens_seen": 3155296256 + }, + { + "epoch": 0.91, + "learning_rate": 4.466666666666667e-05, + "loss": 2.5655, + "theoretical_loss": 3.3079630968657403, + "tokens_seen": 3155361792 + }, + { + "epoch": 0.91, + "learning_rate": 4.464646464646465e-05, + "loss": 2.3712, + "theoretical_loss": 3.3079578684036424, + "tokens_seen": 3155427328 + }, + { + "epoch": 0.91, + "learning_rate": 4.4626262626262625e-05, + "loss": 2.6829, + "theoretical_loss": 3.307952640080539, + "tokens_seen": 3155492864 + }, + { + "epoch": 0.91, + "learning_rate": 4.460606060606061e-05, + "loss": 2.5801, + "theoretical_loss": 3.307947411896425, + "tokens_seen": 3155558400 + }, + { + "epoch": 0.91, + "learning_rate": 4.4585858585858586e-05, + "loss": 2.6777, + "theoretical_loss": 3.307942183851293, + "tokens_seen": 3155623936 + }, + { + "epoch": 0.91, + "learning_rate": 4.456565656565656e-05, + "loss": 2.5232, + "theoretical_loss": 3.3079369559451366, + "tokens_seen": 3155689472 + }, + { + "epoch": 0.91, + "learning_rate": 4.454545454545455e-05, + "loss": 2.7102, + "theoretical_loss": 3.307931728177949, + "tokens_seen": 3155755008 + }, + { + "epoch": 0.91, + "objective/train/docs_used": 1773306, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.426760673522949, + "objective/train/theoretical_loss": 3.3079265005497236, + "objective/train/tokens_used": 1514679776, + "theoretical_loss": 3.3079265005497236, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.91, + "learning_rate": 4.4525252525252524e-05, + "loss": 2.5396, + "theoretical_loss": 3.3079265005497236, + "tokens_seen": 3155820544 + }, + { + "epoch": 0.91, + "learning_rate": 4.450505050505051e-05, + "loss": 2.6083, + "theoretical_loss": 3.307921273060454, + "tokens_seen": 3155886080 + }, + { + "epoch": 0.91, + "learning_rate": 4.448484848484849e-05, + "loss": 2.5161, + "theoretical_loss": 3.3079160457101335, + "tokens_seen": 3155951616 + }, + { + "epoch": 0.91, + "learning_rate": 4.446464646464646e-05, + "loss": 2.4451, + "theoretical_loss": 3.307910818498756, + "tokens_seen": 3156017152 + }, + { + "epoch": 0.91, + "learning_rate": 4.4444444444444447e-05, + "loss": 2.6541, + "theoretical_loss": 3.307905591426314, + "tokens_seen": 3156082688 + }, + { + "epoch": 0.91, + "learning_rate": 4.4424242424242424e-05, + "loss": 2.694, + "theoretical_loss": 3.307900364492802, + "tokens_seen": 3156148224 + }, + { + "epoch": 0.91, + "learning_rate": 4.440404040404041e-05, + "loss": 2.6531, + "theoretical_loss": 3.307895137698212, + "tokens_seen": 3156213760 + }, + { + "epoch": 0.91, + "learning_rate": 4.4383838383838385e-05, + "loss": 2.4741, + "theoretical_loss": 3.307889911042539, + "tokens_seen": 3156279296 + }, + { + "epoch": 0.91, + "learning_rate": 4.436363636363636e-05, + "loss": 2.5313, + "theoretical_loss": 3.3078846845257757, + "tokens_seen": 3156344832 + }, + { + "epoch": 0.91, + "learning_rate": 4.4343434343434346e-05, + "loss": 2.6064, + "theoretical_loss": 3.307879458147915, + "tokens_seen": 3156410368 + }, + { + "epoch": 0.91, + "learning_rate": 4.432323232323232e-05, + "loss": 2.6529, + "theoretical_loss": 3.3078742319089516, + "tokens_seen": 3156475904 + }, + { + "epoch": 0.91, + "learning_rate": 4.43030303030303e-05, + "loss": 2.5587, + "theoretical_loss": 3.3078690058088775, + "tokens_seen": 3156541440 + }, + { + "epoch": 0.91, + "learning_rate": 4.4282828282828284e-05, + "loss": 2.4133, + "theoretical_loss": 3.307863779847687, + "tokens_seen": 3156606976 + }, + { + "epoch": 0.91, + "learning_rate": 4.426262626262626e-05, + "loss": 2.5082, + "theoretical_loss": 3.3078585540253735, + "tokens_seen": 3156672512 + }, + { + "epoch": 0.91, + "learning_rate": 4.4242424242424246e-05, + "loss": 2.6462, + "theoretical_loss": 3.3078533283419302, + "tokens_seen": 3156738048 + }, + { + "epoch": 0.91, + "learning_rate": 4.422222222222223e-05, + "loss": 2.7171, + "theoretical_loss": 3.3078481027973505, + "tokens_seen": 3156803584 + } + ], + "max_steps": 25000, + "num_train_epochs": 9223372036854775807, + "total_flos": 7.6305634689024e+17, + "trial_name": null, + "trial_params": null +}